diff --git a/.clang-format b/.clang-format index 9b3aa8b72..6347f2748 100644 --- a/.clang-format +++ b/.clang-format @@ -1 +1,4 @@ BasedOnStyle: LLVM +AllowShortLoopsOnASingleLine: true +AllowShortBlocksOnASingleLine: Empty +AllowShortIfStatementsOnASingleLine: AllIfsAndElse diff --git a/.github/actions/setup-mlir-python-bindings/action.yml b/.github/actions/setup-mlir-python-bindings/action.yml deleted file mode 100644 index c02ce6f8b..000000000 --- a/.github/actions/setup-mlir-python-bindings/action.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: setup-mlir-python-bindings -description: Fetch and setup MLIR Python bindings -runs: - using: "composite" - steps: - - name: Set LLVM hash - shell: bash - run: | - echo LLVM_HASH=$(cat cmake/llvm-version.txt) >>$GITHUB_ENV - - - name: Fetch requirements for python binding - uses: actions/checkout@v4 - with: - repository: llvm/llvm-project - ref: ${{ env.LLVM_HASH }} - sparse-checkout: mlir/python/requirements.txt - sparse-checkout-cone-mode: false - path: llvm-dep - - - name: Install requirements - shell: bash - run: python3 -m pip install -r llvm-dep/mlir/python/requirements.txt diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..052220c72 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: weekly + + - package-ecosystem: pip + directory: "/python" + schedule: + interval: weekly diff --git a/.github/dockerfiles/env/Dockerfile b/.github/dockerfiles/env/Dockerfile deleted file mode 100644 index 10be05425..000000000 --- a/.github/dockerfiles/env/Dockerfile +++ /dev/null @@ -1,141 +0,0 @@ -FROM python:3.9-bullseye as python_builder - -WORKDIR /app - -RUN set -ex; \ - git clone https://github.com/dirkcgrunwald/jupyter_codeserver_proxy-.git; \ - cd jupyter_codeserver_proxy-; \ - make build-package - -FROM ubuntu:22.04 as conda_builder - -RUN set -ex; \ - export DEBIAN_FRONTEND=noninteractive; \ - apt-get update -y; \ - apt-get install -y curl xz-utils pixz - -ENV USER=jovyan -ENV HOME=/home/${USER} -ENV SHELL=/bin/bash - -ARG CONDA_PREFIX=${HOME}/.conda -ARG CONDA=${CONDA_PREFIX}/bin/conda - -RUN adduser \ - --disabled-password \ - --gecos "Default user" \ - --uid 1000 \ - --home ${HOME} \ - --force-badname \ - ${USER} - -USER ${USER} - -WORKDIR ${HOME} - -# install miniforge to CONDA_PREFIX -RUN set -ex; \ - curl -sSL -o /tmp/miniforge.sh https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-x86_64.sh; \ - /bin/bash /tmp/miniforge.sh -b -p ${CONDA_PREFIX}; \ - rm -f /tmp/miniforge.sh - -RUN echo "conda prefix after initial installation: $(du --human-readable --summarize ${CONDA_PREFIX})" > /dev/stderr - -# install mamba solver -RUN ${CONDA} install -n base conda-libmamba-solver - -RUN echo "conda prefix after installing conda-libmamba-solver: $(du --human-readable --summarize ${CONDA_PREFIX})" > /dev/stderr - -# set libmamba as the default resolver -# https://conda.github.io/conda-libmamba-solver/getting-started/ -RUN ${CONDA} config --set solver libmamba - -# update to conda-forge -RUN ${CONDA} update -n base -c conda-forge conda - -RUN echo "conda prefix after updating to conda-forge: $(du --human-readable --summarize ${CONDA_PREFIX})" > /dev/stderr - -COPY --chown=${USER}:${USER} requirements.txt /tmp/requirements.txt -COPY --from=python_builder --chown=${USER}:${USER} /app/jupyter_codeserver_proxy-/dist/*.whl /opt/jupyter_codeserver_proxy/dist/ - -# install nb_conda_kernels for discovering kernels (https://towardsdatascience.com/get-your-conda-environment-to-show-in-jupyter-notebooks-the-easy-way-17010b76e874) -# install all dependencies -# disable JupyterLab announcement pop-up (https://jupyterlab.readthedocs.io/en/stable/user/announcements.html) -RUN set -e; \ - . ${CONDA_PREFIX}/etc/profile.d/conda.sh; \ - conda activate base; \ - conda install nb_conda_kernels nodejs; \ - pip install --no-cache-dir -r /tmp/requirements.txt /opt/jupyter_codeserver_proxy/dist/*.whl; \ - jupyter labextension disable "@jupyterlab/apputils-extension:announcements" - -RUN echo "conda prefix after installing jupyterhub: $(du --human-readable --summarize ${CONDA_PREFIX})" > /dev/stderr - -# create new conda environment python-3.9 -# install infractl, lit -RUN set -e; \ - . ${CONDA_PREFIX}/etc/profile.d/conda.sh; \ - conda create --name python-3.9 python=3.9 ipykernel s3cmd dvc numpy; \ - conda activate python-3.9; \ - pip --no-cache-dir install infractl lit - -RUN echo "conda prefix after installing infractl: $(du --human-readable --summarize ${CONDA_PREFIX})" > /dev/stderr - -RUN ${CONDA} clean --all --force-pkgs-dirs --yes - -RUN echo "conda prefix after cleaning: $(du --human-readable --summarize ${CONDA_PREFIX})" > /dev/stderr - -# archive conda prefix -RUN tar c -I"pixz -9" -f conda.tar.xz .conda - -RUN echo "conda archive: $(du --human-readable conda.tar*)" > /dev/stderr - -FROM ubuntu:22.04 - -ENV USER=jovyan -ENV HOME=/home/${USER} -ENV SHELL=/bin/bash - -RUN set -ex; \ - export DEBIAN_FRONTEND=noninteractive; \ - apt-get update -y; \ - apt-get install -y software-properties-common curl; \ - curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg > /usr/share/keyrings/githubcli-archive-keyring.gpg; \ - echo "deb [arch=amd64 signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" > /etc/apt/sources.list.d/github-cli.list; \ - apt-get update -y; \ - apt-get install -y --no-install-recommends \ - ca-certificates \ - curl \ - gh jq \ - git \ - bash-completion \ - cmake gcc g++ ninja-build git clang-format \ - sudo \ - pixz \ - ; \ - rm -rf /var/lib/apt/lists/* - -RUN set -ex; \ - adduser \ - --disabled-password \ - --gecos "Default user" \ - --uid 1000 \ - --home ${HOME} \ - --force-badname \ - ${USER}; \ - usermod -aG sudo ${USER}; \ - echo "%sudo ALL=(ALL:ALL) NOPASSWD:ALL" > /etc/sudoers - -COPY --from=conda_builder ${HOME}/conda.tar.xz /template/ - -COPY entrypoint.sh /template/ - -USER ${USER} - -WORKDIR ${HOME} - -RUN curl -fsSL https://code-server.dev/install.sh | sh && rm -rf "${HOME}/.cache" - -EXPOSE 8888 - -ENTRYPOINT ["/bin/bash", "/template/entrypoint.sh"] - diff --git a/.github/dockerfiles/env/build.sh b/.github/dockerfiles/env/build.sh deleted file mode 100644 index c0de12ee3..000000000 --- a/.github/dockerfiles/env/build.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -REGISTRY=localhost:5000 -TAG=graph-compiler-env:0.0.13 - -set -e - -kubectl -n docker-registry port-forward svc/docker-registry 5000:5000 & - -cd $(dirname "$0") -docker build --progress=plain . --tag $REGISTRY/$TAG -docker push $REGISTRY/$TAG diff --git a/.github/dockerfiles/env/entrypoint.sh b/.github/dockerfiles/env/entrypoint.sh deleted file mode 100644 index 0a11100bd..000000000 --- a/.github/dockerfiles/env/entrypoint.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash - -set -e -set -vx - -conda_prefix=$HOME/.conda -jupyterlab_env="base" -kernel_env="python-3.9" - -# Fix the owner and permissions for /home/jovyan -# TODO: check if passwordless sudo is enabled -sudo chown jovyan:jovyan /home/jovyan -sudo chmod 00700 /home/jovyan - -cd ~ - -if [[ ! -d $conda_prefix ]]; then - echo "$conda_prefix does not exists, copying from /template" - tar x -Ipixz -f /template/conda.tar.xz -fi - -if [[ ! -f ~/.profile ]]; then - echo "No .profile in $HOME, creating one" - echo "source $conda_prefix/etc/profile.d/conda.sh" >> ~/.profile - echo "conda activate python-3.9" >> ~/.profile -fi - -if [[ ! -f ~/.jupyter/jupyter_config.json ]]; then - echo "no .jupyter/jupyter_config.json in $HOME, creating one" - mkdir -p ~/.jupyter - # Use kernel display_name, filter out a kernel from jupyterlab environment - cat < ~/.jupyter/jupyter_config.json -{ - "CondaKernelSpecManager": { - "conda_only": true, - "name_format": "{display_name}" - } -} -EOF - - # Use environment name for display_name - kernel_json=$conda_prefix/envs/$kernel_env/share/jupyter/kernels/python3/kernel.json - jq ".display_name = \"$kernel_env\"" $kernel_json > /tmp/kernel.json - mv /tmp/kernel.json $kernel_json - unset kernel_json -fi - -source $conda_prefix/etc/profile.d/conda.sh -conda activate $jupyterlab_env - -unset conda_prefix -unset jupyterlab_env -unset kernel_env - -# Remove default kernel -jupyter kernelspec remove -y python3 || true - -# Remove lost+found -rm -rf lost+found - -exec jupyterhub-singleuser diff --git a/.github/dockerfiles/env/requirements.txt b/.github/dockerfiles/env/requirements.txt deleted file mode 100644 index aafc8c6aa..000000000 --- a/.github/dockerfiles/env/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -# See also -# https://github.com/jupyterhub/zero-to-jupyterhub-k8s/blob/main/images/singleuser-sample/requirements.in -# https://github.com/jupyterhub/zero-to-jupyterhub-k8s/blob/main/images/singleuser-sample/requirements.txt -jupyterhub==4.0.2 -jupyterlab -nbclassic -nbgitpuller -jupyter-server-proxy diff --git a/.github/dockerfiles/runner/Dockerfile b/.github/dockerfiles/runner/Dockerfile deleted file mode 100644 index 3652fa06d..000000000 --- a/.github/dockerfiles/runner/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -FROM summerwind/actions-runner:ubuntu-22.04 - -USER root - -RUN gpasswd --delete runner sudo - -RUN set -ex; \ - export DEBIAN_FRONTEND=noninteractive; \ - echo "deb [arch=amd64 signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" > /etc/apt/sources.list.d/github-cli.list; \ - curl -sSL https://cli.github.com/packages/githubcli-archive-keyring.gpg > /usr/share/keyrings/githubcli-archive-keyring.gpg; \ - apt-get update -y; \ - apt-get install -y --no-install-recommends --fix-missing \ - python3-pip python3-dev \ - cmake gcc g++ ninja-build git clang-format \ - gh \ - libomp-dev \ - ; \ - rm -rf /var/lib/apt/lists/* - -RUN pip install lit numpy - -USER runner -WORKDIR $HOME - -SHELL ["/bin/bash", "-xec"] - diff --git a/.github/dockerfiles/runner/build.sh b/.github/dockerfiles/runner/build.sh deleted file mode 100644 index c14bb54a4..000000000 --- a/.github/dockerfiles/runner/build.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -REGISTRY=localhost:5000 -TAG=graph-compiler-runner:0.0.2 - -kubectl -n docker-registry port-forward svc/docker-registry 5000:5000 & - -cd $(dirname "$0") -docker build . --tag $REGISTRY/$TAG -docker push $REGISTRY/$TAG - diff --git a/.github/workflows/build-gpu-imex.yml b/.github/workflows/build-gpu-imex.yml deleted file mode 100644 index 180241e2d..000000000 --- a/.github/workflows/build-gpu-imex.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Graph Compiler build - -on: - workflow_dispatch: - push: - branches: - - main - pull_request: - branches: - - main - -permissions: read-all - -jobs: - build: - name: Build GPU with IMEX - runs-on: [self-hosted, l0] - - env: - GH_TOKEN: ${{ github.token }} - - steps: - - uses: actions/checkout@v4 - - - name: Setup MLIR Python bindings - id: setup-mlir-python-bindings - uses: ./.github/actions/setup-mlir-python-bindings - - - name: Build - run: | - scripts/compile.sh --imex - - - name: Test - run: | - cmake --build build --target gc-check diff --git a/.github/workflows/build-llvm.yml b/.github/workflows/build-llvm.yml deleted file mode 100644 index 730fce043..000000000 --- a/.github/workflows/build-llvm.yml +++ /dev/null @@ -1,74 +0,0 @@ -name: LLVM Build - -on: - workflow_dispatch: - push: - paths: - - cmake/llvm-version.txt - - cmake/llvm-version-imex.txt - - .github/workflows/build-llvm.yml - -permissions: read-all - -jobs: - build: - name: Build - runs-on: [self-hosted] - strategy: - matrix: - build-type: [CPU, IMEX] - - steps: - - uses: actions/checkout@v4 - - - name: Set LLVM hash - run: | - if [ "${{ matrix.build-type }}" = "IMEX" ]; then - echo LLVM_HASH=$(cat cmake/llvm-version-imex.txt) >>$GITHUB_ENV - echo IMEX_HASH=$(cat cmake/imex-version.txt) >>$GITHUB_ENV - else - echo LLVM_HASH=$(cat cmake/llvm-version.txt) >>$GITHUB_ENV - fi - - - uses: actions/checkout@v4 - with: - repository: intel/mlir-extensions - ref: ${{ env.IMEX_HASH }} - path: mlir-extensions - if: ${{ matrix.build-type == 'IMEX' }} - - - uses: actions/checkout@v4 - with: - repository: llvm/llvm-project - ref: ${{ env.LLVM_HASH }} - path: llvm-project - - - name: Build - run: | - cd llvm-project - if [ "${{ matrix.build-type }}" = "IMEX" ]; then - git apply ../mlir-extensions/build_tools/patches/* - fi - python3 -m pip install -r mlir/python/requirements.txt - mkdir llvm-install - cmake -G Ninja llvm -B build -DCMAKE_INSTALL_PREFIX=llvm-install -DMLIR_ENABLE_BINDINGS_PYTHON=ON -DPython3_EXECUTABLE=$(which python3) \ - -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=true -DLLVM_ENABLE_PROJECTS="mlir" -DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD="SPIRV" -DLLVM_TARGETS_TO_BUILD="X86" -DLLVM_INSTALL_UTILS=true -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DLLVM_INSTALL_GTEST=ON - cmake --build build --target install - cd llvm-install - tar -zcf ../llvm.tgz . - - - name: Upload LLVM for CPU - if: matrix.build-type == 'CPU' - uses: actions/upload-artifact@v4 - with: - name: llvm-${{ env.LLVM_HASH }} - path: ./llvm-project/llvm.tgz - - - name: Upload LLVM for IMEX - if: matrix.build-type == 'IMEX' - uses: actions/upload-artifact@v4 - with: - name: llvm-${{ env.LLVM_HASH }}-imex-patched - path: ./llvm-project/llvm.tgz - - diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index f4773685b..000000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: Graph Compiler build - -on: - workflow_dispatch: - push: - branches: - - main - pull_request: - branches: - - main - -permissions: read-all - -jobs: - build: - name: Build - runs-on: [self-hosted] - - env: - GH_TOKEN: ${{ github.token }} - - steps: - - uses: actions/checkout@v4 - - - name: Setup MLIR Python bindings - id: setup-mlir-python-bindings - uses: ./.github/actions/setup-mlir-python-bindings - - - name: Build - run: | - scripts/compile.sh - - - name: Test - run: | - cmake --build build --target gc-check - - - name: Build and install benchgc - working-directory: build - run: | - ninja benchgc - pip uninstall -y benchgc || true - pip install test/benchgc/dist/benchgc-*.whl - - name: Correctness Test - env: - LD_PRELOAD: /lib/x86_64-linux-gnu/libomp5.so - run: | - scripts/correctness.sh \ No newline at end of file diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml deleted file mode 100644 index 43df7e9b8..000000000 --- a/.github/workflows/clang-tidy.yml +++ /dev/null @@ -1,105 +0,0 @@ -name: check clang-tidy - -permissions: - contents: read - -on: - pull_request: - -jobs: - clang_tidy: - runs-on: "ubuntu-latest" - env: - GH_TOKEN: ${{ github.token }} - - steps: - - name: Install OpenMP - run: "sudo apt install -y libomp-dev opencl-c-headers" - - - name: Fetch sources - uses: actions/checkout@v4 - with: - path: 'graph-compiler' - fetch-depth: 0 - submodules: true - - - name: Fetch code tidy utils - uses: actions/checkout@v4 - with: - repository: 'llvm/llvm-project' - ref: 'main' - sparse-checkout: | - clang-tools-extra/clang-tidy/tool/run-clang-tidy.py - mlir/python/requirements.txt - mlir/.clang-tidy - sparse-checkout-cone-mode: false - path: llvm-project - - - name: Read llvm version and run id - shell: bash - run: | - echo LLVM_HASH=$(cat graph-compiler/cmake/llvm-version.txt) >> $GITHUB_ENV - - - name: Fetch llvm artifact - run: | - gh run download \ - --repo ${{ github.repository }} \ - -n "llvm-${{ env.LLVM_HASH }}" \ - --dir llvm - tar -zxf llvm/llvm.tgz - - - name: Unwrap pre-built llvm - run: | - cd llvm - tar -zxf llvm.tgz - - - name: Get merge base - run: | - cd graph-compiler - echo "MERGE_BASE=$(git merge-base ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }})" >> $GITHUB_ENV - - - name: Get changed files - run: | - cd graph-compiler - echo "CHANGED_FILES=$(git diff --name-only --diff-filter=d $MERGE_BASE ${{ github.event.pull_request.head.sha }} | paste -sd' ')" >> $GITHUB_ENV - - - name: Prepare Environment - shell: bash - run: | - python3 -m pip install -r llvm-project/mlir/python/requirements.txt - python3 -m pip install lit - - - name: Prepare compile commands - shell: bash - run: | - mkdir build - cd build - cmake ../graph-compiler \ - -DCMAKE_BUILD_TYPE=Release \ - -DMLIR_DIR=$(pwd)/../llvm/lib/cmake/mlir \ - -DCMAKE_EXPORT_COMPILE_COMMANDS=True \ - -DCMAKE_C_COMPILER=$(which clang) \ - -DCMAKE_CXX_COMPILER=$(which clang++) \ - -DLLVM_EXTERNAL_LIT=$(which lit) \ - -DDNNL_USE_CLANG_SANITIZER="Undefined" - - - name: Prepare inc file - run: | - cd build - for f in $(find ./include -name Makefile); do - set +e; - targets=$(make -f $f help |grep IncGen); - if [[ $? -eq 0 ]]; then - set -e; - for target in $targets; do - cd ${f%Makefile} && make ${target#...} && cd -; - done - fi ; - set -e; - done - - - name: Perform clang-tidy check - shell: bash - run: | - cd build - python3 ../llvm-project/clang-tools-extra/clang-tidy/tool/run-clang-tidy.py -warnings-as-errors=* -p ./ -config-file ../llvm-project/mlir/.clang-tidy -clang-tidy-binary $(which clang-tidy-16) ${{ env.CHANGED_FILES }} diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml deleted file mode 100644 index a50ed495f..000000000 --- a/.github/workflows/coverity.yml +++ /dev/null @@ -1,104 +0,0 @@ -name: Coverity - -on: - workflow_dispatch: - schedule: - - cron: "5 2 * * *" - -permissions: read-all - -jobs: - coverity: - name: Coverity - env: - GH_TOKEN: ${{ github.token }} - runs-on: - - self-hosted - defaults: - run: - shell: bash -noprofile --norc -eo pipefail {0} - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Download coverity - env: - COVERITY_TOKEN: ${{ secrets.COVERITY_TOKEN }} - run: | - cd $HOME - test ! -f coverity/bin/cov-build && { - curl -X POST https://scan.coverity.com/download/linux64 -d "token=$COVERITY_TOKEN&project=intel%2Fgraph-compiler" -o coverity.tgz - tar zxf coverity.tgz - mv -T cov-analysis-linux64-* coverity - } - - - name: Add coverity to PATH - run: | - echo "$HOME/coverity/bin" >> $GITHUB_PATH - - - name: Show coverity version - run: | - coverity --version - - - name: Add coverity inputs to env - run: | - version_id=$(grep 'project(GraphCompiler VERSION' CMakeLists.txt | awk '{print $3}' | tr -d '"') - version_local_id="$(git rev-parse --short HEAD)" - version="${version_id}+git${version_local_id}" - email="vadim.musin@intel.com" - project_id=30281 - - echo "project_id=$project_id" | tee -a $GITHUB_ENV - echo "email=$email" | tee -a $GITHUB_ENV - echo "version=$version" | tee -a $GITHUB_ENV - - - name: Setup MLIR Python bindings - id: setup-mlir-python-bindings - uses: ./.github/actions/setup-mlir-python-bindings - - - name: Run coverity build - run: | - - cov-build --dir $HOME/cov-int scripts/compile.sh - tail $HOME/cov-int/build-log.txt - - - name: Create coverity results tarball - run: | - cd $HOME - tar zcf cov-int.tgz cov-int - - - name: Version for coverity build - run: | - - - - name: Create coverity build - env: - COVERITY_TOKEN: ${{ secrets.COVERITY_TOKEN }} - run: | - cd $HOME - curl -X POST \ - -d version="$version" \ - -d email="$email" \ - -d token=$COVERITY_TOKEN \ - -d file_name="cov-int.tgz" \ - https://scan.coverity.com/projects/$project_id/builds/init \ - | tee response - upload_url="$(jq -r '.url' response)" - build_id="$(jq -r '.build_id' response)" - echo "upload_url=$upload_url" >> $GITHUB_ENV - echo "build_id=$build_id" | tee -a $GITHUB_ENV - - - name: Upload coverity build - env: - COVERITY_TOKEN: ${{ secrets.COVERITY_TOKEN }} - run: | - cd $HOME - curl -X PUT \ - --header 'Content-Type: application/json' \ - --upload-file cov-int.tgz \ - $upload_url - - curl -X PUT \ - -d token=$COVERITY_TOKEN \ - https://scan.coverity.com/projects/$project_id/builds/$build_id/enqueue diff --git a/.github/workflows/license.yml b/.github/workflows/license.yml deleted file mode 100644 index 0309a6dfd..000000000 --- a/.github/workflows/license.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: Graph Compiler License Check - -on: - pull_request: - -permissions: read-all - -jobs: - license-check: - name: License Check - runs-on: ubuntu-latest - permissions: - contents: write - pull-requests: write - repository-projects: write - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: true - - - name: Get merge base - run: | - echo "MERGE_BASE=`git merge-base ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }}`" >> $GITHUB_ENV - - - name: Get changed files - run: | - echo "CHANGED_FILES=`git diff --name-only --diff-filter=d $MERGE_BASE ${{ github.event.pull_request.head.sha }} | paste -sd,`" >> $GITHUB_ENV - - - name: Perform license check - run: "python scripts/license.py --files $CHANGED_FILES" diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml new file mode 100644 index 000000000..ed1cdf6a7 --- /dev/null +++ b/.github/workflows/smoke.yml @@ -0,0 +1,136 @@ +name: Build and smoke test + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + build: + runs-on: ubuntu-26.04 + env: + GC_BUILD_TYPE: Release + LLVM_BUILD_TYPE: Release + LLVM_BUILD_TARGET: install + CMAKE_C_COMPILER_LAUNCHER: ccache + CMAKE_CXX_COMPILER_LAUNCHER: ccache + CCACHE_SLOPPINESS: include_file_ctime,include_file_mtime + CCACHE_COMPILERCHECK: content + + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: ${{ github.event_name == 'pull_request' && 2 || 0 }} + persist-credentials: false + + - name: Install formatting tools and build dependencies + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends libze-dev ccache + echo "CCACHE_ROOT=$RUNNER_TEMP/ccache" >> "$GITHUB_ENV" + python3 -m venv .venv + echo "$PWD/.venv/bin" >> "$GITHUB_PATH" + . .venv/bin/activate + pip install cmake==4.4.4 nanobind==2.9 lit==23.1.3 \ + clang-format==23.1.3 cmake-format==0.6.13 ruff==0.17.1 + + - name: List changed files + env: + BASE: ${{ github.event_name == 'pull_request' && 'HEAD^1' || github.event.before }} + run: | + if git rev-parse -q --verify "$BASE^{commit}" >/dev/null; then + git diff --name-only --diff-filter=d -z "$BASE" HEAD > "$RUNNER_TEMP/changed-files" + else + git ls-files -z > "$RUNNER_TEMP/changed-files" + fi + + - name: clang-format + run: | + sed -z -nE '/\.(h|cpp)$/p' "$RUNNER_TEMP/changed-files" \ + | xargs -0 -r -P "$(nproc)" clang-format --dry-run --Werror + + - name: cmake-format + run: | + sed -z -nE '/(^|\/)(CMakeLists\.txt|[^/]+\.cmake)$/p' "$RUNNER_TEMP/changed-files" \ + | xargs -0 -r -P "$(nproc)" cmake-format --check + + - name: ruff-format + run: | + sed -z -nE '/\.py$/p' "$RUNNER_TEMP/changed-files" \ + | xargs -0 -r -P "$(nproc)" ruff format --check + + # Caches are saved right after each build, so later failures do not discard them. + # Eviction runs only after a successful build: entries unused by a complete build are stale. + # A failed build still saves its ccache, unevicted, to keep the work done so far. + - uses: actions/cache/restore@v6 + id: llvm + with: + path: externals/llvm/${{ env.LLVM_BUILD_TYPE }} + key: llvm-${{ env.LLVM_BUILD_TYPE }}-${{ hashFiles('cmake/llvm-version.txt', 'patches/*.patch', 'scripts/build.sh') }} + + - uses: actions/cache/restore@v6 + if: steps.llvm.outputs.cache-hit != 'true' + id: llvm-ccache + with: + path: ${{ env.CCACHE_ROOT }}/llvm + key: ccache-llvm-${{ env.LLVM_BUILD_TYPE }}-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: ccache-llvm-${{ env.LLVM_BUILD_TYPE }}- + + # Build steps output cache_miss; a ccache without misses is not saved again. + - name: Build LLVM + if: steps.llvm.outputs.cache-hit != 'true' + id: llvm-build + env: + CCACHE_DIR: ${{ env.CCACHE_ROOT }}/llvm + GC_BUILD_SKIP: 1 + run: &build | + ccache -z + start=$EPOCHSECONDS + ./scripts/build.sh + ccache --evict-older-than "$((EPOCHSECONDS - start + 1))s" + ccache -s + ccache --print-stats | awk -F'\t' '$1 == "cache_miss" { print $1 "=" $2 }' >> "$GITHUB_OUTPUT" + + - uses: actions/cache/save@v6 + if: steps.llvm.outputs.cache-hit != 'true' + with: + path: externals/llvm/${{ env.LLVM_BUILD_TYPE }} + key: ${{ steps.llvm.outputs.cache-primary-key }} + + - uses: actions/cache/save@v6 + if: ${{ !cancelled() && (steps.llvm-build.outputs.cache_miss > 0 || steps.llvm-build.outcome == 'failure') }} + with: + path: ${{ env.CCACHE_ROOT }}/llvm + key: ${{ steps.llvm-ccache.outputs.cache-primary-key }} + + - uses: actions/cache/restore@v6 + id: gc-ccache + with: + path: ${{ env.CCACHE_ROOT }}/gc + key: ccache-gc-${{ env.GC_BUILD_TYPE }}-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: ccache-gc-${{ env.GC_BUILD_TYPE }}- + + - name: Build and install GC + id: gc-build + env: + CCACHE_DIR: ${{ env.CCACHE_ROOT }}/gc + LLVM_BUILD_SKIP: 1 + run: *build + + - uses: actions/cache/save@v6 + if: ${{ !cancelled() && (steps.gc-build.outputs.cache_miss > 0 || steps.gc-build.outcome == 'failure') }} + with: + path: ${{ env.CCACHE_ROOT }}/gc + key: ${{ steps.gc-ccache.outputs.cache-primary-key }} + + - name: Run lit tests + timeout-minutes: 30 + run: cmake --build "build/$GC_BUILD_TYPE" --target gc-check diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml deleted file mode 100644 index 91efdebf6..000000000 --- a/.github/workflows/style.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: Graph Compiler Style Check - -on: - push: - branches: - - main - pull_request: - branches: - - main - -permissions: read-all - -jobs: - build: - name: Style Check - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Check - run: | - clang-format --version - find . -name *.cpp -or -name *.hpp | xargs clang-format --dry-run --Werror -style=file - - python_format: - runs-on: ubuntu-latest - steps: - - name: checkout base version - uses: actions/checkout@v4 - with: - fetch-depth: 100 - ref: ${{ github.event.pull_request.base.sha }} - - - name: checkout head version - uses: actions/checkout@v4 - with: - fetch-depth: 100 - ref: ${{ github.event.pull_request.head.sha }} - - - name: install darker - run: "python3 -m pip install darker darker[isort] darker[flynt]" - - - name: check python format - run: "python3 -m darker --check -i -f --diff -r `git merge-base ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }}`...HEAD ." \ No newline at end of file diff --git a/.gitignore b/.gitignore index 40c724cd9..c7df4b2cf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,10 @@ .*/ +.venv !.github/ +!.gitignore build/ externals/ +graph_compiler.egg-info/ +python/src/*.so compile_commands.json -__pycache__ \ No newline at end of file +__pycache__ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..7ee9c3208 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,27 @@ +repos: + - repo: local + hooks: + - id: clang-format + name: clang-format + entry: clang-format --dry-run --Werror + language: system + files: '\.(h|cpp)$' + + - id: cmake-format + name: cmake-format + entry: cmake-format --check + language: system + files: '(CMakeLists\.txt|.+\.cmake)$' + + - id: ruff-format + name: ruff-format + entry: ruff format --check + language: system + files: '\.py$' + + - id: clang-tidy + name: clang-tidy + entry: clang-tidy -p build --warnings-as-errors='*' + language: system + files: '\.(h|cpp)$' + require_serial: false diff --git a/CMakeLists.txt b/CMakeLists.txt index a404d2345..7c0a77a1a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,151 +1,166 @@ -################################################################################ +# ############################################################################## # Copyright (C) 2025 Intel Corporation # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at # # http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions -# and limitations under the License. -# SPDX-License-Identifier: Apache-2.0 -################################################################################ +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. SPDX-License-Identifier: Apache-2.0 +# ############################################################################## cmake_minimum_required(VERSION 3.20) -project(GraphCompiler VERSION "0.1.0" LANGUAGES C CXX) +project( + GraphCompiler + VERSION "1" + LANGUAGES C CXX) -############################# Cmake options #################################### -set(CMAKE_EXPORT_COMPILE_COMMANDS ON) -set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0") -list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake") - -# Silence a false positive GCC -Wunused-but-set-parameter warning in constexpr -# cases, by marking SelectedCase as used. See -# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=85827 for details. The issue is -# fixed in GCC 10. -if(CMAKE_CXX_COMPILER_ID MATCHES "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS "10.0") - include(CheckCXXCompilerFlag) - check_cxx_compiler_flag("-Wno-unused-but-set-parameter" CXX_SUPPORTS_WNO_UNUSED_BUT_SET_PARAMETER) - if(CXX_SUPPORTS_WNO_UNUSED_BUT_SET_PARAMETER) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-but-set-parameter") - endif() -endif() -################################################################################ +include(GNUInstallDirs) +include(CMakePackageConfigHelpers) -############################ Build options ##################################### -option(GC_ENABLE_LEGACY ON) -option(GC_ENABLE_DNNL_API "Enable the oneDNN library API integration" ON) +# ########################### Build options #################################### option(GC_ENABLE_TEST "Build the tests" ON) -option(GC_ENABLE_TEST_DNNL_API "Build the dnnl tests" ${GC_ENABLE_DNNL_API}) -option(GC_ENABLE_TEST_MLIR "Build the mlir tests" ON) option(GC_ENABLE_TOOLS "Build the tools" ON) option(GC_ENABLE_OPT "Build gc-opt" ${GC_ENABLE_TOOLS}) -option(GC_ENABLE_IMEX "Enable Intel® Extension for MLIR (implicitly enables GPU compilation)" OFF) -option(GC_ENABLE_GPU "Enable GPU runtime and tools components" OFF) -option(GC_ENABLE_BINDINGS_PYTHON "Enable Graph Complier Python Binding" ON) -option(GC_DEV_LINK_LLVM_DYLIB "Link dynamic libraries of LLVM and MLIR. For developers only. Do not use it in packing the library." OFF) -option(GC_ENABLE_RUNTIME_NAIVE_BRGEMM "Use naive BRGEMM as runtime backend for debug purpose." OFF) -option(GC_BENCH_ENABLE "Build benchgc." ON) - -if(GC_ENABLE_LEGACY) - add_subdirectory(legacy/core) -endif() +option(GC_ENABLE_GPU_RUNNER "Build GPU runner" ${GC_ENABLE_TOOLS}) +option(GC_ENABLE_PYTHON_BINDINGS "Build Python bindings" ON) +option(GC_DYLINK "Link dynamic libraries of LLVM and MLIR" OFF) +set(GC_EXTERNALS_DIR + "${PROJECT_SOURCE_DIR}/externals" + CACHE PATH "Directory for external dependencies") +set(MLIR_DIR + "${GC_EXTERNALS_DIR}/llvm-project/build/lib/cmake/mlir" + CACHE PATH "MLIR cmake directory") +set(LLVM_DIR + "${MLIR_DIR}/../llvm" + CACHE PATH "LLVM cmake directory") +# ############################################################################## -if (GC_ENABLE_GPU) - set(GC_ENABLE_GPU ON) +# ############################# Cmake options ################################## +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_FLAGS_DEBUG "-g -O0") +set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG -ffunction-sections -fdata-sections") +set(CMAKE_CXX_FLAGS_RELWITHDEBINFO + "-O2 -g -DNDEBUG -ffunction-sections -fdata-sections") +if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + add_compile_options(-Werror -Werror=missing-declarations) endif() -if (GC_ENABLE_IMEX) - # normalize the value for lit config - set(GC_ENABLE_IMEX ON) +# Recent MLIR marks some of the default op builders that mlir-tblgen generates +# as deprecated, and the generated *.inc code itself calls them from other +# deprecated builders. Our sources never call these builders, so there is +# nothing to fix on our side. Clang ignores calls made from inside deprecated +# functions, but GCC warns about them and -Werror turns the warning into a build +# failure. Keep them as plain warnings for GCC. +if(CMAKE_CXX_COMPILER_ID MATCHES "GNU") + add_compile_options(-Wno-error=deprecated-declarations) endif() -if(GC_ENABLE_DNNL_API) - set(GC_ONEDNN_DIALECT_LIB_NAME MLIROneDNNGraph) - # normalize the value for lit config - set(GC_ENABLE_DNNL_API ON) +# Silence a false positive GCC -Wunused-but-set-parameter warning +if(CMAKE_CXX_COMPILER_ID MATCHES "GNU" AND CMAKE_CXX_COMPILER_VERSION + VERSION_LESS "10.0") + include(CheckCXXCompilerFlag) + check_cxx_compiler_flag("-Wno-unused-but-set-parameter" + CXX_SUPPORTS_WNO_UNUSED_BUT_SET_PARAMETER) + if(CXX_SUPPORTS_WNO_UNUSED_BUT_SET_PARAMETER) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-but-set-parameter") + endif() endif() -################################################################################ +# ############################################################################## -############################## Targets ######################################### +# ############################# Targets ######################################## # All common options, includes etc. are added to this interface target. add_library(GcInterface INTERFACE) -if (GC_ENABLE_GPU) - target_compile_options(GcInterface INTERFACE -DGC_USE_GPU) -endif() target_compile_features(GcInterface INTERFACE cxx_std_17) -target_include_directories(GcInterface INTERFACE - $ - $ - $ -) +target_include_directories( + GcInterface + INTERFACE $ + $ + $) +if(NOT GC_DYLINK AND NOT CMAKE_BUILD_TYPE STREQUAL "Release") + target_compile_options(GcInterface INTERFACE -ffunction-sections + -fdata-sections) +endif() +list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake") include(functions) -include(version) include(mlir) +find_package(Git QUIET) +add_custom_target( + GcVersion ALL + COMMAND + ${CMAKE_COMMAND} -DPROJECT_SOURCE_DIR="${PROJECT_SOURCE_DIR}" + -DBUILD_DIR="${PROJECT_BINARY_DIR}" -DGIT_EXECUTABLE="${GIT_EXECUTABLE}" + -DVERSION_CODE="${PROJECT_VERSION_MAJOR}" + -DLLVM_VERSION_MAJOR="${LLVM_VERSION_MAJOR}" + -DLLVM_VERSION_SUFFIX="${LLVM_VERSION_SUFFIX}" -P + "${PROJECT_SOURCE_DIR}/cmake/version.cmake" + BYPRODUCTS "${PROJECT_BINARY_DIR}/include/gc/Version.h" + COMMENT "Generating Version.h") +add_dependencies(GcInterface GcVersion) + add_subdirectory(include) add_subdirectory(lib) add_subdirectory(src) -add_subdirectory(python) add_subdirectory(test) -################################################################################ +# ############################################################################## -############################### Install ######################################## +# ############################## Install ####################################### install(DIRECTORY ${PROJECT_SOURCE_DIR}/include/ TYPE INCLUDE) -install(DIRECTORY ${PROJECT_BINARY_DIR}/include/ TYPE INCLUDE +install( + DIRECTORY ${PROJECT_BINARY_DIR}/include/ + TYPE INCLUDE REGEX "CMake.*|.*cmake" EXCLUDE) # Export the targets get_property(GC_TOOLS GLOBAL PROPERTY GC_TOOLS) -get_property(GC_MLIR_LIBS GLOBAL PROPERTY GC_MLIR_LIBS) -get_property(GC_PASS_LIBS GLOBAL PROPERTY GC_PASS_LIBS) -get_property(GC_DIALECT_LIBS GLOBAL PROPERTY GC_DIALECT_LIBS) -get_property(IMEX_LIBS GLOBAL PROPERTY IMEX_LIBS) -install(TARGETS - GcInterface - ${GC_TOOLS} - ${GC_MLIR_LIBS} - ${GC_PASS_LIBS} - ${GC_DIALECT_LIBS} - ${IMEX_LIBS} +if(GC_DYLINK) + set(GC_LIBS ${PROJECT_NAME}) +else() + get_property(GC_MLIR_LIBS GLOBAL PROPERTY GC_MLIR_LIBS) + get_property(GC_PASS_LIBS GLOBAL PROPERTY GC_PASS_LIBS) + get_property(GC_DIALECT_LIBS GLOBAL PROPERTY GC_DIALECT_LIBS) + set(GC_LIBS ${PROJECT_NAME} ${GC_MLIR_LIBS} ${GC_PASS_LIBS} + ${GC_DIALECT_LIBS}) +endif() + +install( + TARGETS GcInterface GcLevelZero ${GC_TOOLS} ${GC_LIBS} EXPORT ${PROJECT_NAME}Targets ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} - INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} - PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} -) + INCLUDES + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) + export(EXPORT ${PROJECT_NAME}Targets - FILE "${PROJECT_BINARY_DIR}/${PROJECT_NAME}Targets.cmake" -) -install(EXPORT ${PROJECT_NAME}Targets - FILE ${PROJECT_NAME}Targets.cmake - DESTINATION lib/cmake/${PROJECT_NAME} -) + FILE "${PROJECT_BINARY_DIR}/${PROJECT_NAME}Targets.cmake") -# Generate the config files -include(CMakePackageConfigHelpers) configure_package_config_file( - ${PROJECT_SOURCE_DIR}/cmake/Config.cmake.in + "${PROJECT_SOURCE_DIR}/cmake/${PROJECT_NAME}Config.cmake.in" "${PROJECT_BINARY_DIR}/${PROJECT_NAME}Config.cmake" - INSTALL_DESTINATION "lib/cmake/${PROJECT_NAME}" - NO_SET_AND_CHECK_MACRO - NO_CHECK_REQUIRED_COMPONENTS_MACRO -) + INSTALL_DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}") + write_basic_package_version_file( "${PROJECT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake" - COMPATIBILITY AnyNewerVersion -) -install(FILES - ${PROJECT_BINARY_DIR}/${PROJECT_NAME}Config.cmake - ${PROJECT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake - DESTINATION "lib/cmake/${PROJECT_NAME}" -) -################################################################################ + VERSION "${PROJECT_VERSION}" + COMPATIBILITY SameMajorVersion) + +install( + EXPORT ${PROJECT_NAME}Targets + FILE ${PROJECT_NAME}Targets.cmake + DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}") + +install(FILES "${PROJECT_BINARY_DIR}/${PROJECT_NAME}Config.cmake" + "${PROJECT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake" + DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}") +# ############################################################################## diff --git a/CMakePresets.json b/CMakePresets.json new file mode 100644 index 000000000..700138154 --- /dev/null +++ b/CMakePresets.json @@ -0,0 +1,27 @@ +{ + "version": 3, + "configurePresets": [ + { + "name": "gc", + "displayName": "Graph Compiler", + "generator": "Ninja", + "binaryDir": "${sourceDir}/build" + }, + { + "name": "gc-dbg", + "displayName": "Graph Compiler - Debug", + "inherits": "gc", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug" + } + }, + { + "name": "gc-ldbg", + "displayName": "Graph Compiler - Debug with Dylink", + "inherits": "gc-dbg", + "cacheVariables": { + "GC_DYLINK": "ON" + } + } + ] +} \ No newline at end of file diff --git a/README.md b/README.md index e7d4a3cdc..35129e58f 100644 --- a/README.md +++ b/README.md @@ -1,81 +1,35 @@ -[](https://scan.coverity.com/projects/intel-graph-compiler) +## Build and install Python bindings -# Graph Compiler -Graph Compiler is an end-to-end, MLIR-based compiler designed to enhance the performance of deep learning workloads. It accepts computation graphs from the frontend (primarily linalg on tensors), applies domain-specific optimizations and transformations, generates code, and manages runtime execution. - -## Build instructions - -### All-in-one compile script - -It is recommended for the users to use the all-in-one compile script at `scripts/compile.sh`. It downloads the LLVM dependency and builds the project. - -### Step-by-step build intructions - -To build this project step by step, first you need to find the LLVM commit-id we are using at `cmake/llvm-version.txt`. Then clone specific version of LLVM: +Create a virtual environment: ```bash -export LLVM_COMMIT_HASH=$(< cmake/llvm-version.txt) -git clone https://github.com/llvm/llvm-project -cd llvm-project -git checkout $LLVM_COMMIT_HASH +python -m venv .venv --prompt gc +. .venv/bin/activate +pip install nanobind pytest torch torch-mlir \ + --extra-index-url https://download.pytorch.org/whl/xpu \ + --find-links https://github.com/llvm/torch-mlir-release/releases/expanded_assets/dev-wheels ``` -Build LLVM with the command lines given in `.github/workflows/build-llvm.yml`: +Build the project and install the Python bindings in editable mode: ```bash -mkdir llvm-install -cmake -G Ninja llvm -B build -DCMAKE_INSTALL_PREFIX=llvm-install \ - -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=true -DLLVM_ENABLE_PROJECTS="mlir" -DLLVM_TARGETS_TO_BUILD="X86" -DLLVM_INSTALL_UTILS=true -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DLLVM_INSTALL_GTEST=ON -cmake --build build --target install +pip install -v -e python/ ``` -Notes - * It is recommended to add optional options `-DLLVM_BUILD_LLVM_DYLIB=ON -DLLVM_LINK_LLVM_DYLIB=ON` to the command `cmake -G Ninja llvm ...` above **if you are building for CPU only**. These will enable the build of LLVM/MLIR dynamic libraries and let MLIR/LLVM tools link to them, to reduce the installed binary size of LLVM/MLIR. These options also enable the `GC_DEV_LINK_LLVM_DYLIB` option of graph-compiler repo (see below). - * The option `-DLLVM_INSTALL_GTEST=ON` is optional, if the tests of graph-compiler are disabled (see `GC_ENABLE_TEST` below). - * If you would like to enable GPU components of Graph Compiler, please make sure to statically link Graph Compiler and LLVM(MLIR). It is a known issue that LLVM shared library cannot be linked together with IGC (Intel's low level GPU compiler). Make sure `LLVM_BUILD_LLVM_DYLIB` and `LLVM_LINK_LLVM_DYLIB` are `OFF` (they are off by default). Also make sure Graph Compiler's cmake option `GC_DEV_LINK_LLVM_DYLIB` is `OFF` when configuring Graph Compiler (see below). -* If you would like to enable [`BenchGC`](docs/benchgc_overview.md) or [`MLIR Binding`](python/README.md), please confirm to add options - ``-DPython3_EXECUTABLE=`which python3` -DMLIR_ENABLE_BINDINGS_PYTHON=ON``. - -We have now installed LLVM at `llvm-project/llvm-install`. - -Change working directory to graph-compiler repo and prepare the build directory: +## Run Python examples ```bash -cd /PATH/TO/graph-compiler -mkdir build && cd build +python python/example/matmul.py ``` -Build and run tests: +## Run Python unit tests ```bash -cmake .. -G Ninja \ - -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - -DMLIR_DIR=/PATH/TO/llvm-project/llvm-install/lib/cmake/mlir \ - -DLLVM_EXTERNAL_LIT=$(which lit) -cmake --build . --target gc-check +pytest python/test/test_torch.py -v -s ``` -Notes: - * `/PATH/TO/llvm-project/llvm-install` should be the install path of LLVM. If you installed LLVM elsewhere by `-DCMAKE_INSTALL_PREFIX` option when building LLVM, you need to change the path in `-DMLIR_DIR` accordingly. - * The cmake option `-DLLVM_EXTERNAL_LIT` is for the tests of this project. It requires the `lit` tool to be installed in the system. You can install it via `pip install lit`. If you don't need to run the tests of this repo, you can omit this option in the command line. +## Run gc-opt -More notes if Intel® Extension for MLIR is on (`-DGC_ENABLE_IMEX=ON`): - * make sure the OpenCL runtime is installed in your system. You can either - install using OS-provided package (Ubuntu 22.04) -```sh -sudo apt install -y intel-opencl-icd opencl-c-headers +```bash +./build/bin/gc-opt --gpu-dev-props='arch=pvc' --gc-gpu-pipeline='dump' linalg_on_tensors.mlir ``` - Or, download and install package from: https://github.com/intel/compute-runtime/releases - * the LLVM codebase needs to be patched to support XeGPU lowering (from IMEX). Please follow instructions of [IMEX](https://github.com/intel/mlir-extensions) on patching LLVM. - -Graph Compiler supports the following build-time options. - -| CMake Option | Supported values (defaults in bold) | Description | -|:--------------------------|:---------------------------------------|:----------------------------------------------------------------| -| GC_ENABLE_LEGACY | **ON**, OFF | Controls building the legacy graph-compiler component | -| GC_ENABLE_TEST | **ON**, OFF | Controls building the tests | -| GC_DEV_LINK_LLVM_DYLIB | ON, **OFF** | Controls dynamic link LLVM/MLIR libraries, mainly for developer | -| GC_ENABLE_BINDINGS_PYTHON | **ON**, OFF | Controls building the Python API | -| GC_ENABLE_IMEX | ON, **OFF** | Whether to enable the IMEX components | -| GC_ENABLE_GPU | ON, **OFF** | Whether to enable the GPU tools and components | - diff --git a/THIRD-PARTY-PROGRAMS b/THIRD-PARTY-PROGRAMS index e407a42b8..09c6e635f 100644 --- a/THIRD-PARTY-PROGRAMS +++ b/THIRD-PARTY-PROGRAMS @@ -50,147 +50,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ======================================================================================= -2. Googletest -Copyright 2005, Google Inc. -Copyright 2006, Google Inc. -Copyright 2007, Google Inc. -Copyright 2008, Google Inc. -Copyright 2015, Google Inc. -All rights reserved. - -3-Clause BSD License - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -The following individuals and institutions are among the Contributors: - -* Aaron C. Meadows -* Adriaan de Groot -* Aleksey Avdeev -* Alexander Neundorf -* Alexander Smorkalov -* Alexey Sokolov -* Alex Merry -* Alex Turbov -* Andreas Pakulat -* Andreas Schneider -* André Rigland Brodtkorb -* Axel Huebl, Helmholtz-Zentrum Dresden - Rossendorf -* Benjamin Eikel -* Bjoern Ricks -* Brad Hards -* Christopher Harvey -* Christoph Grüninger -* Clement Creusot -* Daniel Blezek -* Daniel Pfeifer -* Enrico Scholz -* Eran Ifrah -* Esben Mose Hansen, Ange Optimization ApS -* Geoffrey Viola -* Google Inc -* Gregor Jasny -* Helio Chissini de Castro -* Ilya Lavrenov -* Insight Software Consortium -* Jan Woetzel -* Julien Schueller -* Kelly Thompson -* Konstantin Podsvirov -* Laurent Montel -* Mario Bensi -* Martin Gräßlin -* Mathieu Malaterre -* Matthaeus G. Chajdas -* Matthias Kretz -* Matthias Maennich -* Michael Hirsch, Ph.D. -* Michael Stürmer -* Miguel A. Figueroa-Villanueva -* Mike Jackson -* Mike McQuaid -* Nicolas Bock -* Nicolas Despres -* Nikita Krupen'ko -* NVIDIA Corporation -* OpenGamma Ltd. -* Patrick Stotko -* Per Øyvind Karlsen -* Peter Collingbourne -* Petr Gotthard -* Philip Lowman -* Philippe Proulx -* Raffi Enficiaud, Max Planck Society -* Raumfeld -* Roger Leigh -* Rolf Eike Beer -* Roman Donchenko -* Roman Kharitonov -* Ruslan Baratov -* Sebastian Holtermann -* Stephen Kelly -* Sylvain Joubert -* The Qt Company Ltd. -* Thomas Sondergaard -* Tobias Hunger -* Todd Gamblin -* Tristan Carel -* University of Dundee -* Vadim Zhukov -* Will Dicharry - -See version control history for details of individual contributions. - -The above copyright and license notice applies to distributions of -CMake in source and binary form. Third-party software packages supplied -with CMake under compatible licenses provide their own copyright notices -documented in corresponding subdirectories or source files. - ------------------------------------------------------------------------------- - -CMake was initially developed by Kitware with the following sponsorship: - - * National Library of Medicine at the National Institutes of Health - as part of the Insight Segmentation and Registration Toolkit (ITK). - - * US National Labs (Los Alamos, Livermore, Sandia) ASC Parallel - Visualization Initiative. - - * National Alliance for Medical Image Computing (NAMIC) is funded by the - National Institutes of Health through the NIH Roadmap for Medical Research, - Grant U54 EB005149. - - * Kitware, Inc. - -======================================================================================= - -3. LLVM +2. LLVM ============================================================================== The LLVM Project is under the Apache License v2.0 with LLVM Exceptions: ============================================================================== @@ -413,256 +273,3 @@ conflicts with the conditions of the GPLv2, you may retroactively and prospectively choose to deem waived or otherwise exclude such Section(s) of the License, but only in their entirety and only with respect to the Combined Software. - -======================================================================================= -4. pybind11 - Copyright (c) 2016 Wenzel Jakob , All rights reserved. - - xbyak - Copyright (c) 2007 MITSUNARI Shigeo - All rights reserved. - - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Please also refer to the file .github/CONTRIBUTING.md, which clarifies licensing of -external contributions to this project including patches, pull requests, etc. -======================================================================================= -5. Intel® oneAPI Deep Neural Network Library (oneDNN) - Copyright Intel Corporation - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - ============================================================================ - - Copyright 2016-2023 Intel Corporation - Copyright 2018 YANDEX LLC - Copyright 2019-2023 FUJITSU LIMITED - Copyright 2020-2023 Arm Ltd. and affiliates - Copyright 2020-2022 Codeplay Software Limited - Copyright 2021 Alanna Tempest - Copyright 2022-2023 IBM Corporation - Copyright 2023 KNS Group LLC (YADRO) - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - This distribution includes third party software ("third party programs"). - This third party software, even if included with the distribution of - the Intel software, may be governed by separate license terms, including - without limitation, third party license terms, other Intel software license - terms, and open source software license terms. These separate license terms - govern your use of the third party programs as set forth in the - "THIRD-PARTY-PROGRAMS" file. - -======================================================================================= - -The following third party programs have their own third party program files. These additional third party program files are as follows: -1. OneAPI Deep Neural Networks Library - Corresponding file is found at the same level in the distribution (THIRD-PARTY-PROGRAMS-ONEDNN.txt) diff --git a/cmake/Config.cmake.in b/cmake/Config.cmake.in deleted file mode 100644 index 90c5b52a0..000000000 --- a/cmake/Config.cmake.in +++ /dev/null @@ -1,10 +0,0 @@ -@PACKAGE_INIT@ - -include ("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Targets.cmake") - -find_package(MLIR REQUIRED CONFIG) - -include_directories( - ${LLVM_INCLUDE_DIRS} - ${MLIR_INCLUDE_DIRS} -) diff --git a/cmake/GraphCompilerConfig.cmake.in b/cmake/GraphCompilerConfig.cmake.in new file mode 100644 index 000000000..54aaab381 --- /dev/null +++ b/cmake/GraphCompilerConfig.cmake.in @@ -0,0 +1,15 @@ +@PACKAGE_INIT@ + +include(CMakeFindDependencyMacro) + +find_dependency(MLIR REQUIRED CONFIG) +if(NOT TARGET LLVM) + find_dependency(LLVM REQUIRED CONFIG) +endif() + +include("${CMAKE_CURRENT_LIST_DIR}/GraphCompilerTargets.cmake") + +if(TARGET GcInterface) + set_property(TARGET GcInterface APPEND PROPERTY + INTERFACE_INCLUDE_DIRECTORIES "${LLVM_INCLUDE_DIRS};${MLIR_INCLUDE_DIRS}") +endif() diff --git a/cmake/functions.cmake b/cmake/functions.cmake index 6ee674400..65f0325a9 100644 --- a/cmake/functions.cmake +++ b/cmake/functions.cmake @@ -1,172 +1,85 @@ include_guard() -include(FetchContent) - -# A wrapper around FetchContent that could either fetch content from -# a git repository, use a local directory or find_package(). -function(gc_fetch_content - # The content name. Also used as the package name for find_package(). - name - # Git tag, branch name or hash. If the value has the format - # [v]major[.minor[.patch[.tweak]]] and the SKIP_FIND option is not - # specified, use find_package() first. - # The variable GC__VERSION could be used to override this argument. - git_tag_or_version - # Git repository URL. If the variable GC__SRC_DIR is defined, - # the local directory is used instead. - git_repository - # - # Optional arguments: - # SKIP_ADD: Populate but do not add the content to the project. - # SKIP_FIND: Do not use find_package(). - # SET: key=value variables to be set before the content population. -) - string(TOUPPER ${name} uname) - cmake_parse_arguments(GC_${uname} "SKIP_ADD;SKIP_FIND" "" "SET" ${ARGN}) - - if (DEFINED GC_${uname}_SET) - foreach (var ${GC_${uname}_SET}) - string(REGEX REPLACE "([^=]+)=(.*)" "\\1;\\2" var ${var}) - list(GET var 0 key) - list(GET var 1 value) - message(STATUS "Setting ${key}=${value}") - set(${key} ${value}) - endforeach () - endif () - - if (DEFINED GC_${uname}_SRC_DIR) - FetchContent_Declare( - ${name} - SOURCE_DIR ${GC_${uname}_SRC_DIR} - ) - else () - if (DEFINED GC_${uname}_VERSION) - set(git_tag_or_version ${GC_${uname}_VERSION}) - endif () - message(STATUS "${name}_VERSION: ${git_tag_or_version}") - - if (NOT ${uname}_SKIP_FIND AND git_tag_or_version - MATCHES "^v?([0-9]+(\.[0-9]+(\.[0-9]+(\.[0-9]+)?)?)?)$") - string(REPLACE "v" "" version ${git_tag_or_version}) - find_package(${name} ${version} EXACT) - endif () - - if (NOT DEFINED ${name}_VERSION_COUNT) - message(STATUS "Fetching ${name} from ${git_repository}") - set(FETCHCONTENT_QUIET FALSE) - FetchContent_Declare( - ${name} - GIT_REPOSITORY ${git_repository} - GIT_TAG ${git_tag_or_version} - GIT_PROGRESS TRUE - FIND_PACKAGE_ARGS ${FIND_PACKAGE_ARGS} - ) - endif () - endif () - - if (NOT DEFINED ${name}_VERSION_COUNT) - if (GC_${uname}_SKIP_ADD) - if (NOT DEFINED ${name}_POPULATED) - FetchContent_Populate(${name}) - FetchContent_GetProperties(${name}) - set(${name}_POPULATED TRUE PARENT_SCOPE) - endif () - else () - FetchContent_MakeAvailable(${name}) - endif () - - set(${name}_SOURCE_DIR ${${name}_SOURCE_DIR} PARENT_SCOPE) - set(${name}_BINARY_DIR ${${name}_BINARY_DIR} PARENT_SCOPE) - endif () -endfunction() macro(gc_set_mlir_link_components VAR) - if(GC_DEV_LINK_LLVM_DYLIB) - set(${VAR} - MLIR - ) - else() - set(${VAR} - ${ARGN} - ) - endif() + if(GC_DYLINK) + set(${VAR} MLIR) + else() + set(${VAR} ${ARGN}) + endif() endmacro() function(gc_add_mlir_library name) + if(GC_DYLINK AND NOT SHARED IN_LIST ARGN) + add_mlir_library(${name} DISABLE_INSTALL ${ARGN}) + else() add_mlir_library(${ARGV}) + endif() - if(name MATCHES ".+Passes") - set_property(GLOBAL APPEND PROPERTY GC_PASS_LIBS ${name}) - else() - set_property(GLOBAL APPEND PROPERTY GC_MLIR_LIBS ${name}) - endif() + if(name MATCHES ".+Passes") + set_property(GLOBAL APPEND PROPERTY GC_PASS_LIBS ${name}) + else() + set_property(GLOBAL APPEND PROPERTY GC_MLIR_LIBS ${name}) + endif() - if(GcInterface IN_LIST ARGN) - if(SHARED IN_LIST ARGN) - target_link_libraries(${name} PUBLIC GcInterface) - else() - target_link_libraries(obj.${name} PUBLIC GcInterface) - endif() + if(GcInterface IN_LIST ARGN) + if(SHARED IN_LIST ARGN) + target_link_libraries(${name} PUBLIC GcInterface) + else() + target_link_libraries(obj.${name} PUBLIC GcInterface) endif() + endif() endfunction() function(gc_add_mlir_dialect_library name) + if(GC_DYLINK) + add_mlir_dialect_library(${name} DISABLE_INSTALL ${ARGN}) + else() add_mlir_dialect_library(${ARGV}) - target_link_libraries(obj.${name} PUBLIC GcInterface) - set_property(GLOBAL APPEND PROPERTY GC_DIALECT_LIBS ${name}) + endif() + target_link_libraries(obj.${name} PUBLIC GcInterface) + set_property(GLOBAL APPEND PROPERTY GC_DIALECT_LIBS ${name}) - if(GcInterface IN_LIST ARGN) - target_link_libraries(obj.${name} PUBLIC GcInterface) - endif() + if(GcInterface IN_LIST ARGN) + target_link_libraries(obj.${name} PUBLIC GcInterface) + endif() endfunction() function(gc_add_mlir_conversion_library name) + if(GC_DYLINK) + add_mlir_conversion_library(${name} DISABLE_INSTALL ${ARGN}) + else() add_mlir_conversion_library(${ARGV}) - target_link_libraries(obj.${name} PUBLIC GcInterface) - set_property(GLOBAL APPEND PROPERTY GC_PASS_LIBS ${name}) + endif() + target_link_libraries(obj.${name} PUBLIC GcInterface) + set_property(GLOBAL APPEND PROPERTY GC_PASS_LIBS ${name}) - if(GcInterface IN_LIST ARGN) - target_link_libraries(obj.${name} PUBLIC GcInterface) - endif() + if(GcInterface IN_LIST ARGN) + target_link_libraries(obj.${name} PUBLIC GcInterface) + endif() endfunction() function(gc_add_mlir_translation_library name) + if(GC_DYLINK) + add_mlir_translation_library(${name} DISABLE_INSTALL ${ARGN}) + else() add_mlir_translation_library(${ARGV}) - target_link_libraries(obj.${name} PUBLIC GcInterface) - set_property(GLOBAL APPEND PROPERTY GC_MLIR_LIBS ${name}) + endif() + target_link_libraries(obj.${name} PUBLIC GcInterface) + set_property(GLOBAL APPEND PROPERTY GC_MLIR_LIBS ${name}) - if(GcInterface IN_LIST ARGN) - target_link_libraries(obj.${name} PUBLIC GcInterface) - endif() + if(GcInterface IN_LIST ARGN) + target_link_libraries(obj.${name} PUBLIC GcInterface) + endif() endfunction() macro(gc_add_mlir_tool name) - # the dependency list copied from mlir/tools/mlir-cpu-runner/CMakeLists.txt of upstream - if(NOT DEFINED LLVM_LINK_COMPONENTS) - set(LLVM_LINK_COMPONENTS - Core - Support - nativecodegen - native - ) - endif() - if(NOT DEFINED MLIR_LINK_COMPONENTS) - gc_set_mlir_link_components(MLIR_LINK_COMPONENTS - MLIRAnalysis - MLIRBuiltinToLLVMIRTranslation - MLIRExecutionEngine - MLIRIR - MLIRJitRunner - MLIRLLVMDialect - MLIRLLVMToLLVMIRTranslation - MLIRToLLVMIRTranslationRegistration - MLIRParser - MLIRTargetLLVMIRExport - MLIRSupport - ) - endif() - add_mlir_tool(${ARGV}) - #LLVM_LINK_COMPONENTS is processed by LLVM cmake in add_llvm_executable - target_link_libraries(${name} PRIVATE GcInterface ${MLIR_LINK_COMPONENTS}) - llvm_update_compile_flags(${name}) - set_property(GLOBAL APPEND PROPERTY GC_TOOLS ${name}) -endmacro() \ No newline at end of file + add_mlir_tool(${ARGV}) + # LLVM_LINK_COMPONENTS is processed by LLVM cmake in add_llvm_executable + target_link_libraries(${name} PRIVATE GcInterface ${MLIR_LINK_COMPONENTS}) + llvm_update_compile_flags(${name}) + set_property(GLOBAL APPEND PROPERTY GC_TOOLS ${name}) + set_target_properties(${name} PROPERTIES EXCLUDE_FROM_ALL OFF) + target_compile_options(${name} PRIVATE -ffunction-sections -fdata-sections) + target_link_options(${name} PRIVATE -Wl,--gc-sections -Wl,--as-needed) + mlir_check_all_link_libraries(${name}) +endmacro() diff --git a/cmake/gtest.cmake b/cmake/gtest.cmake deleted file mode 100644 index 54924ec99..000000000 --- a/cmake/gtest.cmake +++ /dev/null @@ -1,8 +0,0 @@ -include_guard() - -gc_fetch_content( - GTest - v1.14.0 - https://github.com/google/googletest.git - SET INSTALL_GTEST=OFF -) diff --git a/cmake/imex-version.txt b/cmake/imex-version.txt deleted file mode 100644 index 14006f880..000000000 --- a/cmake/imex-version.txt +++ /dev/null @@ -1 +0,0 @@ -08b8571a1b87329dcefc3e3c5d54f67d3835048e \ No newline at end of file diff --git a/cmake/imex.cmake b/cmake/imex.cmake deleted file mode 100644 index 41d1ded68..000000000 --- a/cmake/imex.cmake +++ /dev/null @@ -1,29 +0,0 @@ -include_guard() - -get_property(IMEX_INCLUDES GLOBAL PROPERTY IMEX_INCLUDES) -if (NOT DEFINED IMEX_INCLUDES) - if(GC_DEV_LINK_LLVM_DYLIB) - message(WARN "GPU backend may not be compatible with dynamic linking to LLVM") - endif() - - # Read the content of imex-version.txt - file(READ "${CMAKE_CURRENT_LIST_DIR}/imex-version.txt" IMEX_HASH) - - # Strip any extra whitespace or newlines - string(STRIP "${IMEX_HASH}" IMEX_HASH) - - # TODO: Change to main https://github.com/intel/mlir-extensions when all the - # required functionality is merged. - set(IMEX_URL https://github.com/intel/mlir-extensions) - gc_fetch_content(imex "${IMEX_HASH}" "${IMEX_URL}" - SET IMEX_CHECK_LLVM_VERSION=ON IMEX_ENABLE_L0_RUNTIME=0 - ) - - set(IMEX_INCLUDES - ${imex_BINARY_DIR}/include - ${imex_SOURCE_DIR}/include - ${imex_SOURCE_DIR}/src - ) - set_property(GLOBAL PROPERTY IMEX_INCLUDES ${IMEX_INCLUDES}) - target_compile_options(GcInterface INTERFACE -DGC_USE_IMEX) -endif () diff --git a/cmake/llvm-version-imex.txt b/cmake/llvm-version-imex.txt deleted file mode 100644 index 33000613b..000000000 --- a/cmake/llvm-version-imex.txt +++ /dev/null @@ -1 +0,0 @@ -add6b2f35f2bcf1f59a2ab2d5b3dab124fe0895a diff --git a/cmake/llvm-version.txt b/cmake/llvm-version.txt index 33000613b..9a7c4a739 100644 --- a/cmake/llvm-version.txt +++ b/cmake/llvm-version.txt @@ -1 +1 @@ -add6b2f35f2bcf1f59a2ab2d5b3dab124fe0895a +6c67a96c5a45da91f18589a0dcd4fc8a61880a52 \ No newline at end of file diff --git a/cmake/mlir.cmake b/cmake/mlir.cmake index 4e5c9b66d..ce77b6cb5 100644 --- a/cmake/mlir.cmake +++ b/cmake/mlir.cmake @@ -2,6 +2,11 @@ include_guard() find_package(MLIR REQUIRED CONFIG) +if(NOT GC_DYLINK) + set(LLVM_LINK_LLVM_DYLIB OFF) + set(MLIR_LINK_MLIR_DYLIB OFF) +endif() + message(STATUS "Using MLIRConfig.cmake in: ${MLIR_DIR}") message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}") @@ -15,17 +20,157 @@ list(APPEND CMAKE_MODULE_PATH "${LLVM_CMAKE_DIR}") include(TableGen) include(AddLLVM) include(AddMLIR) + +# GCC's uninitialized use analysis produces false positives on LLVM's ADT. +# Upstream disables it in llvm/cmake/config-ix.cmake, which is not exported for +# out-of-tree builds, so setting the variable for 'HandleLLVMOptions' manually +# (flang does the same for its standalone build). +if(CMAKE_COMPILER_IS_GNUCXX) + set(USE_NO_MAYBE_UNINITIALIZED 1) +endif() + include(HandleLLVMOptions) -include_directories( - ${LLVM_INCLUDE_DIRS} - ${MLIR_INCLUDE_DIRS} -) +include_directories(SYSTEM ${LLVM_INCLUDE_DIRS} ${MLIR_INCLUDE_DIRS}) +foreach(dir IN LISTS LLVM_INCLUDE_DIRS MLIR_INCLUDE_DIRS) + target_include_directories(GcInterface SYSTEM + INTERFACE $) +endforeach() -set(LLVM_TABLEGEN_FLAGS - -I${PROJECT_BINARY_DIR}/include - -I${PROJECT_SOURCE_DIR}/include -) +set(LLVM_TABLEGEN_FLAGS -I${PROJECT_BINARY_DIR}/include + -I${PROJECT_SOURCE_DIR}/include) string(REPLACE " " ";" GC_LLVM_DEFINITIONS ${LLVM_DEFINITIONS}) target_compile_options(GcInterface INTERFACE ${GC_LLVM_DEFINITIONS}) + +function(gc_remove_dylib_aggregate_deps) + set(worklist ${ARGN}) + set(seen) + while(worklist) + list(POP_FRONT worklist target) + if(NOT TARGET "${target}" OR target IN_LIST seen) + continue() + endif() + list(APPEND seen "${target}") + + get_target_property(libs "${target}" INTERFACE_LINK_LIBRARIES) + if(libs) + set(filtered_libs) + foreach(lib IN LISTS libs) + if(lib STREQUAL "LLVM" + OR lib STREQUAL "MLIR" + OR lib STREQUAL "$" + OR lib STREQUAL "$" + OR lib MATCHES "lib(LLVM|MLIR)\\.so") + continue() + endif() + list(APPEND filtered_libs "${lib}") + if(TARGET "${lib}") + list(APPEND worklist "${lib}") + endif() + endforeach() + set_target_properties("${target}" PROPERTIES INTERFACE_LINK_LIBRARIES + "${filtered_libs}") + endif() + + get_target_property(configs "${target}" IMPORTED_CONFIGURATIONS) + foreach(config IN LISTS configs) + get_target_property(deps "${target}" + IMPORTED_LINK_DEPENDENT_LIBRARIES_${config}) + if(deps) + list(REMOVE_ITEM deps LLVM MLIR) + list(FILTER deps EXCLUDE REGEX "lib(LLVM|MLIR)\\.so") + set_target_properties( + "${target}" PROPERTIES IMPORTED_LINK_DEPENDENT_LIBRARIES_${config} + "${deps}") + endif() + endforeach() + endwhile() +endfunction() + +if(GC_DYLINK) + set(GC_LLVM_LINK_COMPONENTS LLVM) + set(MLIR_LINK_COMPONENTS MLIR) + set(MLIR_EXECUTION_ENGINE MLIRExecutionEngineShared) +else() + set(GC_LLVM_LINK_COMPONENTS Core Support nativecodegen native) + set(MLIR_LINK_COMPONENTS MLIRAnalysis MLIRIR MLIRLLVMDialect MLIRParser + MLIRTargetLLVMIRExport MLIRSupport) + set(MLIR_EXECUTION_ENGINE MLIRExecutionEngine) + set(MLIR_DIALECT_LIBS + MLIRAffineDialect + MLIRAffineTransforms + MLIRArithDialect + MLIRArithTransforms + MLIRArithValueBoundsOpInterfaceImpl + MLIRBufferizationAllExtensions + MLIRBufferizationDialect + MLIRBufferizationTransforms + MLIRControlFlowDialect + MLIRControlFlowTransforms + MLIRDLTIDialect + MLIRFuncAllExtensions + MLIRFuncDialect + MLIRGPUDialect + MLIRGPUPipelines + MLIRGPUTransforms + MLIRIndexDialect + MLIRLinalgDialect + MLIRLinalgTransforms + MLIRMathDialect + MLIRMemRefDialect + MLIRMemRefTransforms + MLIRSCFDialect + MLIRSCFTransforms + MLIRShapeDialect + MLIRTensorAllExtensions + MLIRTensorDialect + MLIRTensorInferTypeOpInterfaceImpl + MLIRTensorTilingInterfaceImpl + MLIRUBDialect + MLIRVectorDialect + MLIRVectorTransforms + MLIRXeGPUDialect + MLIRXeGPUTransforms + MLIRXeGPUUtils + MLIRXeVMDialect + MLIRXeVMTarget) + set(MLIR_CONVERSION_LIBS + MLIRArithToLLVM + MLIRBuiltinToLLVMIRTranslation + MLIRComplexToLLVM + MLIRControlFlowToLLVM + MLIRFuncToLLVM + MLIRGPUToGPURuntimeTransforms + MLIRGPUToLLVMIRTranslation + MLIRIndexToLLVM + MLIRLLVMToLLVMIRTranslation + MLIRMathToLLVM + MLIRMemRefToLLVM + MLIRSCFToGPU + MLIRUBToLLVM + MLIRVectorToLLVM + MLIRVectorToLLVMPass + MLIRVectorToXeGPU + MLIRXeVMToLLVMIRTranslation) +endif() + +if(GC_DYLINK) + target_link_libraries(GcInterface INTERFACE LLVM MLIR + ${MLIR_EXECUTION_ENGINE}) +else() + gc_remove_dylib_aggregate_deps( + ${MLIR_LINK_COMPONENTS} ${MLIR_DIALECT_LIBS} ${MLIR_CONVERSION_LIBS} + ${MLIR_EXECUTION_ENGINE}) + llvm_map_components_to_libnames(GC_LLVM_LIBS ${GC_LLVM_LINK_COMPONENTS}) + target_link_libraries( + GcInterface + INTERFACE ${MLIR_LINK_COMPONENTS} + ${MLIR_DIALECT_LIBS} + ${MLIR_CONVERSION_LIBS} + ${MLIR_EXECUTION_ENGINE} + ${GC_LLVM_LIBS} + LLVMX86CodeGen + LLVMX86AsmParser + LLVMSPIRVCodeGen) +endif() diff --git a/cmake/onednn.cmake b/cmake/onednn.cmake deleted file mode 100644 index 2633f435e..000000000 --- a/cmake/onednn.cmake +++ /dev/null @@ -1,21 +0,0 @@ -include_guard() - -get_property(GC_DNNL_INCLUDES GLOBAL PROPERTY GC_DNNL_INCLUDES) -if (NOT DEFINED GC_DNNL_INCLUDES) - # TODO: Change to main https://github.com/oneapi-src/oneDNN.git when all the - # required functionality is merged. - gc_fetch_content(dnnl dev https://github.com/kurapov-peter/oneDNN.git - SKIP_ADD - ) - - set(GC_DNNL_INCLUDES - $ - $ - $ - ) - - set_property(GLOBAL PROPERTY GC_DNNL_INCLUDES ${GC_DNNL_INCLUDES}) - set_property(GLOBAL PROPERTY GC_DNNL_SOURCE_DIR ${dnnl_SOURCE_DIR}) - - include(onednn_lite_config) -endif () diff --git a/cmake/onednn_lite_config.cmake b/cmake/onednn_lite_config.cmake deleted file mode 100644 index c075614e4..000000000 --- a/cmake/onednn_lite_config.cmake +++ /dev/null @@ -1,348 +0,0 @@ -include_guard() - -get_property(DNNL_INCLUDES GLOBAL PROPERTY GC_DNNL_INCLUDES) -get_property(DNNL_PATH GLOBAL PROPERTY GC_DNNL_SOURCE_DIR) -if (NOT DEFINED DNNL_INCLUDES) - message(FATAL_ERROR "DNNL contents not fetched yet, CMake will exit." ) -endif () - -########## This cmake build lite version of onednn, containing only microkernel related codes - -set(APP_NAME "dnnl_brgemm") - -# Build onednn -set(DNNL_BUILD_TESTS OFF) -set(DNNL_BUILD_EXAMPLES OFF) -set(DNNL_ENABLE_JIT_PROFILING OFF) -set(DNNL_BLAS_VENDOR NONE) -set(DNNL_LIBRARY_TYPE STATIC) - -set(DNNL_GPU_RUNTIME "NONE") -if(NOT DEFINED DNNL_CPU_RUNTIME) - set(DNNL_CPU_RUNTIME "OMP") - set(DNNL_CPU_THREADING_RUNTIME "OMP") -endif() - -if(${DNNL_CPU_RUNTIME} STREQUAL "OMP") - find_package(OpenMP REQUIRED) -endif() - -if(${DNNL_CPU_RUNTIME} STREQUAL "TBB") - find_package(TBB REQUIRED) -endif() - -########## copied from main cmake file of DNNL -# Set the target architecture. -if(NOT DNNL_TARGET_ARCH) - if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64.*|AARCH64.*|arm64.*|ARM64.*)") - set(DNNL_TARGET_ARCH "AARCH64") - elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(ppc64.*|PPC64.*|powerpc64.*)") - set(DNNL_TARGET_ARCH "PPC64") - elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(s390x.*|S390X.*)") - set(DNNL_TARGET_ARCH "S390X") - else() - set(DNNL_TARGET_ARCH "X64") - endif() -endif() - -if(UNIX OR MINGW) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17") -endif() - -########## from cmake/options.cmake -option(DNNL_ENABLE_MAX_CPU_ISA - "enables control of CPU ISA detected by oneDNN via DNNL_MAX_CPU_ISA - environment variable and dnnl_set_max_cpu_isa() function" ON) - -include("${DNNL_PATH}/cmake/Threading.cmake") - -########### copied from cmake/SDL.cmake, for -fstack-protector-strong -if(UNIX) - set(CMAKE_CCXX_FLAGS "-fPIC -Wformat -Wformat-security -ffunction-sections -fdata-sections") - append(CMAKE_CXX_FLAGS_RELEASE "-D_FORTIFY_SOURCE=2") - append(CMAKE_C_FLAGS_RELEASE "-D_FORTIFY_SOURCE=2") - if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") - if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.9) - append(CMAKE_CCXX_FLAGS "-fstack-protector-all") - else() - append(CMAKE_CCXX_FLAGS "-fstack-protector-strong") - endif() - - # GCC might be very paranoid for partial structure initialization, e.g. - # struct { int a, b; } s = { 0, }; - # However the behavior is triggered by `Wmissing-field-initializers` - # only. To prevent warnings on users' side who use the library and turn - # this warning on, let's use it too. Applicable for the library sources - # and interfaces only (tests currently rely on that fact heavily) - append(CMAKE_SRC_CCXX_FLAGS "-Wmissing-field-initializers") - append(CMAKE_EXAMPLE_CCXX_FLAGS "-Wmissing-field-initializers") - elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") - append(CMAKE_CCXX_FLAGS "-fstack-protector-all") - elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel") - append(CMAKE_CXX_FLAGS "-fstack-protector") - endif() - append(CMAKE_C_FLAGS "${CMAKE_CCXX_FLAGS}") - append(CMAKE_CXX_FLAGS "${CMAKE_CCXX_FLAGS}") - if(APPLE) - append(CMAKE_SHARED_LINKER_FLAGS "-Wl,-bind_at_load") - append(CMAKE_EXE_LINKER_FLAGS "-Wl,-bind_at_load") - else() - append(CMAKE_EXE_LINKER_FLAGS "-pie") - append(CMAKE_SHARED_LINKER_FLAGS "-Wl,-z,noexecstack -Wl,-z,relro -Wl,-z,now") - append(CMAKE_EXE_LINKER_FLAGS "-Wl,-z,noexecstack -Wl,-z,relro -Wl,-z,now") - endif() -elseif(MSVC AND ${CMAKE_CXX_COMPILER_ID} STREQUAL MSVC) - set(CMAKE_CCXX_FLAGS "/guard:cf") -endif() -########### END of copy of cmake/SDL.cmake - -########### copied from cmake/platform.cmake, for STDC* and -msse4.1 -add_definitions(-D__STDC_LIMIT_MACROS -D__STDC_CONSTANT_MACROS) -if(MSVC) - set(USERCONFIG_PLATFORM "x64") - append_if(DNNL_WERROR CMAKE_CCXX_FLAGS "/WX") - if(${CMAKE_CXX_COMPILER_ID} STREQUAL MSVC) - append(CMAKE_CCXX_FLAGS "/MP") - # int -> bool - append(CMAKE_CCXX_NOWARN_FLAGS "/wd4800") - # unknown pragma - append(CMAKE_CCXX_NOWARN_FLAGS "/wd4068") - # double -> float - append(CMAKE_CCXX_NOWARN_FLAGS "/wd4305") - # UNUSED(func) - append(CMAKE_CCXX_NOWARN_FLAGS "/wd4551") - # int64_t -> int (tent) - append(CMAKE_CCXX_NOWARN_FLAGS "/wd4244") - endif() - if(CMAKE_CXX_COMPILER_ID STREQUAL "Intel") - append(CMAKE_CCXX_FLAGS "/MP") - set(DEF_ARCH_OPT_FLAGS "-QxSSE4.1") - # disable: loop was not vectorized with "simd" - append(CMAKE_CCXX_NOWARN_FLAGS "-Qdiag-disable:13379") - # disable: loop was not vectorized with "simd" - append(CMAKE_CCXX_NOWARN_FLAGS "-Qdiag-disable:15552") - # disable: unknown pragma - append(CMAKE_CCXX_NOWARN_FLAGS "-Qdiag-disable:3180") - # disable: foo has been targeted for automatic cpu dispatch - append(CMAKE_CCXX_NOWARN_FLAGS "-Qdiag-disable:15009") - # disable: disabling user-directed function packaging (COMDATs) - append(CMAKE_CCXX_NOWARN_FLAGS "-Qdiag-disable:11031") - # disable: decorated name length exceeded, name was truncated - append(CMAKE_CCXX_NOWARN_FLAGS "-Qdiag-disable:2586") - # disable: disabling optimization; runtime debug checks enabled - append(CMAKE_CXX_FLAGS_DEBUG "-Qdiag-disable:10182") - endif() - if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") - append(CMAKE_CCXX_NOEXCEPT_FLAGS "-fno-exceptions") - # Clang cannot vectorize some loops with #pragma omp simd and gets - # very upset. Tell it that it's okay and that we love it - # unconditionally. - append(CMAKE_CCXX_FLAGS "-Wno-pass-failed") - # Clang doesn't like the idea of overriding optimization flags. - # We don't want to optimize jit gemm kernels to reduce compile time - append(CMAKE_CCXX_FLAGS "-Wno-overriding-t-option") - endif() -elseif(UNIX OR MINGW) - append(CMAKE_CCXX_FLAGS "-Wall -Wno-unknown-pragmas") - append_if(DNNL_WERROR CMAKE_CCXX_FLAGS "-Werror") - append(CMAKE_CCXX_FLAGS "-fvisibility=internal") - append(CMAKE_CXX_FLAGS "-fvisibility-inlines-hidden") - append(CMAKE_CCXX_NOEXCEPT_FLAGS "-fno-exceptions") - # compiler specific settings - if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") - if(DNNL_TARGET_ARCH STREQUAL "AARCH64") - set(DEF_ARCH_OPT_FLAGS "-O3") - # For native compilation tune for the host processor - if (CMAKE_SYSTEM_PROCESSOR STREQUAL CMAKE_HOST_SYSTEM_PROCESSOR) - append(DEF_ARCH_OPT_FLAGS "-mcpu=native") - endif() - elseif(DNNL_TARGET_ARCH STREQUAL "PPC64") - set(DEF_ARCH_OPT_FLAGS "-O3") - # For native compilation tune for the host processor - if (CMAKE_SYSTEM_PROCESSOR STREQUAL CMAKE_HOST_SYSTEM_PROCESSOR) - append(DEF_ARCH_OPT_FLAGS "-mcpu=native") - endif() - elseif(DNNL_TARGET_ARCH STREQUAL "S390X") - set(DEF_ARCH_OPT_FLAGS "-O3") - # For native compilation tune for the host processor - if (CMAKE_SYSTEM_PROCESSOR STREQUAL CMAKE_HOST_SYSTEM_PROCESSOR) - append(DEF_ARCH_OPT_FLAGS "-march=native") - endif() - elseif(DNNL_TARGET_ARCH STREQUAL "X64") - set(DEF_ARCH_OPT_FLAGS "-msse4.1") - endif() - # Clang cannot vectorize some loops with #pragma omp simd and gets - # very upset. Tell it that it's okay and that we love it - # unconditionally. - append(CMAKE_CCXX_NOWARN_FLAGS "-Wno-pass-failed") - - if (DEFINED DNNL_USE_CLANG_SANITIZER) - if(DNNL_USE_CLANG_SANITIZER MATCHES "Memory(WithOrigin)?") - if(NOT DNNL_CPU_THREADING_RUNTIME STREQUAL "SEQ") - message(WARNING "Clang OpenMP is not compatible with MSan! " - "Expect a lot of false positives!") - endif() - append(CMAKE_CCXX_SANITIZER_FLAGS "-fsanitize=memory") - if(DNNL_USE_CLANG_SANITIZER STREQUAL "MemoryWithOrigin") - append(CMAKE_CCXX_SANITIZER_FLAGS - "-fsanitize-memory-track-origins=2") - append(CMAKE_CCXX_SANITIZER_FLAGS - "-fno-omit-frame-pointer") - endif() - set(DNNL_ENABLED_CLANG_SANITIZER "${DNNL_USE_CLANG_SANITIZER}") - elseif(DNNL_USE_CLANG_SANITIZER STREQUAL "Undefined") - append(CMAKE_CCXX_SANITIZER_FLAGS "-fsanitize=undefined") - append(CMAKE_CCXX_SANITIZER_FLAGS - "-fno-sanitize=function,vptr") # work around linking problems - append(CMAKE_CCXX_SANITIZER_FLAGS "-fno-omit-frame-pointer") - set(DNNL_ENABLED_CLANG_SANITIZER "${DNNL_USE_CLANG_SANITIZER}") - elseif(DNNL_USE_CLANG_SANITIZER STREQUAL "Address") - append(CMAKE_CCXX_SANITIZER_FLAGS "-fsanitize=address") - set(DNNL_ENABLED_CLANG_SANITIZER "${DNNL_USE_CLANG_SANITIZER}") - elseif(DNNL_USE_CLANG_SANITIZER STREQUAL "Thread") - append(CMAKE_CCXX_SANITIZER_FLAGS "-fsanitize=thread") - set(DNNL_ENABLED_CLANG_SANITIZER "${DNNL_USE_CLANG_SANITIZER}") - elseif(DNNL_USE_CLANG_SANITIZER STREQUAL "Leak") - append(CMAKE_CCXX_SANITIZER_FLAGS "-fsanitize=leak") - set(DNNL_ENABLED_CLANG_SANITIZER "${DNNL_USE_CLANG_SANITIZER}") - else() - message(FATAL_ERROR - "Unsupported Clang sanitizer '${DNNL_USE_CLANG_SANITIZER}'") - endif() - if(DNNL_ENABLED_CLANG_SANITIZER) - message(STATUS - "Using Clang ${DNNL_ENABLED_CLANG_SANITIZER} " - "sanitizer (experimental!)") - append(CMAKE_CCXX_SANITIZER_FLAGS "-g -fno-omit-frame-pointer") - endif() - endif() - - if (DNNL_USE_CLANG_TIDY MATCHES "(CHECK|FIX)" AND ${CMAKE_VERSION} VERSION_LESS "3.6.0") - message(FATAL_ERROR "Using clang-tidy requires CMake 3.6.0 or newer") - elseif(DNNL_USE_CLANG_TIDY MATCHES "(CHECK|FIX)") - find_program(CLANG_TIDY NAMES clang-tidy) - if(NOT CLANG_TIDY) - message(FATAL_ERROR "Clang-tidy not found") - else() - if(DNNL_USE_CLANG_TIDY STREQUAL "CHECK") - set(CMAKE_CXX_CLANG_TIDY ${CLANG_TIDY}) - message(STATUS "Using clang-tidy to run checks") - elseif(DNNL_USE_CLANG_TIDY STREQUAL "FIX") - set(CMAKE_CXX_CLANG_TIDY ${CLANG_TIDY} -fix) - message(STATUS "Using clang-tidy to run checks and fix found issues") - endif() - endif() - endif() - - elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") - if(DNNL_TARGET_ARCH STREQUAL "AARCH64") - set(DEF_ARCH_OPT_FLAGS "-O3") - # For native compilation tune for the host processor - if (CMAKE_SYSTEM_PROCESSOR STREQUAL CMAKE_HOST_SYSTEM_PROCESSOR) - append(DEF_ARCH_OPT_FLAGS "-mcpu=native") - endif() - elseif(DNNL_TARGET_ARCH STREQUAL "PPC64") - set(DEF_ARCH_OPT_FLAGS "-O3") - # In GCC, -ftree-vectorize is turned on under -O3 since 2007. - # For native compilation tune for the host processor - if (CMAKE_SYSTEM_PROCESSOR STREQUAL CMAKE_HOST_SYSTEM_PROCESSOR) - append(DEF_ARCH_OPT_FLAGS "-mcpu=native") - endif() - elseif(DNNL_TARGET_ARCH STREQUAL "S390X") - set(DEF_ARCH_OPT_FLAGS "-O3") - # In GCC, -ftree-vectorize is turned on under -O3 since 2007. - # For native compilation tune for the host processor - if (CMAKE_SYSTEM_PROCESSOR STREQUAL CMAKE_HOST_SYSTEM_PROCESSOR) - append(DEF_ARCH_OPT_FLAGS "-march=native") - endif() - elseif(DNNL_TARGET_ARCH STREQUAL "X64") - set(DEF_ARCH_OPT_FLAGS "-msse4.1") - endif() - # suppress warning on assumptions made regarding overflow (#146) - append(CMAKE_CCXX_NOWARN_FLAGS "-Wno-strict-overflow") - elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Intel") - set(DEF_ARCH_OPT_FLAGS "-xSSE4.1") - # workaround for Intel Compiler that produces error caused - # by pragma omp simd collapse(..) - append(CMAKE_CCXX_NOWARN_FLAGS "-diag-disable:13379") - append(CMAKE_CCXX_NOWARN_FLAGS "-diag-disable:15552") - # disable `was not vectorized: vectorization seems inefficient` remark - append(CMAKE_CCXX_NOWARN_FLAGS "-diag-disable:15335") - # disable: foo has been targeted for automatic cpu dispatch - append(CMAKE_CCXX_NOWARN_FLAGS "-diag-disable:15009") - endif() -endif() - -append(CMAKE_C_FLAGS "${CMAKE_CCXX_FLAGS} ${DEF_ARCH_OPT_FLAGS}") -append(CMAKE_CXX_FLAGS "${CMAKE_CCXX_FLAGS} ${DEF_ARCH_OPT_FLAGS}") - -########### END of copy of cmake/platform.cmake - -########### setting dummy version info -set(DNNL_VERSION_MAJOR 0) -set(DNNL_VERSION_MINOR 0) -set(DNNL_VERSION_PATCH 0) -set(DNNL_VERSION_HASH "N/A") -########### END of setting dummy version info - -add_definitions(-DDNNL_ENABLE_JIT_PROFILING=0) -configure_file( - "${DNNL_PATH}/include/oneapi/dnnl/dnnl_config.h.in" - "${PROJECT_BINARY_DIR}/include/oneapi/dnnl/dnnl_config.h" -) - -configure_file( - "${DNNL_PATH}/include/oneapi/dnnl/dnnl_version.h.in" - "${PROJECT_BINARY_DIR}/include/oneapi/dnnl/dnnl_version.h" -) - -include_directories( - ${PROJECT_BINARY_DIR}/include - ${DNNL_PATH}/src - ${DNNL_PATH}/include - ) - -if(DNNL_ENABLE_MAX_CPU_ISA) - add_definitions(-DDNNL_ENABLE_MAX_CPU_ISA) -endif() - -add_definitions(-DDISABLE_VERBOSE=1) -file(GLOB_RECURSE DNNL_SOURCES - ${DNNL_PATH}/src/cpu/x64/brgemm/*.cpp - ${DNNL_PATH}/src/cpu/x64/injectors/*.cpp - ${DNNL_PATH}/src/cpu/x64/cpu_isa_traits.cpp - ${DNNL_PATH}/src/cpu/x64/jit_avx512_core_bf16cvt.cpp - ${DNNL_PATH}/src/cpu/x64/amx_tile_configure.[ch]pp - ${DNNL_PATH}/src/cpu/x64/jit_uni_convert_xf16.[ch]pp - ${DNNL_PATH}/src/cpu/jit_utils/jit_utils.cpp - ${DNNL_PATH}/src/cpu/platform.[ch]pp - ${DNNL_PATH}/src/cpu/bfloat16.cpp - ${DNNL_PATH}/src/cpu/binary_injector_utils.cpp - ${DNNL_PATH}/src/common/fpmath_mode.cpp - ${DNNL_PATH}/src/common/utils.cpp - ${DNNL_PATH}/src/common/bfloat16.[ch]pp - ${DNNL_PATH}/src/common/float8.[ch]pp - ${DNNL_PATH}/src/common/memory_debug.cpp - ${DNNL_PATH}/src/common/primitive_attr.cpp - ${DNNL_PATH}/src/common/broadcast_strategy.cpp - ${DNNL_PATH}/src/common/primitive_exec_types.cpp - ${DNNL_PATH}/src/common/memory.cpp - ${DNNL_PATH}/src/common/memory_zero_pad.cpp - ${DNNL_PATH}/src/common/memory_desc_wrapper.cpp - ${DNNL_PATH}/src/common/memory_desc.cpp - ${DNNL_PATH}/src/common/dnnl_thread.cpp - ${DNNL_PATH}/src/common/verbose.cpp - ${DNNL_PATH}/src/common/dnnl_debug.cpp - ${DNNL_PATH}/src/common/dnnl_debug_autogenerated.cpp - ${DNNL_PATH}/src/cpu/x64/jit_avx512_core_fp8cvt.cpp - ) - -add_library(dnnl_brgemm OBJECT ${DNNL_SOURCES}) -set_property(TARGET dnnl_brgemm PROPERTY POSITION_INDEPENDENT_CODE ON - CXX_VISIBILITY_PRESET "hidden" - VISIBILITY_INLINES_HIDDEN 1) - -# Currently build objs only -set_property(GLOBAL APPEND PROPERTY GC_DNNL_LIB_DEPS - $) diff --git a/cmake/version.cmake b/cmake/version.cmake index a6b1cb9aa..aca81d9e3 100644 --- a/cmake/version.cmake +++ b/cmake/version.cmake @@ -1,25 +1,86 @@ -include_guard() - -string(REPLACE "." ";" VERSION_LIST ${PROJECT_VERSION}) -list(GET VERSION_LIST 0 GC_VERSION_MAJOR) -list(GET VERSION_LIST 1 GC_VERSION_MINOR) -list(GET VERSION_LIST 2 GC_VERSION_PATCH) - -find_package(Git) -if(GIT_FOUND) - execute_process(COMMAND ${GIT_EXECUTABLE} -c log.showSignature=false log --no-abbrev-commit --oneline -1 --format=%H - WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} - RESULT_VARIABLE RESULT - OUTPUT_VARIABLE GC_VERSION_HASH - OUTPUT_STRIP_TRAILING_WHITESPACE) +# Computes GC_VERSION_STRING and writes include/gc/Version.h into BUILD_DIR. +# Called at build time via add_custom_target. +# +# Required inputs: PROJECT_SOURCE_DIR, BUILD_DIR, GIT_EXECUTABLE, VERSION_CODE, +# LLVM_VERSION_MAJOR, LLVM_VERSION_SUFFIX + +set(sfx "") + +if(GIT_EXECUTABLE AND EXISTS "${PROJECT_SOURCE_DIR}/.git") + execute_process( + COMMAND ${GIT_EXECUTABLE} rev-parse --short HEAD + WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}" + OUTPUT_VARIABLE hash + OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET) + + execute_process( + COMMAND ${GIT_EXECUTABLE} ls-files --modified --others --exclude-standard + WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}" + OUTPUT_VARIABLE dirty1 + OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET) + + execute_process( + COMMAND ${GIT_EXECUTABLE} diff --name-only --cached + WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}" + OUTPUT_VARIABLE dirty2 + OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET) + + set(dirty_content "") + if(dirty1 OR dirty2) + set(dirty_list) + if(dirty1) + string(REPLACE "\n" ";" dirty1_list "${dirty1}") + list(APPEND dirty_list ${dirty1_list}) + endif() + if(dirty2) + string(REPLACE "\n" ";" dirty2_list "${dirty2}") + list(APPEND dirty_list ${dirty2_list}) + endif() + list(REMOVE_DUPLICATES dirty_list) + list(SORT dirty_list) + foreach(f ${dirty_list}) + if(EXISTS "${PROJECT_SOURCE_DIR}/${f}") + file(READ "${PROJECT_SOURCE_DIR}/${f}" c) + string(APPEND dirty_content "${c}") + else() + string(APPEND dirty_content "-${f}") + endif() + endforeach() + endif() + + string(SHA1 dirty_hash "${dirty_content}") + if(dirty_content) + string(SHA1 dirty_hash "${dirty_content}") + set(sfx "${hash}-${dirty_hash}") + elseif(hash) + set(sfx "${hash}") + endif() endif() -if(NOT GIT_FOUND OR RESULT) - set(GC_VERSION_HASH "N/A") +if(sfx) + set(version_string "${VERSION_CODE}-${sfx}") +else() + set(version_string "${VERSION_CODE}") +endif() +if(LLVM_VERSION_SUFFIX) + set(llvm_version_string "${LLVM_VERSION_MAJOR}-${LLVM_VERSION_SUFFIX}") +else() + set(llvm_version_string "${LLVM_VERSION_MAJOR}") endif() -target_compile_options(GcInterface INTERFACE - -DGC_VERSION_MAJOR=${GC_VERSION_MAJOR} - -DGC_VERSION_MINOR=${GC_VERSION_MINOR} - -DGC_VERSION_PATCH=${GC_VERSION_PATCH} - -DGC_VERSION_HASH="${GC_VERSION_HASH}") +set(content + "// Auto-generated by cmake/version.cmake - do not edit. +#pragma once + +#define GC_VERSION_CODE ${VERSION_CODE} +#define GC_VERSION_STRING \"${version_string}-llvm-${llvm_version_string}\" +") +set(out "${BUILD_DIR}/include/gc/Version.h") +if(EXISTS "${out}") + file(READ "${out}" existing) + if(existing STREQUAL content) + return() + endif() +endif() +file(MAKE_DIRECTORY "${BUILD_DIR}/include/gc") +file(WRITE "${out}" "${content}") diff --git a/docs/CPU_pipeline_overview.md b/docs/CPU_pipeline_overview.md deleted file mode 100644 index 49ac79805..000000000 --- a/docs/CPU_pipeline_overview.md +++ /dev/null @@ -1,113 +0,0 @@ -# Graph Compiler CPU Compilation Flow Overview - -Graph Compiler is an MLIR based end-to-end DL compiler. The entire compilation process is divided into front-end, middle-end and back-end. Different compilation stages will use different combinations of dialects, and together with various transformation passes to perform various optimizations and graph lowering transformations. The entire process will transform IR from hardware-independent abstract expression to hardware-related concrete expression, and finally generate an executable kernel. - -Meanwhile, as an MLIR down-stream project, Graph Compiler's implementation not only uses the existing dialects and passes from MLIR up-stream, but also defines new dialects and passes. Most of the new implementations are upstream-able, and we will do so in the future. - -The content introduced in this document does not represent the current implemented status, but the target status after the implementation is completed. - -### Front-End - -The Graph Compiler front-end takes OneDNN Graph dialect as input. oneDNN Graph dialect is a newly defined dialect, which aims to describe the computation graph defined by oneDNN Graph. The ops in Dialect follow the [oneDNN Graph specification](https://oneapi-src.github.io/oneDNN/graph_supported_operations.html). - -oneDNN graph dialect example: - -```mlir -func.func @mlp(%in: tensor<128x512xbf16>, - %weight0: tensor<512x256xbf16>, %bias0: tensor<256xbf16>) -> tensor<128x256xbf16> { - // layer 0 - %0 = onednn_graph.matmul %in, %weight0, %bias0 : (tensor<128x512xbf16>, tensor<512x256xbf16>, tensor<256xbf16>) -> tensor<128x256xbf16> - %1 = onednn_graph.relu %0 : (tensor<128x256xbf16>) -> tensor<128x256xbf16> - return %1 : tensor<128x256xbf16> -} -``` - -There's no planned optimization passe in front-end. The only transformation pass is to lowering OneDNN Graph dialect into Linalg dialect. - -### Middle-End - -Middle-end is mainly responsible for general optimizations that are independent of the target hardware, and most of the transformations apply to both CPU and GPU. Some of the transformations need to query the target hardware information, such as cache level and capacity. The Hardware abstract layer(HAL) is the interface for abstracting and describing the target hardware information. Therefore, the same pass can generate different optimization results for different hardware under the guidance of HAL. - -According to the different dialect combinations used, middle-end is divided into the following stages: - -#### Linalg on Tensor - -This is the intermediate representation closest to the framework calculation graph. The example IR looks like: - -```mlir -func.func @mlp(%in: tensor<128x512xbf16>, - %weight0: tensor<512x256xbf16>, %bias0: tensor<256xbf16>) -> tensor<128x256xbf16> { - %0 = tensor.empty() : tensor<128x256xbf16> - %cst = arith.constant 0.000000e+00 : bf16 - %1 = linalg.fill ins(%cst : bf16) outs(%0 : tensor<128x256xbf16>) -> tensor<128x256xbf16> - %2 = linalg.matmul ins(%in, %weight0 : tensor<128x512xbf16>, tensor<512x256xbf16>) outs(%1 : tensor<128x256xbf16>) -> tensor<128x256xbf16> - %3 = tensor.empty() : tensor<128x256xbf16> - %broadcasted = linalg.broadcast ins(%bias0 : tensor<256xbf16>) outs(%3 : tensor<128x256xbf16>) dimensions = [0] - %4 = tensor.empty() : tensor<128x256xbf16> - %5 = linalg.add ins(%2, %broadcasted : tensor<128x256xbf16>, tensor<128x256xbf16>) outs(%4: tensor<128x256xbf16>) -> tensor<128x256xbf16> - %6 = tensor.empty() : tensor<128x256xbf16> - %7 = linalgx.relu ins(%5 : tensor<128x256xbf16>) outs(%6 : tensor<128x256xbf16>) -> tensor<128x256xbf16> - return %7 : tensor<128x256xbf16> -} -``` - -In this stage, GC will perform some analysis and transformation related to the whole graph. The main transformations include: - -* Padding propagation : insert tensor.pad op to adjust tensor shape if the shape is not divisible for target tiling size. -* Layout propagation : insert tensor.pack and tensor.unpack to adjust tensor layout if blocking layout is preferred. -* Tensor constant propagation : identify folding with constant tensor and build folding block. -* Matmul lowering : lower Linalg.matmul into scf.forall with linalg.batch_reduce_matmul. -* Fine-grain fusion: fuse element-wise/broadcast/reduce/movement ops into base op(e.g. matmul). -* Lower linalg to arith/math on virtual vector : lower Linalg to Arith/Math and tiling tensor to virtual vector. - -### Tensor and scf loop with arith/math on virtual vector - -In this stage, most of the Linalg ops are lowered to Scf loops with Arith and Math ops. Both Arith and Math ops use tile tensor as input and output. The tile tensor here can be multi-dimensional tensor in any shape, regardless of the hardware register width. The tile size is chosen based on L1 cache capacity, that is, it is a good abstraction to partition the problem size to this granularity, since the microkernel, pre-op, and post-op, works at the tensor size fitting within l1 cache size. Meanwhile, converting Linalg into Arith and Math can further expose the implementation details of Linalg ops, which allow us to further simplify the computation after fusion. - -IR example: - -```mlir -func.func @add_tensor(%arg0: tensor<4x8x31xf32>, %arg1: tensor<4x8x31xf32>) -> tensor<4x8x31xf32> { - %0 = tensor.empty() : tensor<4x8x31xf32> - %init = arith.constant 0: index - %c1 = arith.constant 1: index - %first_dim = arith.constant 4: index - %second_dim = arith.constant 8: index - // assume our tile shape is [31] - %third_dim = arith.constant 31: index - scf.for %c5 = %init to %first_dim step %c1 { - scf.for %c6 = %init to %second_dim step %c1 { - scf.for %c7 = %init to %third_dim step %c1 { - %1 = vector.transfer_read %args0[%c5,%c6,%c7] {permutation_map = affine_map<() -> ()>} : tensor<31xf32>, vector<31xf32> - %2 = vector.transfer_read %args0[%c5,%c6,%c7] {permutation_map = affine_map<() -> ()>} : tensor<31xf32>, vector<31xf32> - %3 = arith.add %1, %2 : vector<31xf32> - vector.transfer_write %3, %0[%c5, %c6, %c7] : vector<31xf32>, tensor<31xf32> - } - } - } - return %0: tensor<4x8x31xf32> -} -``` - -The main transformations in this stage include: -* Bfloat16 promotion and cast eliminatation : legalize the Arith and Math ops by inserting `arith.extf` and `arith.truncf` pairs if target device doesn't support, remove pair of redundant `arith.extf` and `arith.truncf` pairs to improve performance and accuracy. -* Lower to physical vector : Lower virtual vector to physical vector based on physical register width of target device. - -### Backend-End - -Back-end is responsible for device dependent optimization. The use of dialect will vary with the target device. This document will focus on the backend implementation for CPU. - -The implementation of BRGEMM is the key to CPU performance.In GC we plan to introduce two different implementations: - -* The BRGEMM provided by the library, such as onednn. In order to better abstract and describe the kernel provided by the library, we introduced the microkernel dialect. - -* The BRGEMM generated by MLIR. In this approach, The AMX dialect will be used to simplify tile config processing and optimization. - -By default GC will use openmp dialect to handle task parallelism. But for better performance and support for non-openmp threadpools, we also introduced the CPURuntime dialect. This dialect also introduces some runtime function calls specifically designed for the CPU, such as thread-local memory allocator, which can improve performance on the CPU. - -The main transformations are: -* Memref lowering and scheduling : lower tensor dialect to memref dialect and perform memory related optimization including memory hoist and rescheduling. -* Microkernel dialect and lowering : lower linalg.batch_reduce_matmul to microkernel dialect and further lower to a function call to dnnl brgemm, or an MLIR-based brgemm implementation. -* Parallelcpu dialect and lowering : lower to parallelcpu dialect for Nested parallel loop support and other CPU runtime calls. - -In the last step, everything will lower to LLVM dialect. We don't plan to introduce any transformation on LLVM dialect, just leverage the upstream implementation for this. diff --git a/docs/GPU_pipeline_overview.md b/docs/GPU_pipeline_overview.md deleted file mode 100644 index b1d760fbe..000000000 --- a/docs/GPU_pipeline_overview.md +++ /dev/null @@ -1,72 +0,0 @@ -# GPU Pipeline overview - -This is a living document for GPU pipeline design. It's purpose is to keep the decision history and provide a guiding overview for development. We expect swift changes in the design as we go, so this mostly highlights guiding principles. - -## Initial state description - -The primary goal of the design is to adhere to certain qualities of the final solution. -The spirit of the design is to reuse the existing parts, prefer upstream, and target long-term support in conjunction with other devices. - -At the highest level, the pipeline can be split into three main stages: -1. High-level platform-independent* transformations. These are to be shared with other flows (e.g., fusion). -2. GPU-specific transformations. These are responsible for HW mapping and include everything until a SPIR-V is emitted. -3. Code generation. This is tailored to a particular platform and is performed by a backend. - -There are existing paths for each stage (sometimes multiple, the choice affects other parts). A short landscape description follows. - -### Landscape -There are two primary ways of generating GPU target binary code, both going through IGC: scalar and vector paths. - -The scalar (aka SIMT) path relies on IGC's vectorization capabilities to map logical threads to SIMD lanes. Handling synchronization (e.g., cross-lane communication) is the main burden for otherwise transformation-amenable representation. - -The vector (aka SIMD) path in IGC expects the IR to have a certain explicitly-vectorized form, primarily built via a set of intrinsics (VC-intinsics). The main complexity of the approach for the pipeline is handling data/compute distribution between those vectors and handling such a deviation from other GPU types lowering paths. - -Today, there are two main options to reach the low-level compiler: -1. Lower to SPIR-V dialect and serialize it (IMEX). -2. Lower to LLVM IR and use the SPIR-V Translator (Triton). - -Both produce a SPIR-V that can be consumed by IGC. - -Going up the pipeline, the abstractions needed to express specific ISA semantics (e.g., DPAS and nd-load required for efficient contraction implementation) are covered by XeGPU dialect. The dialect allows for both SIMT and SIMD -style lowering. - -TODO: gpu(x), linalg-to-scf, gpu-map-parallel-loops. - -### Integration -There are three major point of integration that affect the way the pipeline is built: -1. Input representation. -2. Memory management. -3. Runtime interfaces. - -The primary input for our pipelines is linalg on tesnors with named ops. These are pretty flexible (adding more to the upstream is more-or-less straightforward) and cover a lot of ground. - -Memory management has to deal with weight caching, dynamic shapes, input/output handling, etc. Certain decisions on the compiler user side lead to additional complications in the pipeline. -For example, having to deal with 'logical' tenors for OneDNN imposes constraints on constant folding. - -The choice of runtime interface defines how much additional logic should reside in the pipeline. For managed devices (such as a GPU) there are two distinct options: -1. The compiler only emits a binary for the target device. -2. The compiler emits a binary and a launch stub that interacts with an appropriate runtime. -The latter provides more context, and thus, potentially more opportunities for optimization. The former gives more control to the user and simplifies the pipeline. - -### The path of least resistance -First milestone for the pipeline creation aims at taking what's working now and putting it together. - -This includes: -- Going through XeGPU dialect -- Using IMEX's XeGPU lowering -- Adapting TPP's linalg-to-xegpu - -## Decisions - -### Compilation -* Generate the code with kernel outlining. The motivation is that the compiler can take over some of the scheduling-related tasks. The implies the interface with a framework needs to expose synchronization mechanism (e.g., pass a GPU queue). This also affects kernel caching. JITed or non-JITed execution (GPU module converted to serialized SPIR-V or to an actual target-specific binary) are similar cases from that point of view. Both will need to retrieve the artifact and pass it to the lowered from `gpu.launch` runtime call. -* To align with the future pipelines, the target representation for the gpu module is LLVM. The actual path to the binary will be hidden inside `gpu-module-to-binary` implementation. From the kernel lowering perspective, the outlook of the target pipeline looks like: - -``` -builtin.module( - gpu-kernel-outlining, - xe-attach-target{chip=xe_3 O=3}, - gpu.module(convert-gpu-to-llvm-spv), - gpu-to-llvm, - gpu-module-to-binary -) -``` diff --git a/docs/benchgc_overview.md b/docs/benchgc_overview.md deleted file mode 100644 index cbc0aa921..000000000 --- a/docs/benchgc_overview.md +++ /dev/null @@ -1,39 +0,0 @@ -# BenchGC Overview - -BenchGC is a testing tool for the Graph Compiler project, focusing on tuning performance and verifying the correctness of the Graph Compiler. The benchmark and validation work will be supported on both CPU and GPU. - -BenchGC will also be part of the CI test and checking the regular correctness check for each PR. - -### What can BenchGC provide? -* Working modes - - Correctness - - Performance - - Tuning -* Validation scope - - Single MLIR linalg op - - Predefined pattern (such as MLP) - - MLIR Module provided by user -* Data filling strategy - - Benchdnn filling strategy (mainly for single op validation) - - Common distribution filling - - Load from pytorch dump -* Result comparison strategy - - Benchdnn compare strategy (mainly for single op validation) - - User defined threshold - -### BenchGC workflow -```mermaid -flowchart TB - SINGLE_OP_TEST["Single Op Test"] -- dialect, op, shape, dtype, attr --> MLIR_MODULE["MLIR Module"] - USER_MLIR_TEST["User MLIR Test"] -- load a MLIR file --> MLIR_MODULE["MLIR Module"] - PATTERN_TEST["Pattern Test"] -- pattern specified attr, shape, dtype --> MLIR_MODULE["MLIR Module"] - MLIR_MODULE["MLIR Module"] -- correctness --> REF_MLIR_RUNNER["Reference MLIR Runner"] - MLIR_MODULE["MLIR Module"] --> GRAPH_COMPILER_LIBRARY["Graph Compiler Library"] - REF_MLIR_RUNNER["Reference MLIR Runner"] -- pytorch --> COMPARISON{"Compare"} - GRAPH_COMPILER_LIBRARY["Graph Compiler Library"] -- correctness --> COMPARISON{"Compare"} - GRAPH_COMPILER_LIBRARY["Graph Compiler Library"] -- performance --> PERF_RESULT["Performance Result"] - COMPARISON{"Compare"} --> PASS["Pass"] - COMPARISON{"Compare"} --> FAIL["Fail"] -``` -### Acknowledgements -This work is inspired by the [benchdnn](https://github.com/oneapi-src/oneDNN/tree/main/tests/benchdnn) component from [oneDNN](https://github.com/oneapi-src/oneDNN/) project developed by Intel. diff --git a/docs/deep_tile_matmul_design.md b/docs/deep_tile_matmul_design.md deleted file mode 100644 index 4e48d82de..000000000 --- a/docs/deep_tile_matmul_design.md +++ /dev/null @@ -1,594 +0,0 @@ -# DOC: deep-Tiled Matmul - -## Introduction - -Tiling and parallelization are important for the performance of a computation intensitive workload (matmul, convolution, and e.t.c). Modern hardware is often equipped with multiple cores and multiple levels of cache, each with different characteristics in terms of size, latency, and bandwidth. To achieve good performance, it is important to utilize the parallelism of the underlying hardware and minimize the number of cache misses to improve the performance of the generated code. The goal of this document is to provide a design overview of the deep-tiled matmul in the graph compiler and its current situation in the community. - -## Current Situation in the MLIR Community - -According to the last section, tiling and parallelization are two important optimization techniques used in compilers to improve the performance of the generated code(matmul, convolution, and e.t.c). The code template could allow some complex optimization(some nontrivial memory copy/reuse to maximize the hardware efficiency), which is hard to write a unified pass in the compiler. - -In the upstream MLIR, there is already some support for tiling and parallelization optimization. The `Linalg` dialect provides a tiling interface to support tiling optimization. Besides, for better representing the concept of schedule, it also introduces the `Transform` dialect to declare the `schedule` in an IR form(vertical to the `payload`). - -This section will introduce the current situation in the MLIR community about the tiling interface, `Transform` dialect, hardware abstration layer and what is missing in the current upstream MLIR. - -### Tiling Interface And the Related Pass - -The MLIR provides the tiling interface as follows to support some simple tiling optimization. - -The tiling interface is a set of methods that an operation can implement to provide information about its iteration space and how it can be tiled. The tiling interface is used by the tiling pass to generate a tiled implementation of the operation. It could easily transform the operation like: - -```MLIR -%0 = linalg.generic ins(%in) outs(%out) {indexing_maps = [affine_map<(d0) -> (d0)>], - iterator_types = ["parallel"]} - : tensor -> tensor -``` - -into: - -```MLIR -%1 = scf.for %iv = %c0 to %dim_0 step %c4 iter_args(%arg3 = %out) -> (tensor) { - %2 = tensor.extract_slice %in[%iv] [%c4] [1] : tensor to tensor - %3 = tensor.extract_slice %out[%iv] [%c4] [1] : tensor to tensor - %4 = linalg.generic ins(%2) outs(%3) ["parallel"] : tensor -> tensor - %5 = tensor.insert_slice %4, %arg3 : tensor - scf.yield %5 -} -``` - -The tiling interface further provides several functions like `tileUsingSCF(RewriterBase &rewriter, TilingInterface op, const SCFTilingOptions &options)` to support tile an op inherited the tiling interface, where the SCFTilingOption contains the loop type(scf::For or scf::Forall), interchange vector, mapping vector, and tile size. Through this function, the user could easily generate a tiled implementation of the operation on the parallel axis. - -```c++ -class TilingInterface : public ::mlir::OpInterface { -public: - using ::mlir::OpInterface::OpInterface; - template - struct Trait : public detail::TilingInterfaceTrait {}; - /// Returns a list of iterator types that describe the number of loops. - SmallVector getLoopIteratorTypes(); - /// Returns a list of ranges that describe the loop bounds and - /// step for the loops of the operation. - SmallVector getIterationDomain(OpBuilder & b); - /// Method to generate the tiled implementation of an operation. - /// - /// The iteration space of the operation is returned by - /// `getIterationDomain`. The caller provides the information of the - /// tile within this iteration space whose implementation the - /// caller needs. - /// - `offsets` provides the offset of the tile in the coordinate system - /// of the original iteration space, i.e., if an iteration space - /// dimension had non-zero offset, it must be included in the offset - /// provided here (as opposed to zero-based offset "relative" to the - /// iteration space). - /// - `sizes` provides the size of the tile. - /// - /// The method returns the operation that is the tiled - /// implementation. - FailureOr getTiledImplementation(OpBuilder & b, ArrayRef offsets, ArrayRef sizes); - /// Method to return the position of the result tile computed by the tiled operation. - /// - /// Specifies what tile of the result of the original tensor is computed - /// by the tiled implementation. Expects the same `offsets` and `sizes` as - /// used to obtain the tiled implementation of the operation. - LogicalResult getResultTilePosition(OpBuilder & b, unsigned resultNumber, ArrayRef offsets, ArrayRef sizes, SmallVector & resultOffsets, SmallVector & resultSizes); - /// Method to generate the code that produces a tile of the result. - /// - /// Generates the IR that computes the tile of a result of the - /// operation. The `offsets` and `sizes` describe the tile of - /// the output required. This is different from - /// `getTiledImplementation` which generates the tiled - /// implementation of the operation given a tile of the - /// iteration space. This method generates a tiled - /// implementation of the operation based on the tile of the - /// result required. This method enables fusion by using tile - /// and fuse. The method returns failure if the operation can't be - /// tiled to generate the result tile. In practical terms this - /// implies it cannot be tiled and fused with its consumers. - /// - /// - `offsets` provides the offset of the tile in the coordinate system - /// of the original iteration space, i.e., if an iteration space - /// dimension had non-zero offset, it must be included in the offset - /// provided here (as opposed to zero-based offset "relative" to the - /// iteration space). - /// - `sizes` provides the size of the tile. - FailureOr generateResultTileValue(OpBuilder & b, unsigned resultNumber, ArrayRef offsets, ArrayRef sizes); - /// Generates the scalar implementation of the operation. - /// - /// Given the list `ivs` that represent points in the iteration space - /// (as specified by `getIterationDomain()`) returns the scalar operations - /// that represent the computation at that point in the iteration space. - /// This method is typically used as the "exit path", i.e. once all - /// transformations are done, this method can be used to lower to scalar - /// code that can then be lowered to LLVM or SPIR-V dialects. - LogicalResult generateScalarImplementation(OpBuilder & b, Location loc, ValueRange ivs); -}; - -struct SCFTilingOptions { - /// Computation function that returns the tile sizes for each operation. - /// Delayed construction of constant tile sizes should occur to interoperate - /// with folding. - SCFTileSizeComputationFunction tileSizeComputationFunction = nullptr; - - /// The interchange vector to reorder the tiled loops. - SmallVector interchangeVector = {}; - - /// Specify which loop construct to use for tile and fuse. - enum class LoopType { ForOp, ForallOp }; - LoopType loopType = LoopType::ForOp; - - /// Specify mapping of loops to devices. This is only respected when the loop - /// constructs support such a mapping (like `scf.forall`). Will be ignored - /// when using loop constructs that dont support such a mapping (like - /// `scf.for`) - SmallVector mappingVector = {}; -}; -FailureOr tileUsingSCF(RewriterBase &rewriter, - TilingInterface op, - const SCFTilingOptions &options); - -/// Rewrite a TilingInterface `op` to a tiled `scf.forall`, applying -/// tiling by `numThreads`. -/// If non-empty, the `mapping` is added as an attribute to the -/// resulting `scf.forall`. -/// Zero tile sizes indicate that the dimension is not tiled, and can be -/// thought of as tiling by the full size of data. It is the user's -/// responsibility to ensure that `numThreads` is a valid tiling specification -/// (i.e. that only tiles parallel dimensions, e.g. in the Linalg case). -struct ForallTilingResult { - Operation *tileOp; - Operation *tiledOp; -}; -FailureOr tileToForallOp(RewriterBase &builder, - TilingInterface op, - ArrayRef numThreads, - std::optional mapping); - -/// Same as `tileToForallOp`, but calculate the number of threads -/// required using the given tileSizes. -FailureOr -tileToForallOpUsingTileSizes(RewriterBase &builder, TilingInterface op, - ArrayRef tileSizes, - std::optional mapping); -``` - -The above tiling interface only supports the tiling on the parallel axis. But in a workload like matmul, it is often required to do a tiling on the reduction axis for better performance considering the size of available memory/cache, computation intensity, cache communication, etc. - -```MLIR -%red = linalg.generic {indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>, - affine_map<(d0, d1) -> (d0)>], - iterator_types = ["parallel", "reduction"]} - ins(%arg0 : tensor) - outs(%out : tensor) { - ^bb0(%arg7: f32, %arg9: f32): - %1 = arith.addf %arg7, %arg9 : f32 - linalg.yield %1 : f32 - } -> tensor -``` - -into: - -```MLIR -%0 = tensor.empty(%dim_1) : tensor -%1 = linalg.fill ins(%cst : f32) outs(%0 : tensor) -> tensor -%2 = scf.for %arg2 = %c0 to %dim_0 step %c5 iter_args(%arg3 = %1) -> (tensor) { - %extracted_slice = tensor.extract_slice %1[0, 0] [%dim, 5] [1, 1] : tensor to tensor - %extracted_slice_2 = tensor.extract_slice %arg0[0, %arg2] [%dim, 5] [1, 1] : tensor to tensor - %4 = linalg.generic {indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>, - affine_map<(d0, d1) -> (d0, d1)>], - iterator_types = ["parallel", "parallel"]} - ins(%extracted_slice_2 : tensor) - outs(%extracted_slice : tensor) { - ^bb0(%in: f32, %out: f32): - %5 = arith.addf %in, %out : f32 - linalg.yield %5 : f32 - } -> tensor - %dim_3 = tensor.dim %1, %c0 : tensor - %inserted_slice = tensor.insert_slice %4 into %arg3[0, 0] [%dim_3, 5] [1, 1] : tensor into tensor - scf.yield %inserted_slice : tensor -} -%3 = linalg.generic {indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>, - affine_map<(d0, d1) -> (d0)>], - iterator_types = ["parallel", "reduction"]} - ins(%2 : tensor) - outs(%arg1 : tensor) { - ^bb0(%in: f32, %out: f32): - %4 = arith.addf %in, %out : f32 - linalg.yield %4 : f32 - } -> tensor -``` - -To support this kind of tiling, the MLIR also provide a `PartialReductionOpInterface` based on TilingInterface. The `PartialReductionOpInterface` is an interface with a set of methods that provide information about its partial reduction and how it can be tiled. Based on the `PartialReductionOpInterface`, it further provides a function `tileReductionUsingScf(RewriterBase &b, PartialReductionOpInterface op, ArrayRef tileSize)` to support tile an op inherited the `PartialReductionOpInterface`, where the `tileSize` is the tile size for the reduction axis. - -```c++ -class PartialReductionOpInterface : public ::mlir::OpInterface { -public: - using ::mlir::OpInterface::OpInterface; - template - struct Trait : public detail::PartialReductionOpInterfaceTrait {}; - /// Method to generate a tensor initalized with the identity value of the - /// operation reduction. The tensor shape is equal to operation result - /// shape with new dimension for each non zero tile size. - FailureOr generateInitialTensorForPartialReduction(OpBuilder & b, Location loc, ArrayRef sizes, ArrayRef reductionDim); - /// Method to generate a tiled version of the operation where the tiled - /// reduction dimension are converted to parallel dimensions with a size - /// less or equal to the tile size. This is meant to be used with - /// `mergeReductions` method which will combine the partial reductions. - Operation*tileToPartialReduction(OpBuilder & b, Location loc, ValueRange init, ArrayRef offsets, ArrayRef sizes, ArrayRef reductionDims); - /// Method to merge partial reductions for an operation that has been - /// tiled along the reduction dimensions. This will only apply the - /// reduction the operation. - Operation*mergeReductions(OpBuilder & b, Location loc, ValueRange partialReduce, ArrayRef reductionDim); - -/// Method to tile a reduction and generate a parallel op within a serial loop. -/// Each of the partial reductions are calculated in parallel. Then after the -/// loop all the partial reduction are merged into a final reduction. -/// For example for the following sequence -/// -/// ```mlir -/// %0 = linalg.generic %in ["parallel", "reduction"] -/// : tensor<7x9xf32> -> tensor<7xf32> -/// ``` -/// -/// into: -/// -/// ```mlir -/// %0 = linalg.fill ... : tensor<7x4xf32> -/// %1 = scf.for ... iter_args(%arg0 = %0) -/// %2 = tensor.extract_slice %arg0 : tensor<7x4xf32> -> tensor<7x?xf32> -/// %3 = tensor.extract_slice %in : tensor<7x9xf32> -> tensor<7x?xf32> -/// %4 = linalg.generic %2, %3 ["parallel", "parallel"] -/// : tensor<7x?xf32> -> tensor<7x?xf32> -/// %5 = tensor.insert_slice %3, %0[0, 0] : tensor<7x4xf32> -/// } -/// %6 = linalg.generic %1 ["parallel", "reduction"] -/// : tensor<7x4xf32> -> tensor<7xf32> -/// ``` -FailureOr -tileReductionUsingScf(RewriterBase &b, PartialReductionOpInterface op, - ArrayRef tileSize); - -/// Method to tile a reduction to parallel iterations computing partial -/// reductions. After the loop all the partial reduction are merged into a final -/// reduction. For example for the following sequence -/// -/// ```mlir -/// %0 = linalg.generic %in ["parallel", "reduction"] -/// : tensor<7x9xf32> -> tensor<7xf32> -/// ``` -/// -/// into: -/// -/// ```mlir -/// %0 = linalg.fill ... : tensor<7x4xf32> -/// %1 = scf.forall (%iv) in (%c4) shared_outs(%arg0 = %0) -/// -> (tensor<7x4xf32>) { -/// %2 = tensor.extract_slice %arg3 : tensor<7x4xf32> to tensor<7xf32> -/// %3 = tensor.extract_slice %in : tensor<7x9xf32> -> tensor<7x?xf32> -/// %4 = linalg.generic %2, %3 ["parallel", "reduction"] -/// : tensor<7x?xf32> -> tensor<7xf32> -/// %5 = tensor.insert_slice %3, %arg0[0, %iv] : tensor<7x4xf32> -/// } -/// %6 = linalg.generic %1 ["parallel", "reduction"] -/// : tensor<7x -FailureOr -tileReductionUsingForall(RewriterBase &b, PartialReductionOpInterface op, - ArrayRef numThreads, - ArrayRef tileSizes = {}, - std::optional mapping = std::nullopt); -}; -``` - -### Hardware Abstraction Layer(HAL) - -To achieve the best performance, a good schedule requires the hardware information as a reference. Hardware information like cache size, thread number, etc. is often needed to generate the best schedule. Hardware Abstraction Layer(HAL) is a layer of software that provides a hardware-independent interface to the underlying hardware. The mainstream dl compiler or performance library has a way to get the hardware information to guide the schedule like [IREE](https://iree.dev/developers/design-docs/design-roadmap/#hal-hardware-abstraction-layer-and-multi-architecture-executables), [TVM](https://tvm.apache.org/docs/arch/device_target_interactions.html#tvm-target-specific-overview), [onednn](https://github.com/oneapi-src/oneDNN), etc. However, the MLIR doesn't have such a hardware abstraction layer(HAL) to provide the hardware information. - -## Deep-Tiled Matmul Introduction - -This section will introduce the concept of the deep-tiled matmul optimization(nested matmul/managed_matmul in graph compiler v1) and how it could improve the performance. - -Deep-tiled matmul originally is a [matmul code template](https://github.com/oneapi-src/oneDNN/blob/main/src/graph/backend/graph_compiler/core/src/ops/templates/managed_matmul_core.cpp) in the [onednn graph compiler v1](https://arxiv.org/ftp/arxiv/papers/2301/2301.01333.pdf) with well-tuned default parameters to deliver good performance in the e2e model. The basic idea of the deep-tiled matmul is to partition the iteration space of the matmul into 9 loops as the pseudocode shown below. The outermost 3 loops(`Mthreads, NThreads, KThreads`) are used to partition the iteration space of the matmul according to the number of threads, which is used to balance the workload distribution among the threads and minimize the cache synchronization/communication overhead. The middle 3 loops(`MBlock, NBlock, KBlock`) are used to partition the iteration space of the matmul and control the loop order according to the L2 cache size in the CPU, which is used to improve the data locality of the generated code. The innermost 3 loops(`innermostMBlock, innermostNBlock, innermostKBlock`) are used to partition the iteration space of the matmul and control the loop order according to the L1 cache size in CPU, which could further improve the data locality of the generated code. At this level, the matmul will be converted to the micro-kernel call [*brgemm*](https://arxiv.org/pdf/2104.05755.pdf) which is a highly optimized vectorized kernel(appling the optimiztion like unroll, operation interleave, prefetch, nt load/store, particularly tuned memory accessing pattern, carefully handcrafted register allocation). Though the tiling strategy above is based on the CPU model, it could be easily extended to the concept of the other hardware like GPU, FPGA, etc.(`global/shared memory`, `L1/2 cache size`, `execution model(threads, warp, block, grid, etc)`, etc) - -```c++ -parameter M, N, K, MBlock, NBlock, KBlock, MThreads, NThreads, KThreads, innermostMBlock, innermostNBlock, innermostKBlock -tensor A, B, C -tempC = create_tensor for C -> tensor([KThreads, M, N]) -parallel_for([PM, PN, PK]: [MThreads, NThreads, KThreads]) { - ASlice = extract_slice from A -> tensor([MOuterBlock, KOuterBlock]) - BSlice = extract_slice from B -> tensor([KOuterBlock, NOuterBlock]) - CSlice = extract_slice from C -> tensor([MOuterBlock, NOuterBlock]) - MNumBlock = MOuterBlock / MBlock - NNumBlock = NOuterBlock / NBlock - KNumBlock = KOuterBlock / KBlovk - for([om, on, ok]: [MNumBlock, NNumBlock, KNumBlock]) { - ASlice2 = extract_slice from ASlice -> tensor([MBlock, KBlock]) - BSlice2 = extract_slice from BSlice -> tensor([KBlock, NBlock]) - CSlice2 = extract_slice from CSlice -> tensor([1, MBlock, NBlock]) - MNumInnerBlock = MBlock / innermostMBlock - NNumInnerBlock = NBlock / innermostNBlock - KNumInnerBlock = KBlock / innermostKBlock - for([im, in]: [MNumInnerBlock, NNumInnerBlock]) { - ASlice3 = extract_slice from ASlice2 -> tensor([innermostMBlock, KBlock]) - BSlice3 = extract_slice from BSlice2 -> tensor([KBlock, innermostNBlock]) - CSlice3 = extract_slice from CSlice2 -> tensor([innermostMBlock, innermostNBlock]) - if(ok == 0) { - init CSlice3 with 0 (could use init_brgemm when it is avaliable) - } - brgemm(bs=KNumInnerBlock, M=innermostMBlock, N=innermostNBlock, K=innermostKBlock, -A=ASlice3, B=BSlice3, C=CSlice4, onlyUpdate=(ok!=0)); - } - } -} -C = final_reduce(tempC) -> [M, N] -``` - -## Proposal - -This section will present a proposal based on the [Option 4](#option-4---outer-loop-based-on-tiling-interface--inner-loop-through-a-predefined-template-with-ir-builder) above to implement the deep-tiled matmul in the graph compiler v2. According to the discussion above, option 4 could deliver high performance and maximally reuse the current existing work MLIR, which minimizes the difficulty of acceptance by the community. In the meantime, future optimizations like `loop reorder`, and `axis split` could be easily added by changing the parameter. So this is the recommended way in this document and the detail will be introduced in the following. - -### Position - -> The transformation control infrastructure provided by this dialect is positioned roughly between rewrite patterns and passes. A transformation that is executed by a transform operation is likely to be sufficiently complex to require at least a set of patterns to be implemented. It is also expected to be more focused than a pass: a pass typically applies identical transformations everywhere in the IR, a transform dialect-controlled transformation would apply to a small subset of operations selected, e.g., by a pattern-matching operation or generated by a previous transformation. It is discouraged, although technically possible, to run a pass pipeline as part of the transform op implementation. *From [MLIR documentation](https://mlir.llvm.org/docs/Dialects/Transform/)* - -As MLIR mentions in the documentation, the scope order from largest to smallest is `pass > Transform dialect > rewrite patterns`. The deep-tiled matmul only applies to the operation `matmul` and `batch_matmul`. So it is better to implement it as a rewrite pattern. To better meet the upstream's need, it could be warped into an operation of the `Transform` dialect so that it could become a part of the `Transform` schedule. - -In the graph compiler v2, this could be further warped in a pass `deepTilingRewriteForContractionOperation`, which could also contain other deep-tiling rewrite patterns in the future(`paddedConvolution`, `reduceLoweringConvolution`, `depthwiseConvolution`, etc). This pass is expected to be executed after the `padding/layout propagation`-related pass and before the `fusion` '-related pass. `Layout` related pass could convert the input/output tensor to the required blocked layout to achieve better performance. And fusion-related pass may depend on the tiled matmul's `insert_slice/extract_slice` as the anchor to do fusion. - -```MLIR -... -layout propogation related pass(pack/unpack, pad, propogation, etc) - -deepTilingRewriteForContractionOperation(deep-tiled matmul, deep-tiled padded conv, conv1x1, depthwise conv, etc) - -fusion related pass -... -``` - -Besides, this rewrite pattern is expected to be a part of the linalg dialect. This is similar to the existing rewrite `ConvertConv2DToImg2Col` in MLIR. In graph compiler v2, it could be a part of `linalgX` before upstream. - -### Outer Loop Generation - -For outer loop generation, we will generate the loop step by step according to the parameters/config(`outermost loop for multicore -> loop for L2 cache -> loop for L1 cache`). This part would be implemented based on the tiling interface and its related utility function, which could maximally reuse the existing work in the MLIR and decrease the difficulty of the maintenance. Besides, function like `tileToForallOp` provides an `interchange` parameter which makes it easy to change the loop order according to the workload characteristics. This way could be also easily reused by other operations like `convolution`, `depthwise convolution`, etc because they have a similar structure in this part. - -The expected implementation in pseudocode code is as follows - -```c++ -// generate outer loop with MThreads, NThreads -linalg::tileToForallOp(rewriter, cast(matmul), {MThreads, NThreads}); -// generate outer reduction loop with KThreads -linalg::tileReductionUsingForall(rewriter, cast(matmul), KThreads, tileSizes); -// generate the middle three loops(MBlock, NBlock, KBlock) -scf::tileUsingSCF(rewriter, cast(matmul),tileOption); -// generate the inner loops(innerMostMBlock, innerMostNBlock, innerMostKBlock) -scf::tileUsingSCF(rewriter, cast(matmul),tileOption); -``` - -As mentioned in the [Current Situation in the MLIR Community](#current-situation-in-the-mlir-community), there are still some missing things in the current MLIR like the lack of balance211 for not perfectly divisible cases, inefficient partial K threads position for cpu, etc. These should be further enhanced in future work. - -### Inner Loop Body Generation - -Compared to outer loop generation, the inner loop body generation is sometimes op-specific. For example, the `squeeze stride` optimization for convolution doesn't make any sense for `matmul`. Besides, this part is possibly more complex than the outer-loop(may have tail processing, non-trivial memory copy/init) and hard to unify a pass to do it. So it is better to implement it as a predefined template through IR builder which could make the code more flexible. We could also add easy builder/util support to make it more readable. - -Below is the expected pseudocode of the inner loop body for the deep-tiled matmul in the graph compiler v2. - -```c++ -A = tensor.extract_slice -B = tensor.extract_slice -C = tensor.extract_slice -D3 = scf.if(ok == 0) { - D1 = init_brgemm(A,B,C) tensor<...>, tensor<...>, tensor<...> -> tensor<...> -} else { - D2 = brgemm(A,B,C) tensor<...>, tensor<...>, tensor<...> -> tensor<...> -} -> tensor<...> -tensor.insert_slice D3 -``` - -The inner loop body will convert the `matmul` to the `batch_reduce_gemm`, which will be finally converted to the microkernel [`brgemm`](https://github.com/oneapi-src/oneDNN/pull/1852) call. - -### Config/Schedule - -```c++ -struct MatmulConfig { - int MThreads, NThreads, KThreads; - int MBlock, NBlock, KBlock; - int innerMostMBlock, innerMostNBlock, innerMostKBlock; - int loopOrder; -}; -``` - -The above is the expected config for the deep-tiled matmul. The `MThreads, NThreads, KThreads` is used to partition the iteration space of the matmul according to the number of threads. The `MBlock, NBlock, KBlock` is used to partition the iteration space of the matmul and control the loop order according to the L2 cache size in the CPU. The `innerMostMBlock, innerMostNBlock, innerMostKBlock` is used to partition the iteration space of the matmul and control the loop order according to the L1 cache size in the CPU. The `loopOrder` is used to control the loop order/iterate order according to the workload characteristics. - -A default heuristic config corresponding to these items will be tuned for the performance. - -1. For `MThreads, NThreads, KThreads`, we should rely on the available threads, required memory for the input/output/temp buffer, and the L2/L3 cache size to build a cost model that maximizes the workload balance, threads utilization and minimize the cache synchronization. But the threads on the K axis should be set carefully as it may hurt performance in most cases (performance gain on large K but small M, N). -2. For `MBlock, NBlock, KBlock`, the L2 cache size and the required memory for every core are needed to build a cost model so that the L2 cache misses would be minimized. -3. For `innerMostMBlock, innerMostNBlock, innerMostKBlock`, we need to know the L1 cache size, the size of available registers and vector/matrix-vector (amx-like) length to decide the innermost block size so that the hardware efficiency can be maximized. Besides, if we convert the brgemm to an external library function call, the cost of the function call is also needed to be considered. In the case that M/N/K is not divisible by vector length, we usually will choose a factor of the M/N/K as the innermost block size or do the packing/unpacking to make it divisible in advance(a tradeoff between reducing memory copy and maximize hardware efficiency). -4. The `loopOrder` is mainly related to the workload characteristics(data, weight, output size), the cache size and where the actual data/weight is located at L1/L2/L3/memory. This will have an impact on the visit order of the memory and finally impact the cache data locality. - -The description above shows what should be considered from the horizontal view(`[M/N/K]threads`, `[M/N/K]block`, `innermost[M/N/K]Block`, `loop order`) in the config. However, in the vertical view(`MThreads, MBlock, innermostMBlock`, `N...`, `K...`), they will have some interdependence that will also impact the performance, and the order to decide them will matter. The breakdown of how to decide is as follows. - -1. Firstly, we need to decide the `innerMostBlock[M/N/K]` which will impact the maximum hardware efficiency we can achieve, especially for the machine with a specialized matrix computation unit(amx-like). For example, if the physical matrix vector size is 16x64 and we choose the innermost block size as 8x32, then the theoretical efficiency will be a quarter of the maximum. Even for the vector instruction set like `avx512, avx2, etc`, the `innermostBlock` still matters because they still require the `innermostBlock` to align with the vector length(64/32/...). So the priority of the `innermostBlock` is the highest. -2. After the `innermostBlock` is decided, the input and output matrix will be divided into `[M/N/K]NumBlock` blocks with block size `[M/N/K]innermostBlock`. Then we will decide what `[M/N/K]Threads` should use to distribute these blocks so that the best workload balance, compute intensity and cache utilization can be achieved. -3. After step 2, the number of innermost blocks for every thread has been decided. Then we will decide the `[M/N/K]Block` to further partition the iteration space of the matmul so that the L2 cache misses in a single core would be minimized. This should be the multiples of the `innermost[M/N/K]Block`. -4. After above steps, all tile size is decided and we have enough infomation about where the data is located(L1/L2/L3 and their size). The `loopOrder` could be decided to maximize the data locality/data reuse. What it decides is the order of these loops(`pmpnpkomonokiminik`, `pnpmpkokonominimik`, etc where `p` is the outermost parallel loop, `o` is the middle outer loop, `i` is the innermost loop). - -**Note**: In the graph compiler v1, we also consider the impact of the previous matmul as this will decide where the output of the previous matmul is located (3rd core's l2 cache or 4th core's). This could be also further enhanced in the future. - -The heuristic default config will be implemented as an [analysis pass](https://mlir.llvm.org/docs/PassManagement/#analysis-management). In this way, the heuristic is maximally isolated from the real IR transformation and easier to be accepted by the upstream community(who want to separate the heuristics from passes as much as possible). By the way, other passes like layout/padding propagation could also know which tile size is preferable by the matmul and will not have a dependence cycle among these passes. - -All choices above need to be under the guidance of HAL. But the HAL support(multi-level cache size, machine kind, available threads, register vector length) is not fully ready in the MLIR now. So there is a risk here to tune a good performance for general. - -### Expected IR Change - -Below is a matmul example(`M=256, K=128, N=512`) of the expected IR change after applying the deep-tiled matmul rewrite pattern(with config `MThreads=2, NThreads=2, KThreads=1, MBlock=128, NBlock=256, KBlock=128, innerMostMBlock=32, innerMostKBlock=32, loopOrder=0`). - -```MLIR -%0 = linalg.matmul ins(%cst_0, %cst_1 : tensor<256x128xf32>, tensor<128x512xf32>) outs(%cst_2 : tensor<256x512xf32>) -> tensor<256x512xf32> -``` - -into: - -```MLIR -%0 = scf.forall (%arg0, %arg1) in (2, 2) shared_outs(%arg2 = %cst_3) -> (tensor<256x512xf32>) { - %1 = affine.apply affine_map<(d0) -> (d0 * 128)>(%arg0) - %2 = affine.apply affine_map<(d0) -> (d0 * 256)>(%arg1) - %3 = affine.apply affine_map<(d0) -> (d0 * 128)>(%arg0) - %4 = affine.apply affine_map<(d0) -> (d0 * 256)>(%arg1) - %5 = affine.apply affine_map<(d0) -> (d0 * 128)>(%arg0) - %6 = affine.apply affine_map<(d0) -> (d0 * 256)>(%arg1) - %extracted_slice = tensor.extract_slice %cst_1[%3, 0] [128, 128] [1, 1] : tensor<256x128xf32> to tensor<128x128xf32> - %extracted_slice_5 = tensor.extract_slice %cst_2[0, %4] [128, 256] [1, 1] : tensor<128x512xf32> to tensor<128x256xf32> - %extracted_slice_6 = tensor.extract_slice %arg2[%5, %6] [128, 256] [1, 1] : tensor<256x512xf32> to tensor<128x256xf32> - %c0 = arith.constant 0 : index - %c128 = arith.constant 128 : index - %c128_7 = arith.constant 128 : index - %7 = scf.for %arg3 = %c0 to %c128 step %c128_7 iter_args(%arg4 = %extracted_slice_6) -> (tensor<128x256xf32>) { - %c0_8 = arith.constant 0 : index - %c256 = arith.constant 256 : index - %c256_9 = arith.constant 256 : index - %10 = scf.for %arg5 = %c0_8 to %c256 step %c256_9 iter_args(%arg6 = %arg4) -> (tensor<128x256xf32>) { - %c0_10 = arith.constant 0 : index - %c128_11 = arith.constant 128 : index - %c128_12 = arith.constant 128 : index - %11 = scf.for %arg7 = %c0_10 to %c128_11 step %c128_12 iter_args(%arg8 = %arg6) -> (tensor<128x256xf32>) { - %extracted_slice_13 = tensor.extract_slice %extracted_slice[%arg3, %arg7] [128, 128] [1, 1] : tensor<128x128xf32> to tensor<128x128xf32> - %extracted_slice_14 = tensor.extract_slice %extracted_slice_5[%arg7, %arg5] [128, 256] [1, 1] : tensor<128x256xf32> to tensor<128x256xf32> - %extracted_slice_15 = tensor.extract_slice %arg8[%arg3, %arg5] [128, 256] [1, 1] : tensor<128x256xf32> to tensor<128x256xf32> - %c0_16 = arith.constant 0 : index - %c128_17 = arith.constant 128 : index - %c32 = arith.constant 32 : index - %12 = scf.for %arg9 = %c0_16 to %c128_17 step %c32 iter_args(%arg10 = %extracted_slice_15) -> (tensor<128x256xf32>) { - %c0_18 = arith.constant 0 : index - %c256_19 = arith.constant 256 : index - %c32_20 = arith.constant 32 : index - %13 = scf.for %arg11 = %c0_18 to %c256_19 step %c32_20 iter_args(%arg12 = %arg10) -> (tensor<128x256xf32>) { - %c0_21 = arith.constant 0 : index - %c128_22 = arith.constant 128 : index - %c128_23 = arith.constant 128 : index - %14 = scf.for %arg13 = %c0_21 to %c128_22 step %c128_23 iter_args(%arg14 = %arg12) -> (tensor<128x256xf32>) { - %extracted_slice_24 = tensor.extract_slice %extracted_slice_13[%arg9, %arg13] [32, 128] [1, 1] : tensor<128x128xf32> to tensor<32x128xf32> - %extracted_slice_25 = tensor.extract_slice %extracted_slice_14[%arg13, %arg11] [128, 32] [1, 1] : tensor<128x256xf32> to tensor<128x32xf32> - %extracted_slice_26 = tensor.extract_slice %arg14[%arg9, %arg11] [32, 32] [1, 1] : tensor<128x256xf32> to tensor<32x32xf32> - %expanded = tensor.expand_shape %extracted_slice_24 [[0, 1], [2]] : tensor<32x128xf32> into tensor<1x32x128xf32> - %expanded_27 = tensor.expand_shape %extracted_slice_25 [[0, 1], [2]] : tensor<128x32xf32> into tensor<1x128x32xf32> - %15 = linalg.batch_reduce_matmul ins(%expanded, %expanded_27 : tensor<1x32x128xf32>, tensor<1x128x32xf32>) outs(%extracted_slice_26 : tensor<32x32xf32>) -> tensor<32x32xf32> - %inserted_slice_28 = tensor.insert_slice %15 into %arg14[%arg9, %arg11] [32, 32] [1, 1] : tensor<32x32xf32> into tensor<128x256xf32> - scf.yield %inserted_slice_28 : tensor<128x256xf32> - } - scf.yield %14 : tensor<128x256xf32> - } - scf.yield %13 : tensor<128x256xf32> - } - %inserted_slice = tensor.insert_slice %12 into %arg8[%arg3, %arg5] [128, 256] [1, 1] : tensor<128x256xf32> into tensor<128x256xf32> - scf.yield %inserted_slice : tensor<128x256xf32> - } - scf.yield %11 : tensor<128x256xf32> - } - scf.yield %10 : tensor<128x256xf32> - } - %8 = affine.apply affine_map<(d0) -> (d0 * 128)>(%arg0) - %9 = affine.apply affine_map<(d0) -> (d0 * 256)>(%arg1) - scf.forall.in_parallel { - tensor.parallel_insert_slice %7 into %arg2[%8, %9] [128, 256] [1, 1] : tensor<128x256xf32> into tensor<256x512xf32> - } -} -``` - -When the `KThreads=2`, there will be partial reduction in the loop - -```MLIR -%0 = scf.forall (%arg0, %arg1) in (2, 2) shared_outs(%arg2 = %cst_3) -> (tensor<256x512xf32>) { - %1 = affine.apply affine_map<(d0) -> (d0 * 128)>(%arg0) - %2 = affine.apply affine_map<(d0) -> (d0 * 256)>(%arg1) - %3 = affine.apply affine_map<(d0) -> (d0 * 128)>(%arg0) - %4 = affine.apply affine_map<(d0) -> (d0 * 256)>(%arg1) - %5 = affine.apply affine_map<(d0) -> (d0 * 128)>(%arg0) - %6 = affine.apply affine_map<(d0) -> (d0 * 256)>(%arg1) - %extracted_slice = tensor.extract_slice %cst_1[%3, 0] [128, 128] [1, 1] : tensor<256x128xf32> to tensor<128x128xf32> - %extracted_slice_5 = tensor.extract_slice %cst_2[0, %4] [128, 256] [1, 1] : tensor<128x512xf32> to tensor<128x256xf32> - %extracted_slice_6 = tensor.extract_slice %arg2[%5, %6] [128, 256] [1, 1] : tensor<256x512xf32> to tensor<128x256xf32> - %c0 = arith.constant 0 : index - %c0_7 = arith.constant 0 : index - %c2_8 = arith.constant 2 : index - %7 = tensor.empty() : tensor<128x256x2xf32> - %cst_9 = arith.constant 0.000000e+00 : f32 - %8 = linalg.fill ins(%cst_9 : f32) outs(%7 : tensor<128x256x2xf32>) -> tensor<128x256x2xf32> - %c2_10 = arith.constant 2 : index - %9 = scf.forall (%arg3) in (2) shared_outs(%arg4 = %8) -> (tensor<128x256x2xf32>) { - %13 = affine.apply affine_map<(d0) -> (d0 * 64)>(%arg3) - %extracted_slice_11 = tensor.extract_slice %arg4[0, 0, %arg3] [128, 256, 1] [1, 1, 1] : tensor<128x256x2xf32> to tensor<128x256xf32> - %c0_12 = arith.constant 0 : index - %c128 = arith.constant 128 : index - %c128_13 = arith.constant 128 : index - %14 = affine.apply affine_map<()[s0, s1] -> (s0 * s1)>()[%arg3, %c128_13] - %15 = affine.apply affine_map<()[s0, s1] -> (s0 + s1)>()[%14, %c0_12] - %16 = affine.apply affine_map<()[s0, s1] -> (s0 * s1)>()[%c2_10, %c128_13] - %17 = scf.for %arg5 = %15 to %c128 step %16 iter_args(%arg6 = %extracted_slice_11) -> (tensor<128x256xf32>) { - %extracted_slice_14 = tensor.extract_slice %extracted_slice[0, %arg5] [128, 128] [1, 1] : tensor<128x128xf32> to tensor<128x128xf32> - %extracted_slice_15 = tensor.extract_slice %extracted_slice_5[%arg5, 0] [128, 256] [1, 1] : tensor<128x256xf32> to tensor<128x256xf32> - %extracted_slice_16 = tensor.extract_slice %arg6[0, 0] [128, 256] [1, 1] : tensor<128x256xf32> to tensor<128x256xf32> - %c0_17 = arith.constant 0 : index - %c128_18 = arith.constant 128 : index - %c128_19 = arith.constant 128 : index - %18 = scf.for %arg7 = %c0_17 to %c128_18 step %c128_19 iter_args(%arg8 = %extracted_slice_16) -> (tensor<128x256xf32>) { - %c0_20 = arith.constant 0 : index - %c256 = arith.constant 256 : index - %c256_21 = arith.constant 256 : index - %19 = scf.for %arg9 = %c0_20 to %c256 step %c256_21 iter_args(%arg10 = %arg8) -> (tensor<128x256xf32>) { - %c0_22 = arith.constant 0 : index - %c128_23 = arith.constant 128 : index - %c64 = arith.constant 64 : index - %20 = scf.for %arg11 = %c0_22 to %c128_23 step %c64 iter_args(%arg12 = %arg10) -> (tensor<128x256xf32>) { - %extracted_slice_24 = tensor.extract_slice %extracted_slice_14[%arg7, %arg11] [128, 64] [1, 1] : tensor<128x128xf32> to tensor<128x64xf32> - %extracted_slice_25 = tensor.extract_slice %extracted_slice_15[%arg11, %arg9] [64, 256] [1, 1] : tensor<128x256xf32> to tensor<64x256xf32> - %extracted_slice_26 = tensor.extract_slice %arg12[%arg7, %arg9] [128, 256] [1, 1] : tensor<128x256xf32> to tensor<128x256xf32> - %c0_27 = arith.constant 0 : index - %c128_28 = arith.constant 128 : index - %c32 = arith.constant 32 : index - %21 = scf.for %arg13 = %c0_27 to %c128_28 step %c32 iter_args(%arg14 = %extracted_slice_26) -> (tensor<128x256xf32>) { - %c0_30 = arith.constant 0 : index - %c256_31 = arith.constant 256 : index - %c32_32 = arith.constant 32 : index - %22 = scf.for %arg15 = %c0_30 to %c256_31 step %c32_32 iter_args(%arg16 = %arg14) -> (tensor<128x256xf32>) { - %c0_33 = arith.constant 0 : index - %c64_34 = arith.constant 64 : index - %c64_35 = arith.constant 64 : index - %23 = scf.for %arg17 = %c0_33 to %c64_34 step %c64_35 iter_args(%arg18 = %arg16) -> (tensor<128x256xf32>) { - %extracted_slice_36 = tensor.extract_slice %extracted_slice_24[%arg13, %arg17] [32, 64] [1, 1] : tensor<128x64xf32> to tensor<32x64xf32> - %extracted_slice_37 = tensor.extract_slice %extracted_slice_25[%arg17, %arg15] [64, 32] [1, 1] : tensor<64x256xf32> to tensor<64x32xf32> - %extracted_slice_38 = tensor.extract_slice %arg18[%arg13, %arg15] [32, 32] [1, 1] : tensor<128x256xf32> to tensor<32x32xf32> - %expanded = tensor.expand_shape %extracted_slice_36 [[0, 1], [2]] : tensor<32x64xf32> into tensor<1x32x64xf32> - %expanded_39 = tensor.expand_shape %extracted_slice_37 [[0, 1], [2]] : tensor<64x32xf32> into tensor<1x64x32xf32> - %24 = linalg.batch_reduce_matmul ins(%expanded, %expanded_39 : tensor<1x32x64xf32>, tensor<1x64x32xf32>) outs(%extracted_slice_38 : tensor<32x32xf32>) -> tensor<32x32xf32> - %inserted_slice_40 = tensor.insert_slice %24 into %arg18[%arg13, %arg15] [32, 32] [1, 1] : tensor<32x32xf32> into tensor<128x256xf32> - scf.yield %inserted_slice_40 : tensor<128x256xf32> - } - scf.yield %23 : tensor<128x256xf32> - } - scf.yield %22 : tensor<128x256xf32> - } - %inserted_slice_29 = tensor.insert_slice %21 into %arg12[%arg7, %arg9] [128, 256] [1, 1] : tensor<128x256xf32> into tensor<128x256xf32> - scf.yield %inserted_slice_29 : tensor<128x256xf32> - } - scf.yield %20 : tensor<128x256xf32> - } - scf.yield %19 : tensor<128x256xf32> - } - %inserted_slice = tensor.insert_slice %18 into %arg6[0, 0] [128, 256] [1, 1] : tensor<128x256xf32> into tensor<128x256xf32> - scf.yield %inserted_slice : tensor<128x256xf32> - } - scf.forall.in_parallel { - tensor.parallel_insert_slice %17 into %arg4[0, 0, %arg3] [128, 256, 1] [1, 1, 1] : tensor<128x256xf32> into tensor<128x256x2xf32> - } - } - %10 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1)>], iterator_types = ["parallel", "parallel", "reduction"]} ins(%9 : tensor<128x256x2xf32>) outs(%extracted_slice_6 : tensor<128x256xf32>) { - ^bb0(%in: f32, %out: f32): - %13 = arith.addf %in, %out : f32 - linalg.yield %13 : f32 - } -> tensor<128x256xf32> - %11 = affine.apply affine_map<(d0) -> (d0 * 128)>(%arg0) - %12 = affine.apply affine_map<(d0) -> (d0 * 256)>(%arg1) - scf.forall.in_parallel { - tensor.parallel_insert_slice %10 into %arg2[%11, %12] [128, 256] [1, 1] : tensor<128x256xf32> into tensor<256x512xf32> - } -} -``` \ No newline at end of file diff --git a/docs/dialect_overview.png b/docs/dialect_overview.png deleted file mode 100644 index 26ffa9188..000000000 Binary files a/docs/dialect_overview.png and /dev/null differ diff --git a/docs/memref_schedule.md b/docs/memref_schedule.md deleted file mode 100644 index dafbfd6bb..000000000 --- a/docs/memref_schedule.md +++ /dev/null @@ -1,121 +0,0 @@ -# RFC: Compile-time memref.alloc Scheduling/Merging - -This document proposes a compile-time optimization on existing `memref.alloc` to reduce memory usage and improve memory locality. - -## Current status of bufferization and memref pass pipeline -Bufferization is a process in the current MLIR of converting ops with tensor semantics to ops with memref semantics. Currently, MLIR has two different bufferization passes, one-shot-bufferization and older/partial bufferization (legacy version). -One-Shot Bufferize is a new tensor bufferization pass designed for IR in destination-passing style, and with aggressive in-place bufferization. The older/partial bufferization was built around multiple dialects. The community is trying to gradually deprecate the older bufferization and replace them with one-shot bufferization. -The goal of bufferization is to use as little memory as possible and copy as little memory as possible, as a result, the exsiting focus is to determine in-place or out-of-place among the OpOperand and OpResult of individual ops, while not considering much about the overall memory reuse across Operators within a sub-graph (or partition). - -The current implementation of Bufferization and memref pass pipeline focuses on copy-avoidance and in-place reusing of the memory. Consider a computation graph of 4 layers of matmul sharing the same weight: -```mlir -func.func @mlp(%x: tensor<128x128xf32>, %y: tensor<128x128xf32>) -> tensor<128x128xf32> { - %a0 = tensor.empty() : tensor<128x128xf32> - %a = linalg.matmul ins(%x, %y: tensor<128x128xf32>, tensor<128x128xf32>) outs(%a0: tensor<128x128xf32>) -> tensor<128x128xf32> - %b0 = tensor.empty() : tensor<128x128xf32> - %b = linalg.matmul ins(%a, %y: tensor<128x128xf32>, tensor<128x128xf32>) outs(%b0: tensor<128x128xf32>) -> tensor<128x128xf32> - %c0 = tensor.empty() : tensor<128x128xf32> - %c = linalg.matmul ins(%b, %y: tensor<128x128xf32>, tensor<128x128xf32>) outs(%c0: tensor<128x128xf32>) -> tensor<128x128xf32> - %d0 = tensor.empty() : tensor<128x128xf32> - %d = linalg.matmul ins(%c, %y: tensor<128x128xf32>, tensor<128x128xf32>) outs(%d0: tensor<128x128xf32>) -> tensor<128x128xf32> - return %d : tensor<128x128xf32> -} -``` - -The bufferization pass will create an `memref.alloc` for each of the tensor `a0`, `b0` and `c0`. The bufferization result should be like: - -```mlir -func.func @mlp(%x: memref<128x128xf32>, %y: memref<128x128xf32>) -> memref<128x128xf32> { - %a0 = memref.alloc() : memref<128x128xf32> - linalg.matmul ins(%x, %y: memref<128x128xf32>, memref<128x128xf32>) outs(%a0: memref<128x128xf32>) - %b0 = memref.alloc() : memref<128x128xf32> - linalg.matmul ins(%a0, %y: memref<128x128xf32>, memref<128x128xf32>) outs(%b0: memref<128x128xf32>) - %c0 = memref.alloc() : memref<128x128xf32> - linalg.matmul ins(%b0, %y: memref<128x128xf32>, memref<128x128xf32>) outs(%c0: memref<128x128xf32>) - %d0 = memref.alloc() : memref<128x128xf32> - linalg.matmul ins(%c0, %y: memref<128x128xf32>, memref<128x128xf32>) outs(%d0: memref<128x128xf32>) - return %d0 : memref<128x128xf32> -} -``` - -Without further optimizations, 3 temp buffers will be allocated at the runtime for these tensors. However, as we can see in the IR, the buffer `a0` is no longer used when buffer `c0` is allocated. So buffer `c0` can reuse the memory buffer of buffer `a0`, to reduce the memory size footprint and improve the locality. - -An observation of the current bufferization and memref passes is that they do not consider the memory buffer planning - to reuse the buffer/memref for less total size and better locality. - -## Proposal -This RFC proposes an optimization to consolidate multiple allocations (`memref.alloc` ops) into a single `memref.alloc` op and each static-shaped `memref.alloc` op will be transformed into a "slice" from the `single allocated buffer` with `memref.view` and some compile-time decided `offsets`. This optimization works on `memref` instead of `tensor` ops, so it should be executed after bufferization pass, and before buffer-deallocation. - -While merging the memory allocations, the transform should consider the lifetime of each allocated `memref`s. By lifetime, we mean the range of time when an memref allocated from `memref.alloc` is actively used. The references on `view`s of a "base" `memref` should contribute to the lifetime of the "base". A later `memref.alloc` should consider to reuse the memory of a previously allocated memref, if the lifetime of these two does not overlap. The transform will perform the "reusing" of memory by setting the `offset` of the later `memref.view` to a position within the memory range of a previous allocation's `memref.view` on the `single allocated buffer`. - -Below is the expected transformation result of the example IR in the above section: - -```mlir -func.func @mlp(%x: memref<128x128xf32>, %y: memref<128x128xf32>) -> memref<128x128xf32> { - %single_buffer = memref.alloc() : memref<131072xi8> // 128*128*sizeof(f32)*2 - %a0 = memref.view %single_buffer[0][] : memref<131072xi8> to memref<128x128xf32> // a0 takes the memory from byte offset 0 - linalg.matmul ins(%x, %y: memref<128x128xf32>, memref<128x128xf32>) outs(%a0: memref<128x128xf32>) - %b0 = memref.view %single_buffer[65536][] : memref<131072xi8> to memref<128x128xf32> // b0 takes the memory from byte offset 128*128*sizeof(f32) - linalg.matmul ins(%a0, %y: memref<128x128xf32>, memref<128x128xf32>) outs(%b0: memref<128x128xf32>) - %c0 = memref.view %single_buffer[0][] : memref<131072xi8> to memref<128x128xf32> // c0 takes the memory from byte offset 0 - linalg.matmul ins(%b0, %y: memref<128x128xf32>, memref<128x128xf32>) outs(%c0: memref<128x128xf32>) - %d0 = memref.alloc() : memref<128x128xf32> // d0 is returned, do not merge - linalg.matmul ins(%c0, %y: memref<128x128xf32>, memref<128x128xf32>) outs(%d0: memref<128x128xf32>) - return %d0 : memref<128x128xf32> -} -``` - -There is one single allocation `single_buffer` for all temp buffers and `alloc` ops for `a0`, `b0` and `c0` are removed. The returned memref `d0` is untouched. The memrefs `a0`, `b0` and `c0` are replaced by `memref.view` on `single_buffer`. Since `a0` and `b0`'s lifetime overlaps, the transformation will "allocate" different memory ranges on the `single_buffer` - note that `a0` and `b0` has different offsets `%single_buffer[0]` and `%single_buffer[65536]` and the memory ranges does not overlap. The memref `c0` does not overlap with `a0` in their lifetime, so that `c0` can reuse the memory range of `a0` by setting of offset to `%single_buffer[0]`, which is the same of `a0`. The final allocation size of temp memory buffer will be `128*128*sizeof(f32)*2` instead of three `memref<128x128xf32>` buffers in the original IR. - -The transformation should only consider to merge a `memref.alloc` only if - * the ownership of the memref does not escape from the function. That is, the current function is responsible to alloc and dealloc this memref - * and, the allocated memref is contiguous and has static shape and identical layout. - -In this RFC, we call these `memref.alloc` **mergeable** allocations. - -The memrefs passed by function arguments, or returned by the function will be untouched by this optimization. - -## Other solutions - -Another (not yet existing) approach to resolve the memory reusing issue is to insert `memref.dealloc` as soon as the buffer is no longer used. For example, in the above "matmul" example, `memref.dealloc` can be inserted after the last use of `a0` at `linalg.matmul ins(%a0, %y...)`. So even without memref merging transformation, a common runtime memory allocator will try to reuse the memory free'd by `memref.dealloc(%a0)` when allocating buffer for `c0`. However, there are some disadvantages of this approach comparing to the compile-time memref merging transformation of this proposal: -1. it depends on the implementation of the runtime memory allocator. -2. the runtime memory allocator does not have full picture of the future allocation/deallocation patterns of the program. For example, if we change the above example to make buffer size `c0` greater than size of `a0`, the runtime memory allocator will not likely to be able to reuse the memory of `a0` for `c0`, becuase the free memory chunk size of `a0` does not fit allocation of `c0`. In contrast, the proposed optimization of this document has the knowledge of the allocation patterns. Thus, it can put the memory chunk for `a0` in a right place of the `single allocation buffer`, so that the allocation of `c0` can fit into it. -3. calling runtime memory allocator for each buffer introduces more run time overhead than a single merged allocation after allocation merging. - -However, utilizing runtime memory allocator can be viewed as a supplementary approach of the allocation merging at compile-time, for example, to handle memref with dynamic shapes. These two memory optimization approaches should coexist and cowork in the pass pipeline. - -## Implementation -*The detail implementation will leverage the exising algorithm used in GC V1.* - -The transformation first needs to identify the `alloc scopes`, which are mlir `Block`s - * implementing `AutomaticAllocationScope` - * and is not `scf.for` (allocations in an `scf.for` can be hoisted to parent `AutomaticAllocationScope`) - -For example, below is an example IR of a function with nested `scf.forall` ops. - -```mlir -func.func @mlp(...) { // <---- alloc scope 1 - scf.for(...) { // <---- NOT an alloc scope! - // allocation inside will be merge to alloc scope 1 above - } - ... - scf.forall(...) { // <---- alloc scope 2 - ... - // allocation here will be merge to alloc scope 2 - %buf = memref.alloc() : ... - scf.forall(...) { // <---- alloc scope 3 - } - } -} -``` - -There will be three `alloc scopes` as marked in the comments above. An `alloc scope` marks the position to insert the `single allocation buffer` after allocation merging. After the transformation, all "mergeable" `memref.alloc` will be merged to the `single allocation buffer` of the nearest ancestor `alloc scope`. - -The transformantion is consist of an analysis sub-pass and a mutation sub-pass. For each `alloc scope`, the analysis sub-pass finds the lifetime of each mergeable `memref.alloc` belonging to the `alloc scope`. And given the lifetime of each allocation, a memory planning algorithm will be run to find the `single allocation buffer` size of each `alloc scope` and the `offset` for each mergeable allocation within its `single allocation buffer`. Based on the memory planning result, the mutation sub-pass transforms the IR to -1. insert `memref.alloc` at the front of `alloc scope` body for its `single allocation buffer` -2. replace mergeable `memref.alloc` with `memref.view` on its `alloc scope`'s `single allocation buffer` - -Ticks are assigned on each operation in the `func.func` by a increasing counter with pre-order recursive walking of the IR, as the "execution tick" for each operation. The lifetime analysis pass will assign two integers for each mergeable allocations as the analysis result: `begin_tick` and `end_tick`, to indicate the first and last tick of the use of the allocated memref in the IR. There should be special handling for loop and branch ops (`RegionBranchOpInterface` or `LoopLikeOpInterface`) which references memrefs allocated in parent scopes, to avoid wrong reuse of buffers used in the loop. - -The analysis result for each mergeable allocations will be an integer range `[begin_tick,end_tick]`, where `begin_tick <= end_tick`. - -The collected ticks for each buffer will be processed by the memory planning algorithm. It should output the total size of the `single allocation buffers` for each `alloc scopes`, and the `offsets` for each individual mergeable buffers. The algorithm should also consider the locality of the buffer to use, when multiple buffer localtion candidates are available. diff --git a/include/CMakeLists.txt b/include/CMakeLists.txt index 49854749f..16579558f 100644 --- a/include/CMakeLists.txt +++ b/include/CMakeLists.txt @@ -1 +1 @@ -add_subdirectory(gc) \ No newline at end of file +add_subdirectory(gc) diff --git a/include/gc-c/Dialects.h b/include/gc-c/Dialects.h deleted file mode 100644 index 2a8670c43..000000000 --- a/include/gc-c/Dialects.h +++ /dev/null @@ -1,37 +0,0 @@ - -/* - * Copyright (C) 2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef GC_MLIR_C_DIALECTS_H -#define GC_MLIR_C_DIALECTS_H - -#include "mlir-c/IR.h" - -#ifdef __cplusplus -extern "C" { -#endif - -#ifdef GC_HAS_ONEDNN_DIALECT -MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(OneDNNGraph, onednn_graph); -#endif -MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(CPURuntime, cpuruntime); - -#ifdef __cplusplus -} -#endif -#endif // GC_MLIR_C_DIALECTS_H \ No newline at end of file diff --git a/include/gc-c/Passes.h b/include/gc-c/Passes.h deleted file mode 100644 index 0cbd13069..000000000 --- a/include/gc-c/Passes.h +++ /dev/null @@ -1,35 +0,0 @@ - -/* - * Copyright (C) 2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef GCEXT_MLIR_C_PASSES_H -#define GCEXT_MLIR_C_PASSES_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "gc/Dialect/CPURuntime/Transforms/CPURuntimePasses.capi.h.inc" -#include "gc/Transforms/Passes.capi.h.inc" - -MLIR_CAPI_EXPORTED void mlirRegisterAllGCPassesAndPipelines(void); - -#ifdef __cplusplus -} -#endif -#endif // GCEXT_MLIR_C_PASSES_H \ No newline at end of file diff --git a/include/gc/Analysis/MatmulConfigAnalysis.h b/include/gc/Analysis/MatmulConfigAnalysis.h deleted file mode 100644 index 0b5d390dd..000000000 --- a/include/gc/Analysis/MatmulConfigAnalysis.h +++ /dev/null @@ -1,161 +0,0 @@ -//===-- MatmulConfigAnalysis.h - the analysis for matmul config -*- C++ -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef MLIR_ANALYSIS_MATMULCONFIGANALYSIS_H -#define MLIR_ANALYSIS_MATMULCONFIGANALYSIS_H - -#include "gc/Dialect/Linalgx/LinalgxOps.h" -#include "gc/Dialect/Linalgx/Utils.h" -#include "mlir/Dialect/DLTI/DLTI.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Interfaces/DataLayoutInterfaces.h" - -namespace mlir { -namespace gc { - -using namespace mlir; - -// The configuration for matmul tiling -// TODO: support batch matmul -struct MatmulConfig { - // The number of threads distributed to M, N, K - uint32_t MThreads, NThreads, KThreads; - // The outer block size for M, N, K which will be used to decide the loop tile - // size in single thread - uint32_t MBlock, NBlock, KBlock; - // The innermost block size for M, N, K which will be directly converted to - // brgemm. - uint32_t innerMostMBlock, innerMostNBlock, innerMostKBlock; -}; - -bool validateConfig(const MatmulConfig &cfg, ArrayRef shape, - bool allowIndivisibleInnerblock, bool isVNNIMM2D); - -enum DimType { Batch, M, N, K }; - -// Extract the index of the given DimType in the DimType list -inline SmallVector extractDimTypeIdx(ArrayRef tyList, - DimType ty) { - SmallVector idxList; - for (auto [idx, type] : llvm::enumerate(tyList)) { - if (type == ty) { - idxList.push_back(idx); - } - } - return idxList; -} - -inline void getDimTypeFromIterators(linalg::LinalgOp linalgOp, - SmallVectorImpl &dimTypes) { - SmallVector iteratorTypes = - linalgOp.getIteratorTypesArray(); - - for (const auto &&[idx, iterType] : llvm::enumerate(iteratorTypes)) { - if (iterType == mlir::utils::IteratorType::parallel) { - SmallVector> operandDimPairs; - linalgOp.mapIterationSpaceDimToAllOperandDims(idx, operandDimPairs); - if (operandDimPairs.size() == 3) { - dimTypes.push_back(DimType::Batch); - } else if (llvm::any_of(operandDimPairs, - [&](std::pair pair) { - return pair.first == - dyn_cast( - linalgOp.getOperation()) - .lhs(); - })) { - dimTypes.push_back(DimType::M); - } else { - dimTypes.push_back(DimType::N); - } - } else if (iterType == mlir::utils::IteratorType::reduction) { - dimTypes.push_back(DimType::K); - } - } -} - -inline SmallVector -matchOperandToDimTypes(linalg::LinalgOp linalgOp, OpOperand *operand, - ArrayRef allDimTypes) { - ArrayRef map = - linalgOp.getMatchingIndexingMap(operand).getResults(); - SmallVector res; - for (const AffineExpr &dim : map) { - AffineDimExpr dimExpr = dyn_cast(dim); - res.push_back(allDimTypes[dimExpr.getPosition()]); - } - return res; -} - -inline SmallVector> -getContractionOpOperandDimType(linalg::LinalgOp linalgOp) { - SmallVector dimTypes; - getDimTypeFromIterators(linalgOp, dimTypes); - SmallVector ADimTypes = matchOperandToDimTypes( - linalgOp, linalgOp.getDpsInputOperand(0), dimTypes); - SmallVector BDimTypes = matchOperandToDimTypes( - linalgOp, linalgOp.getDpsInputOperand(1), dimTypes); - SmallVector CDimTypes = - matchOperandToDimTypes(linalgOp, linalgOp.getDpsInitOperand(0), dimTypes); - - return SmallVector>{ADimTypes, BDimTypes, CDimTypes}; -} - -// Get the operand dim type for every operand for the given linalg op -inline FailureOr>> -getOprandDimType(linalg::LinalgOp &linalgOp) { - // TODO: replace the linalgx op with generic op - if (llvm::isa(linalgOp.getOperation())) { - return getContractionOpOperandDimType(linalgOp); - } else if (linalgx::isGenericPackedMatmulOp( - linalgOp.getOperation(), linalgx::PackingType::VNNI_MM2D)) { - return SmallVector>{ - SmallVector{DimType::M, DimType::K}, - SmallVector{DimType::N, DimType::K, DimType::K, DimType::N, - DimType::K}, - SmallVector{DimType::M, DimType::N, DimType::M, DimType::N}}; - } else if (linalgx::isGenericPackedMatmulOp( - linalgOp.getOperation(), linalgx::PackingType::VNNI_MM4D)) { - return SmallVector>{ - SmallVector{DimType::M, DimType::K, DimType::M, DimType::K}, - SmallVector{DimType::N, DimType::K, DimType::K, DimType::N, - DimType::K}, - SmallVector{DimType::M, DimType::N, DimType::M, DimType::N}}; - } else if (linalgx::isGenericPackedMatmulOp(linalgOp.getOperation(), - linalgx::PackingType::MM4D)) { - return SmallVector>{ - SmallVector{DimType::M, DimType::K, DimType::M, DimType::K}, - SmallVector{DimType::N, DimType::K, DimType::K, DimType::N}, - SmallVector{DimType::M, DimType::N, DimType::M, DimType::N}}; - } - return failure(); -} - -// The analysis to extract the matmul configuration from the given linalg op -struct MatmulConfigAnalysis { -public: - // Extract the matmul configuration from the given linalg op - MatmulConfigAnalysis(Operation *root) : root(root){}; - - // Get the matmul configuration - MatmulConfig getConfig(); - - void setAllowIndivisibleInnerBlock(bool allow) { - allowIndivisibleInnerBlock = allow; - } - -private: - MatmulConfig config = MatmulConfig{1, 1, 1, 1, 1, 1, 1, 1, 1}; - Operation *root; - bool hasConfig = false; - bool allowIndivisibleInnerBlock = true; -}; - -} // namespace gc -} // namespace mlir - -#endif \ No newline at end of file diff --git a/include/gc/Analysis/TargetDescriptionAnalysis.h b/include/gc/Analysis/TargetDescriptionAnalysis.h deleted file mode 100644 index 3fe3576d1..000000000 --- a/include/gc/Analysis/TargetDescriptionAnalysis.h +++ /dev/null @@ -1,95 +0,0 @@ -//===-- TargetDescriptionAnalysis.h - target description class --*- C++ -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef MLIR_ANALYSIS_TARGETDESCRIPTIONANALYSIS_H -#define MLIR_ANALYSIS_TARGETDESCRIPTIONANALYSIS_H - -#include "mlir/Dialect/DLTI/DLTI.h" -#include "mlir/IR/BuiltinDialect.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/PatternMatch.h" -#include "mlir/Interfaces/DataLayoutInterfaces.h" -#include "mlir/Support/LLVM.h" -#include "llvm/ADT/StringRef.h" - -namespace mlir { -namespace gc { - -using namespace mlir; - -enum DeviceType { CPU = 0 }; - -class TargetDescriptionAnalysisBase { -public: - TargetDescriptionAnalysisBase(Operation *op, DeviceType device) - : ctx(op->getContext()), device(device), - layout(isa(op) ? dyn_cast(op) - : op->getParentOfType()), - loc(op->getLoc()) {} - - // get the device ID - DeviceType getDevice() { return device; } - - // get the MLIR context - MLIRContext *getContext() { return ctx; } - - // get the data layout - DataLayout getLayout() { return layout; } - - // get the property value by key - std::optional getPropertyValue(StringRef key); - - // get the location - Location getLocation() { return loc; } - - // check if the property exists - bool hasProperty(StringRef key) { return getPropertyValue(key).has_value(); } - - // emit warning if the property is not found - template - void emitNotFoundWarning(Location loc, StringRef key, T value); - - // the map from device type to device string - static llvm::DenseMap DeviceKeyMap; - - // set the emit warning flag - void setEmitWarning(bool emit) { emitWarning = emit; } - -private: - MLIRContext *ctx; - DeviceType device; - DataLayout layout; - Location loc; - bool emitWarning = false; -}; - -class CPUTargetDescriptionAnalysis : public TargetDescriptionAnalysisBase { -public: - static constexpr StringLiteral kL1CacheSize = "L1_cache_size_in_bytes"; - static constexpr StringLiteral kL2CacheSize = "L2_cache_size_in_bytes"; - static constexpr StringLiteral kL3CacheSize = "L3_cache_size_in_bytes"; - static constexpr StringLiteral kMaxVectorWidth = "max_vector_width"; - static constexpr StringLiteral kNumThreads = "num_threads"; - - // get runtime OMP_NUM_THREADS - unsigned getNumThreads(); - - // get cache size by cacheLevel - unsigned getCacheSize(uint8_t cacheLevel); - - // get the maximum vector length in bits - unsigned getMaxVectorWidth(); - - CPUTargetDescriptionAnalysis(Operation *op) - : TargetDescriptionAnalysisBase(op, DeviceType::CPU) {} -}; - -} // namespace gc -} // namespace mlir - -#endif \ No newline at end of file diff --git a/include/gc/CMakeLists.txt b/include/gc/CMakeLists.txt index ec5380c5c..557daa847 100644 --- a/include/gc/CMakeLists.txt +++ b/include/gc/CMakeLists.txt @@ -1,3 +1,2 @@ add_subdirectory(Dialect) add_subdirectory(Transforms) -add_subdirectory(Conversion) diff --git a/include/gc/Conversion/CMakeLists.txt b/include/gc/Conversion/CMakeLists.txt deleted file mode 100644 index cecfeb051..000000000 --- a/include/gc/Conversion/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -set(LLVM_TARGET_DEFINITIONS Passes.td) -mlir_tablegen(Passes.h.inc -gen-pass-decls -name GCConversion) -mlir_tablegen(Passes.capi.h.inc -gen-pass-capi-header --prefix GCConversion) -mlir_tablegen(Passes.capi.cpp.inc -gen-pass-capi-impl --prefix GCConversion) -add_public_tablegen_target(GCConversionPassIncGen) - -add_mlir_doc(Passes GCConversionPasses ./ -gen-pass-doc) \ No newline at end of file diff --git a/include/gc/Conversion/Passes.h b/include/gc/Conversion/Passes.h deleted file mode 100644 index 3eb6c09b9..000000000 --- a/include/gc/Conversion/Passes.h +++ /dev/null @@ -1,23 +0,0 @@ -//===-- Passes.h - Conversion Passes ---------------------------*- C++ -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef GC_CONVERSION_PASSES_H -#define GC_CONVERSION_PASSES_H - -#include "gc/Conversion/XeVMToLLVM/XeVMToLLVM.h" -#include "mlir/Pass/Pass.h" - -namespace mlir { - -/// Generate the code for registering conversion passes. -#define GEN_PASS_REGISTRATION -#include "gc/Conversion/Passes.h.inc" - -} // namespace mlir - -#endif // GC_CONVERSION_PASSES_H diff --git a/include/gc/Conversion/Passes.td b/include/gc/Conversion/Passes.td deleted file mode 100644 index 91cc04745..000000000 --- a/include/gc/Conversion/Passes.td +++ /dev/null @@ -1,25 +0,0 @@ -//===-- Passes.td - Conversion pass definition file --------*- tablegen -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef GC_CONVERSION_PASSES -#define GC_CONVERSION_PASSES - -include "mlir/Pass/PassBase.td" - -//===----------------------------------------------------------------------===// -// XeVMToLLVM -//===----------------------------------------------------------------------===// - -def ConvertXeVMToLLVMPass : Pass<"convert-xevm-to-llvm"> { - let summary = "Convert XeVM to LLVM dialect"; - let dependentDialects = [ - "xevm::XeVMDialect", - ]; -} - -#endif // GC_CONVERSION_PASSES diff --git a/include/gc/Conversion/XeVMToLLVM/XeVMToLLVM.h b/include/gc/Conversion/XeVMToLLVM/XeVMToLLVM.h deleted file mode 100644 index b693d1514..000000000 --- a/include/gc/Conversion/XeVMToLLVM/XeVMToLLVM.h +++ /dev/null @@ -1,28 +0,0 @@ -//===-- XeVMToLLVM.h - Convert XeVM to LLVM dialect -------------*- C++ -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -#ifndef MLIR_CONVERSION_XEVMTOLLVM_XEVMTOLLVMPASS_H_ -#define MLIR_CONVERSION_XEVMTOLLVM_XEVMTOLLVMPASS_H_ - -#include - -namespace mlir { -class DialectRegistry; -class LLVMTypeConverter; -class RewritePatternSet; -class Pass; - -#define GEN_PASS_DECL_CONVERTXEVMTOLLVMPASS -#include "gc/Conversion/Passes.h.inc" - -void populateXeVMToLLVMConversionPatterns(RewritePatternSet &patterns); - -void registerConvertXeVMToLLVMInterface(DialectRegistry ®istry); - -} // namespace mlir - -#endif // MLIR_CONVERSION_XEVMTOLLVM_XEVMTOLLVMPASS_H_ diff --git a/include/gc/Dialect/CMakeLists.txt b/include/gc/Dialect/CMakeLists.txt index c2fe46c4d..d0201e3e4 100644 --- a/include/gc/Dialect/CMakeLists.txt +++ b/include/gc/Dialect/CMakeLists.txt @@ -1,5 +1 @@ -add_subdirectory(CPURuntime) -add_subdirectory(OneDNNGraph) -add_subdirectory(Microkernel) add_subdirectory(Linalgx) -add_subdirectory(LLVMIR) diff --git a/include/gc/Dialect/CPURuntime/CMakeLists.txt b/include/gc/Dialect/CPURuntime/CMakeLists.txt deleted file mode 100644 index 9f57627c3..000000000 --- a/include/gc/Dialect/CPURuntime/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -add_subdirectory(IR) -add_subdirectory(Transforms) diff --git a/include/gc/Dialect/CPURuntime/IR/CMakeLists.txt b/include/gc/Dialect/CPURuntime/IR/CMakeLists.txt deleted file mode 100644 index fb73ae02b..000000000 --- a/include/gc/Dialect/CPURuntime/IR/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_mlir_dialect(CPURuntimeOps cpuruntime) diff --git a/include/gc/Dialect/CPURuntime/IR/CPURuntimeDialect.h b/include/gc/Dialect/CPURuntime/IR/CPURuntimeDialect.h deleted file mode 100644 index 757182964..000000000 --- a/include/gc/Dialect/CPURuntime/IR/CPURuntimeDialect.h +++ /dev/null @@ -1,18 +0,0 @@ -//===- CPURuntimeDialect.h - CPU Runtime dialect ----------------*- C++ -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef CPURUNTIME_CPURUNTIMEDIALECT_H -#define CPURUNTIME_CPURUNTIMEDIALECT_H - -#include "mlir/Bytecode/BytecodeOpInterface.h" -#include "mlir/IR/Dialect.h" -#include "mlir/Interfaces/DestinationStyleOpInterface.h" - -#include "gc/Dialect/CPURuntime/IR/CPURuntimeOpsDialect.h.inc" - -#endif // CPURUNTIME_CPURUNTIMEDIALECT_H diff --git a/include/gc/Dialect/CPURuntime/IR/CPURuntimeDialect.td b/include/gc/Dialect/CPURuntime/IR/CPURuntimeDialect.td deleted file mode 100644 index 06f3af526..000000000 --- a/include/gc/Dialect/CPURuntime/IR/CPURuntimeDialect.td +++ /dev/null @@ -1,34 +0,0 @@ -//===- CPURuntimeDialect.td - CPU Runtime Dialect ---------------*- C++ -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef CPUPARALLEL_DIALECT -#define CPUPARALLEL_DIALECT - -include "mlir/IR/OpBase.td" - -//===----------------------------------------------------------------------===// -// CPURuntime dialect definition. -//===----------------------------------------------------------------------===// - -def CPURuntime_Dialect : Dialect { - let name = "cpuruntime"; - let summary = "A dialect for CPU parallel primitives."; - let description = [{ - This dialect contains primitives for CPU runtime. - }]; - let cppNamespace = "::mlir::cpuruntime"; -} - -//===----------------------------------------------------------------------===// -// Base cpuruntime operation definition. -//===----------------------------------------------------------------------===// - -class CPURuntime_Op traits = []> : - Op; - -#endif // CPUPARALLEL_DIALECT diff --git a/include/gc/Dialect/CPURuntime/IR/CPURuntimeOps.h b/include/gc/Dialect/CPURuntime/IR/CPURuntimeOps.h deleted file mode 100644 index 5ce667a91..000000000 --- a/include/gc/Dialect/CPURuntime/IR/CPURuntimeOps.h +++ /dev/null @@ -1,26 +0,0 @@ -//===- CPURuntimeOps.h - CPU Runtime Ops ====--------------------*- C++ -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef CPURUNTIME_CPURUNTIMEOPS_H -#define CPURUNTIME_CPURUNTIMEOPS_H - -#include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/OpenMP/OpenMPDialect.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/Dialect.h" -#include "mlir/IR/OpDefinition.h" -#include "mlir/Interfaces/ControlFlowInterfaces.h" -#include "mlir/Interfaces/InferTypeOpInterface.h" -#include "mlir/Interfaces/SideEffectInterfaces.h" - -#define GET_OP_CLASSES -#include "gc/Dialect/CPURuntime/IR/CPURuntimeOps.h.inc" - -#endif // CPURUNTIME_CPURUNTIMEOPS_H diff --git a/include/gc/Dialect/CPURuntime/IR/CPURuntimeOps.td b/include/gc/Dialect/CPURuntime/IR/CPURuntimeOps.td deleted file mode 100644 index 30ca35868..000000000 --- a/include/gc/Dialect/CPURuntime/IR/CPURuntimeOps.td +++ /dev/null @@ -1,87 +0,0 @@ -//===- CPURuntimeOps.td - CPU Runtime Ops ------------------*- tablegen -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef CPURUNTIME_OPS -#define CPURUNTIME_OPS - -include "gc/Dialect/CPURuntime/IR/CPURuntimeDialect.td" -include "mlir/Interfaces/SideEffectInterfaces.td" - -def CPURuntime_PrintfOp : CPURuntime_Op<"printf", [MemoryEffects<[MemWrite]>]>, - Arguments<(ins StrAttr:$format, - Variadic>:$args)> { - let summary = "C-style printf"; - let description = [{ - `cpuruntime.printf` takes a literal format string `format` and an arbitrary number of - scalar arguments that should be printed. - - The format string is a C-style printf string, subject to any restrictions - imposed by the target platform. - }]; - let assemblyFormat = [{ - $format attr-dict ($args^ `:` type($args))? - }]; -} - -def CPURuntime_AllocOp : CPURuntime_Op<"alloc", [ - AttrSizedOperandSegments, - MemoryEffects<[MemAlloc]> - ]> { - - let summary = "CPURuntime memory allocation operation in a stack way."; - let description = [{ - The `cpuruntime.alloc` operation allocates a region of memory on the CPU. Unlike the memref.alloc operation, - which is more general-purpose, cpuruntime.alloc must adhere to a First-In-Last-Out (FILO) order, similar to a - stack's behavior. This means that memory must be freed in the reverse order of allocation. Additionally, - `thread_local` is a performance hint, indicating that the allocated memory is intended for use by a single - thread within a parallel context to avoid synchronization overhead. - - Example: - - ```mlir - // For dynamic size memref - %memref = cpuruntime.alloc (%width) : memref<64x?xf32> - - // For static size memref - %memref = cpuruntime.alloc () : memref<64x32xf32> - - // For memref with symbolic operand in affine map - %memref = cpuruntime.alloc [%symbol_value] (%width) : memref<64x?xf32, affine_map<(d0, d1)[s0] -> (d0 + s0, d1)> - - // For thread local alloc - %memref = cpuruntime.alloc thread_local () : memref<64x32xf32> - ``` - }]; - - let arguments = (ins Variadic:$dynamicSizes, Variadic:$symbolOperands, UnitAttr:$threadLocal); - let results = (outs Res]>:$memref); - - let assemblyFormat = [{ - (`thread_local` $threadLocal^)? `(` $dynamicSizes `)` (`` `[` $symbolOperands^ `]`)? attr-dict `:` type($memref) - }]; -} - -def CPURuntime_DeallocOp : CPURuntime_Op<"dealloc", [ - MemoryEffects<[MemFree]> - ]> { - - let summary = "CPU memory deallocation operation"; - - let description = [{ - The `cpuruntime.dealloc` operation frees the region of memory referenced by a - memref which was originally created by the `cpuruntime.alloc` operation. - It is similar to the `memref.dealloc` op. - }]; - - let arguments = (ins Arg:$memref, UnitAttr:$threadLocal); - - - let assemblyFormat = [{ (`thread_local` $threadLocal^)? $memref attr-dict `:` type($memref) }]; -} - -#endif // CPURUNTIME_OPS diff --git a/include/gc/Dialect/CPURuntime/Transforms/CMakeLists.txt b/include/gc/Dialect/CPURuntime/Transforms/CMakeLists.txt deleted file mode 100644 index 763ffab86..000000000 --- a/include/gc/Dialect/CPURuntime/Transforms/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -set(LLVM_TARGET_DEFINITIONS CPURuntimePasses.td) -mlir_tablegen(CPURuntimePasses.h.inc --gen-pass-decls -name CPURuntime) -mlir_tablegen(CPURuntimePasses.capi.h.inc -gen-pass-capi-header --prefix CPURuntime) -mlir_tablegen(CPURuntimePasses.capi.cpp.inc -gen-pass-capi-impl --prefix CPURuntime) -add_public_tablegen_target(MLIRCPURuntimePassesIncGen) diff --git a/include/gc/Dialect/CPURuntime/Transforms/CPURuntimePasses.h b/include/gc/Dialect/CPURuntime/Transforms/CPURuntimePasses.h deleted file mode 100644 index 8fde8f4fd..000000000 --- a/include/gc/Dialect/CPURuntime/Transforms/CPURuntimePasses.h +++ /dev/null @@ -1,29 +0,0 @@ -//===- CPURuntimePasses.h - CPU Runtime Passes ------------------*- C++ -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef CPURUNTIME_CPURUNTIMEPASSES_H -#define CPURUNTIME_CPURUNTIMEPASSES_H - -#include "gc/Dialect/CPURuntime/IR/CPURuntimeDialect.h" -#include "gc/Dialect/CPURuntime/IR/CPURuntimeOps.h" -#include "mlir/Pass/Pass.h" -#include - -namespace mlir { -namespace cpuruntime { -void registerConvertCPURuntimeToLLVMInterface(DialectRegistry ®istry); - -#define GEN_PASS_DECL -#include "gc/Dialect/CPURuntime/Transforms/CPURuntimePasses.h.inc" - -#define GEN_PASS_REGISTRATION -#include "gc/Dialect/CPURuntime/Transforms/CPURuntimePasses.h.inc" -} // namespace cpuruntime -} // namespace mlir - -#endif diff --git a/include/gc/Dialect/CPURuntime/Transforms/CPURuntimePasses.td b/include/gc/Dialect/CPURuntime/Transforms/CPURuntimePasses.td deleted file mode 100644 index 20c81e10a..000000000 --- a/include/gc/Dialect/CPURuntime/Transforms/CPURuntimePasses.td +++ /dev/null @@ -1,22 +0,0 @@ -//===- CPURuntimePasses.td - CPU Runtime Passes -----------------*- C++ -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef CPURUNTIME_PASS -#define CPURUNTIME_PASS - -include "mlir/Pass/PassBase.td" - -def CPURuntimeToLLVM: Pass<"convert-cpuruntime-to-llvm"> { - let summary = "Convert cpuruntime to LLVM dialect"; - let description = [{ - This pass converts supported cpuruntime ops to LLVM dialect instructions. - }]; - let dependentDialects = ["LLVM::LLVMDialect"]; -} - -#endif // CPURUNTIME_PASS diff --git a/include/gc/Dialect/LLVMIR/CMakeLists.txt b/include/gc/Dialect/LLVMIR/CMakeLists.txt deleted file mode 100644 index e2ef3f9fb..000000000 --- a/include/gc/Dialect/LLVMIR/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -add_mlir_dialect(XeVMOps xevm) -add_mlir_doc(XeVMOps XeVMDialect Dialects/ -gen-dialect-doc -dialect=xevm) -set(LLVM_TARGET_DEFINITIONS XeVMOps.td) -mlir_tablegen(XeVMOpsEnums.h.inc -gen-enum-decls) -mlir_tablegen(XeVMOpsEnums.cpp.inc -gen-enum-defs) -mlir_tablegen(XeVMOpsAttributes.h.inc -gen-attrdef-decls -attrdefs-dialect=xevm) -mlir_tablegen(XeVMOpsAttributes.cpp.inc -gen-attrdef-defs -attrdefs-dialect=xevm) -add_public_tablegen_target(MLIRXeVMConversionsIncGen) diff --git a/include/gc/Dialect/LLVMIR/XeVMDialect.h b/include/gc/Dialect/LLVMIR/XeVMDialect.h deleted file mode 100644 index 80f68248f..000000000 --- a/include/gc/Dialect/LLVMIR/XeVMDialect.h +++ /dev/null @@ -1,27 +0,0 @@ -//===-- XeVMDialect.h - MLIR XeVM target definitions ------------*- C++ -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef MLIR_DIALECT_LLVMIR_XEVMDIALECT_H_ -#define MLIR_DIALECT_LLVMIR_XEVMDIALECT_H_ - -#include "mlir/Bytecode/BytecodeOpInterface.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" -#include "mlir/IR/Dialect.h" -#include "mlir/IR/OpDefinition.h" - -#include "gc/Dialect/LLVMIR/XeVMOpsEnums.h.inc" - -#define GET_ATTRDEF_CLASSES -#include "gc/Dialect/LLVMIR/XeVMOpsAttributes.h.inc" - -#define GET_OP_CLASSES -#include "gc/Dialect/LLVMIR/XeVMOps.h.inc" - -#include "gc/Dialect/LLVMIR/XeVMOpsDialect.h.inc" - -#endif /* MLIR_DIALECT_LLVMIR_XEVMDIALECT_H_ */ diff --git a/include/gc/Dialect/LLVMIR/XeVMOps.td b/include/gc/Dialect/LLVMIR/XeVMOps.td deleted file mode 100644 index 4b4ee6814..000000000 --- a/include/gc/Dialect/LLVMIR/XeVMOps.td +++ /dev/null @@ -1,328 +0,0 @@ -//===-- XeVMOps.td - XeVM dialect definition ---------------*- tablegen -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -#ifndef XEVMIR_OPS -#define XEVMIR_OPS - -include "mlir/Dialect/GPU/IR/CompilationAttrInterfaces.td" -include "mlir/Dialect/LLVMIR/LLVMOpBase.td" -include "mlir/Interfaces/SideEffectInterfaces.td" - -include "mlir/IR/OpBase.td" -include "mlir/IR/EnumAttr.td" - -def XeVM_Dialect : Dialect { - let name = "xevm"; - let cppNamespace = "::mlir::xevm"; - let dependentDialects = ["LLVM::LLVMDialect"]; - - let extraClassDeclaration = [{ - /// Get the name for the attribute used to specify cache control - /// decorations. - static constexpr ::llvm::StringRef getCacheControlsAttrName() { - return ::llvm::StringLiteral("xevm.DecorationCacheControl"); - } - }]; - - let useDefaultAttributePrinterParser = 1; -} - -class XeVM_Attr traits = []> - : AttrDef { - let mnemonic = attrMnemonic; -} - -class XeVM_Op traits = []> : - Op; - -def XeVM_ElemType : AnyTypeOf<[AnyI8, AnyI16, AnyI32, F32, F16, BF16]>; - -class XeVM_LoadCacheControl : I32EnumAttr, - I32EnumAttrCase<"UC", 1, !strconcat(cacheMnemonic, "UC")>, // uncached - I32EnumAttrCase<"C", 2, !strconcat(cacheMnemonic, "C")>, // cached - I32EnumAttrCase<"S", 3, !strconcat(cacheMnemonic, "S")>, // streaming - I32EnumAttrCase<"IAR", 4, !strconcat(cacheMnemonic, "IAR")>, // invalidate-after-read - ]> { - let cppNamespace = "::mlir::xevm"; -} - -def XeVM_L1LoadCacheControl : XeVM_LoadCacheControl<"L1">; -def XeVM_L3LoadCacheControl : XeVM_LoadCacheControl<"L3">; - -class XeVM_StoreCacheControl : I32EnumAttr, - I32EnumAttrCase<"UC", 1, !strconcat(cacheMnemonic, "UC")>, // uncached - I32EnumAttrCase<"WT", 2, !strconcat(cacheMnemonic, "WT")>, // write-through - I32EnumAttrCase<"S", 3, !strconcat(cacheMnemonic, "S")>, // streaming - I32EnumAttrCase<"WB", 4, !strconcat(cacheMnemonic, "WB")>, // write back - ]> { - let cppNamespace = "::mlir::xevm"; -} - -def XeVM_L1StoreCacheControl : XeVM_StoreCacheControl<"L1">; -def XeVM_L3StoreCacheControl : XeVM_StoreCacheControl<"L3">; - -def XeVM_BlockLoad2dOp : XeVM_Op<"blockload2d">, - Results<(outs FixedVectorOf<[XeVM_ElemType]>:$res)>, - Arguments<(ins - Arg:$ptr, - I32:$base_width, - I32:$base_height, - I32:$base_pitch, - I32:$x, - I32:$y, - I32Attr:$elem_size_in_bits, - I32Attr:$tile_width, - I32Attr:$tile_height, - I32Attr:$v_blocks, - I1Attr:$transpose, - I1Attr:$vnni_transform, - DefaultValuedAttr:$l1_cache_control, - DefaultValuedAttr:$l3_cache_control - )> { - - let summary = "2D block load"; - - let description = [{ - The `xevm.blockload2d` operation loads a two dimensional matrix tile - from a larger matrix residing in memory. The parameters are: - $ptr - the base address of the matrix containing the tile to load - $base_width, $base_height, $base_pitch - the shape of matrix - $x, $y, $tile_width, $tile_height - the starting offsets and shape of the tile to load - $elem_size_in_bits - the size in bits of the matrix element - - 32 for f32, bf32 - - 16 for f16, int16, bf16 - - 8 for int8, int4, int2 - $v_blocks - number of tiles to load - $transpose - transpose the tile in registers (useful for 32 bit element type) - $vnni_transform - transpose and pack the submatrix in registers (useful for < 32 bit element types) - $cache_control - an enumerator that sets the L1 and L3 cache behaviour - - Notes: - - pitch is the physical stride between the first columns of the current row and the subsequent row, - this may include (possibly implicit) padding, alignment, or other factors. - - the $transpose and $vnni_transform parameters are mutual exclusive - - transposing the tile loaded is typically used for the B matrix operand - (D = C + A * B), where A has row-major layout in registers and B should have column-major layout. - - if the tile loaded contains out of bound elements of the matrix, they are filled with 0. - - coordinate is provided in elements, while width and pitch are provided in bytes. - }]; - - let assemblyFormat = [{ - operands ` ` `{` `elem_size_in_bits` `=` $elem_size_in_bits `,` `tile_width` `=` $tile_width `,` - `tile_height` `=` $tile_height `,` `v_blocks` `=` $v_blocks `,` `transpose` `=` $transpose `,` - `vnni_transform` `=` $vnni_transform `,` `l1_cache_control` `=` $l1_cache_control `,` - `l3_cache_control` `=` $l3_cache_control `}` attr-dict `:` functional-type(operands, results) - }]; - - let hasVerifier = 1; -} - -def XeVM_BlockStore2dOp : XeVM_Op<"blockstore2d">, - Arguments<(ins - Arg:$ptr, - I32:$base_width, - I32:$base_height, - I32:$base_pitch, - I32:$x, - I32:$y, - I32Attr:$elem_size_in_bits, - I32Attr:$tile_width, - I32Attr:$tile_height, - I32Attr:$v_blocks, - FixedVectorOf<[XeVM_ElemType]>:$stored_val, - DefaultValuedAttr:$l1_cache_control, - DefaultValuedAttr:$l3_cache_control - )> { - - let summary = "2D block store"; - - let description = [{ - The `xevm.blockstore2d` operation stores a two dimensional tile into a - larger matrix residing in memory. The parameters are: - $ptr - the base address of the matrix where to store the tile - $base_width, $base_height, $base_pitch - the shape of the matrix - $x, $y, $tile_width, $tile_height - the starting offsets and shape of the tile to store - $elem_size_in_bits - the size in bits of the matrix element - - 32 for f32, bf32 - - 16 for f16, int16, bf16 - - 8 for int8, int4, int2 - $v_blocks - number of tiles to store - $cache_control - an enumerator that sets the L1 and L3 cache behaviour - $stored_val - the tile to store - - Notes: - - pitch is the physical stride between the first columns of the current row and the subsequent row, - this may include (possibly implicit) padding, alignment, or other factors. - - coordinate is provided in elements, while width and pitch are provided in bytes. - }]; - - let assemblyFormat = [{ - operands ` ` `{` `elem_size_in_bits` `=` $elem_size_in_bits `,` `tile_width` `=` $tile_width `,` - `tile_height` `=` $tile_height `,` `v_blocks` `=` $v_blocks `,` `l1_cache_control` `=` $l1_cache_control `,` - `l3_cache_control` `=` $l3_cache_control `}` - attr-dict `:` `(` type(operands) `)` - }]; - - let hasVerifier = 1; -} - -def XeVM_BlockPrefetch2dOp : XeVM_Op<"blockprefetch2d">, - Arguments<(ins - Arg:$ptr, - I32:$base_width, - I32:$base_height, - I32:$base_pitch, - I32:$x, - I32:$y, - I32Attr:$elem_size_in_bits, - I32Attr:$tile_width, - I32Attr:$tile_height, - I32Attr:$v_blocks, - DefaultValuedAttr:$l1_cache_control, - DefaultValuedAttr:$l3_cache_control - )> { - - let summary = "2D block prefetch"; - - let description = [{ - The `xevm.blockprefetch2d` operation prefetches a two dimensional tile - from a larger matrix residing in memory. The parameters are: - $ptr - the base address of the matrix containing the tile to prefetch - $base_width, $base_height, $base_pitch - the shape of the matrix - $x, $y, $tile_width, $tile_height - the starting offsets and shape of tile to prefetch - $elem_size_in_bits - the size in bits of the matrix element - - 32 for f32, bf32 - - 16 for f16, int16, bf16 - - 8 for int8, int4, int2 - $v_blocks - number of tiles to prefetch - $cache_control - an enumerator that sets the L1 and L3 cache behaviour - - Notes: - - pitch is the physical stride between the first columns of the current row and the subsequent row, - this may include (possibly implicit) padding, alignment, or other factors. - - coordinate is provided in elements, while width and pitch are provided in bytes. - }]; - - let assemblyFormat = [{ - operands ` ` `{` `elem_size_in_bits` `=` $elem_size_in_bits `,` `tile_width` `=` $tile_width `,` - `tile_height` `=` $tile_height `,` `v_blocks` `=` $v_blocks `,` `l1_cache_control` `=` $l1_cache_control `,` - `l3_cache_control` `=` $l3_cache_control `}` - attr-dict `:` `(` type(operands) `)` - }]; - - let hasVerifier = 1; -} - -def XeVM_MatrixElemType : AnyTypeOf<[AnyI8, AnyI16, AnyI32, F32, F16, BF16]>; - -/// Enum attribute of the different precision types. -def XeVM_PrecisionTypeAttr : I32EnumAttr<"PrecisionType", - "XeVM precision type", - [ - I32EnumAttrCase<"UNUSED", 0, "unused">, - I32EnumAttrCase<"U8", 1, "u8">, - I32EnumAttrCase<"U4", 2, "u4">, - I32EnumAttrCase<"U2", 3, "u2">, - I32EnumAttrCase<"S8", 4, "i8">, - I32EnumAttrCase<"S4", 5, "i4">, - I32EnumAttrCase<"S2", 6, "i2">, - I32EnumAttrCase<"BF8", 7, "bf8">, - I32EnumAttrCase<"TF32", 8, "tf32">, - I32EnumAttrCase<"BF16", 9, "bf16">, - I32EnumAttrCase<"FP16", 10, "f16"> - ]> { - let cppNamespace = "::mlir::xevm"; -} - -def XeVM_DPASOp : XeVM_Op<"dpas">, - Results<(outs FixedVectorOf<[XeVM_MatrixElemType]>:$d)>, - Arguments<(ins - FixedVectorOfRankAndType<[1], [XeVM_MatrixElemType]>:$c, - FixedVectorOfRankAndType<[1], [XeVM_MatrixElemType]>:$a, - FixedVectorOfRankAndType<[1], [XeVM_MatrixElemType]>:$b, - XeVM_PrecisionTypeAttr:$pa, - XeVM_PrecisionTypeAttr:$pb, - I32Attr:$rc - )> { - - let summary = "Matrix multiply-add"; - - let description = [{ - The `xevm.dpas` operation is a matrix multiplication plus accumulation: - - D = C + A x B - - where the A, B, C input matrices and the result D have shapes: - D : MxN - C : MxN - A : MxK - B : KxN - - Shape restrictions: - M : must be 1, 2, 4, or 8 - N : fixed execution size, must be 16 - K : systolic_depth * OPS_PER_CHAN - OPS_PER_CHAN - 1 : for TF32 - 2 : for 16-bit precision(BF, HF) - 4 : for 8-bit precision (FP8, UB, B) - 8 : for less-then 8 bit precision (U4/S4, U2/S2). - - If systolic_depth is 8, K would be 8, 16, 32, or 64 (based on OPS_PER_CHAN). - $a, $b, $c, $d - matrix A, B, C, D, respectively - $pa, $pb - precision of matrix A and B resepectively - $rc - repeat count - - Further restrictions as well as more details can be found here: - https://registry.khronos.org/OpenCL/extensions/intel/cl_intel_subgroup_matrix_multiply_accumulate.html - }]; - - let assemblyFormat = [{ - operands ` ` `{` `pa` `=` $pa `,` `pb` `=` $pb `,` `rc` `=` $rc `}` attr-dict `:` functional-type(operands, results) - }]; - - // let hasVerifier = 1; -} - -def XeVM_TargetAttr : XeVM_Attr<"XeVMTarget", "target"> { - let description = [{ - GPU target attribute for controlling compilation of targets. All - parameters decay into default values if not present. - - Examples: - - 1. Target with default values. - ``` - gpu.module @mymodule [#xevm.target] attributes {...} { - ... - } - ``` - }]; - let parameters = (ins - DefaultValuedParameter<"int", "2", "Optimization level to apply.">:$O, - StringRefParameter<"Target triple.", "\"spirv64-unknown-unknown\"">:$triple, - StringRefParameter<"Target chip.", "\"pvc\"">:$chip - ); - let assemblyFormat = [{ - (`<` struct($O, $triple, $chip)^ `>`)? - }]; - let builders = [ - AttrBuilder<(ins CArg<"int", "2">:$optLevel, - CArg<"StringRef", "\"spirv64-unknown-unknown\"">:$triple, - CArg<"StringRef", "\"pvc\"">:$chip), [{ - return Base::get($_ctxt, optLevel, triple, chip); - }]> - ]; - let skipDefaultBuilders = 1; - let genVerifyDecl = 1; -} - -#endif // XEVMIR_OPS diff --git a/include/gc/Dialect/Linalgx/CMakeLists.txt b/include/gc/Dialect/Linalgx/CMakeLists.txt index 1aceb8345..eda9369ad 100644 --- a/include/gc/Dialect/Linalgx/CMakeLists.txt +++ b/include/gc/Dialect/Linalgx/CMakeLists.txt @@ -1,9 +1,4 @@ add_mlir_dialect(LinalgxOps linalgx) -set(LLVM_TARGET_DEFINITIONS LinalgxStructuredOps.td) -mlir_tablegen(LinalgxStructuredOps.h.inc -gen-op-decls) -mlir_tablegen(LinalgxStructuredOps.cpp.inc -gen-op-defs) -add_public_tablegen_target(MLIRLinalgxStructuredOpsIncGen) add_mlir_doc(LinalgxOps LinalgxOps gc/Dialect/Linalgx/ -gen-op-doc) add_mlir_doc(LinalgxDialect LinalgxDialect gc/Dialect/Linalgx/ -gen-dialect-doc) -add_mlir_doc(LinalgxStructuredOps LinalgxStructuredOps gc/Dialect/Linalgx/ -gen-dialect-doc) diff --git a/include/gc/Dialect/Linalgx/LinalgxDialect.td b/include/gc/Dialect/Linalgx/LinalgxDialect.td index 1bb9521de..23cc589b5 100644 --- a/include/gc/Dialect/Linalgx/LinalgxDialect.td +++ b/include/gc/Dialect/Linalgx/LinalgxDialect.td @@ -20,7 +20,7 @@ def LinalgxDialect : Dialect { let summary = "A dialect for linalg extensions."; let description = [{ The dialect primarily aims to fill the gap for matmul ops - missing tiling semantics when consuming packed tensors. + missing tiling semantics when consuming packed tensors. }]; let cppNamespace = "::mlir::linalgx"; @@ -30,21 +30,8 @@ def LinalgxDialect : Dialect { "math::MathDialect", "memref::MemRefDialect", "tensor::TensorDialect", + "linalg::LinalgDialect" ]; - - let extraClassDeclaration = [{ - /// Attribute name used to memoize indexing maps for named ops. - constexpr const static ::llvm::StringLiteral - kMemoizedIndexingMapsAttrName = "linalg.memoized_indexing_maps"; - - using RegionBuilderFunType = llvm::function_ref< - void(ImplicitLocOpBuilder &b, Block &, ArrayRef)>; - RegionBuilderFunType getRegionBuilder(StringRef name) { - return namedStructuredOpRegionBuilders.lookup(name); - } - private: - llvm::StringMap namedStructuredOpRegionBuilders; - }]; } #endif // LINALGX_DIALECT diff --git a/include/gc/Dialect/Linalgx/LinalgxOps.h b/include/gc/Dialect/Linalgx/LinalgxOps.h index 9ea73a91e..58fa3da18 100644 --- a/include/gc/Dialect/Linalgx/LinalgxOps.h +++ b/include/gc/Dialect/Linalgx/LinalgxOps.h @@ -22,7 +22,4 @@ #define GET_OP_CLASSES #include "gc/Dialect/Linalgx/LinalgxOps.h.inc" -#define GET_OP_CLASSES -#include "gc/Dialect/Linalgx/LinalgxStructuredOps.h.inc" - #endif // GC_DIALECTS_LINALGXOPS_H diff --git a/include/gc/Dialect/Linalgx/LinalgxOps.td b/include/gc/Dialect/Linalgx/LinalgxOps.td index 4491967c3..44869e041 100644 --- a/include/gc/Dialect/Linalgx/LinalgxOps.td +++ b/include/gc/Dialect/Linalgx/LinalgxOps.td @@ -11,8 +11,193 @@ include "LinalgxDialect.td" +include "mlir/Dialect/Linalg/IR/LinalgInterfaces.td" +include "mlir/Interfaces/ControlFlowInterfaces.td" +include "mlir/Interfaces/DestinationStyleOpInterface.td" +include "mlir/Interfaces/InferTypeOpInterface.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "mlir/Interfaces/TilingInterface.td" +include "mlir/Interfaces/ViewLikeInterface.td" + // Base class for Linalg dialect ops that do not correspond to library calls. class Linalgx_Op traits = []> : Op; -#endif // LINALGX_OPS \ No newline at end of file +def Linalgx_AttentionOp : Linalgx_Op<"attention", + [ + DeclareOpInterfaceMethods, + DeclareOpInterfaceMethods, + DeclareOpInterfaceMethods, + DeclareOpInterfaceMethods]> { + let summary = [{Attention operator.}]; + let description = [{ + Computes the scaled dot product attention function: + + attention(Q, K, V, scale) = softmax(Q @ K.T * scale) @ V + + Here Q, K, V are given tensors and scale is a scalar value specifying + the scale to use. + + If an additional mask argument M is included, the result of the first matmul is modified according to: + + Q @ K.T += M + }]; + + let arguments = (ins AnyShaped:$query, + AnyShaped:$key, + AnyShaped:$value, + AnyFloat:$scale, + Optional:$mask, + AnyShaped:$output, + AffineMapArrayAttr:$indexing_maps, + OptionalAttr:$decomposition_config + ); + + let results = (outs Variadic:$results); + let hasVerifier = 1; + let hasCustomAssemblyFormat = 1; + let assemblyFormat = [{ + attr-dict + prop-dict + `ins` `(` $query `,` $key `,` $value `,` $scale (`,` $mask^)? `:` type($query) `,` type($key) `,` type($value) `,` type($scale) (`,` type($mask)^ )?`)` + `outs` `(` $output `:` type($output) `)` + (`->` type($results)^)? + }]; + + let builders = [ + OpBuilder<(ins "TypeRange":$results, + "Value":$query, + "Value":$key, + "Value":$value, + "Value":$scale, + "Value":$output, + "ArrayAttr":$indexing_maps, + CArg<"std::optional", "std::nullopt">:$mask)>, + OpBuilder<(ins "TypeRange":$results, + "ValueRange":$inputOperands, + "ValueRange":$initOperands, + "ArrayAttr":$indexing_maps)> + ]; + + let extraClassDeclaration = [{ + SmallVector getIndexingMapsArray(); + + ArrayRef getShape(OpOperand* opOperand); + + AffineMap getQueryMap() { + return cast(getIndexingMapsArray()[0]); + } + AffineMap getKeyMap() { + return cast(getIndexingMapsArray()[1]); + } + AffineMap getValueMap() { + return cast(getIndexingMapsArray()[2]); + } + AffineMap getScaleMap() { + return cast(getIndexingMapsArray()[3]); + } + std::optional getMaskMap() { + if (getMask()) { + return cast(getIndexingMapsArray()[4]); + } + return std::nullopt; + } + AffineMap getOutputMap() { + return cast(getIndexingMapsArray()[getNumDpsInputs()]); + } + int64_t getIterationDomainRank() { + return getQueryMap().getNumDims(); + } + + /* Decomposition control attributes */ + + // Attributes to set on QK and PV matmul after decomposition. + static StringRef getQKAttrStr() { return "qk_attrs"; } + static StringRef getPVAttrStr() { return "pv_attrs"; } + }]; + + let hasCanonicalizer = 1; +} + +def Linalgx_GroupedMatmulOp : Linalgx_Op<"grouped_matmul", + [ + DeclareOpInterfaceMethods, + DeclareOpInterfaceMethods, + DeclareOpInterfaceMethods]> { + let summary = [{Grouped matmul with the semantics of PyTorch's grouped_mm.}]; + let description = [{ + Same as `torch._grouped_mm(lhs, weights.transpose(-2, -1), offs=offsets)`: + the rows of `lhs` are split into consecutive groups, one per expert, and + each group is multiplied by its own expert's weights. + + lhs [total, K] + weights [E, N, K] (K innermost, i.e. transposed) + offsets [E] end row of each group (i32/i64) + output [total, N] + + Example: 3 experts with 2, 0 and 3 rows, so `total = 5` and + `offsets = [2, 2, 5]`: + + rows [0, 2) -> out = lhs[0:2] * weights[0]^T + rows [2, 2) -> expert 1 has no rows + rows [2, 5) -> out = lhs[2:5] * weights[2]^T + + `max_group_size` is an optional performance hint: an upper bound on the + number of rows in any group. It defaults to `total` and only shrinks the + loop over rows, but it must be a real bound - rows of a group longer than + `max_group_size` are not computed. + }]; + + let arguments = (ins AnyShaped:$lhs, + AnyShaped:$weights, + AnyShaped:$offsets, + Optional:$max_group_size, + AnyShaped:$output, + AffineMapArrayAttr:$indexing_maps + ); + + let results = (outs Variadic:$results); + let hasVerifier = 1; + let assemblyFormat = [{ + attr-dict + prop-dict + `ins` `(` $lhs `,` $weights `,` $offsets `:` + type($lhs) `,` type($weights) `,` type($offsets) `)` + (`max_group_size` `(` $max_group_size^ `)`)? + `outs` `(` $output `:` type($output) `)` + (`->` type($results)^)? + }]; + + let extraClassDeclaration = [{ + SmallVector getIndexingMapsArray(); + + ArrayRef getShape(OpOperand* opOperand); + + /* `indexing_maps` holds one map per *shaped* operand, in operand order; + `max_group_size` is a scalar and has no map. */ + AffineMap getLhsMap() { return getIndexingMapsArray()[0]; } + AffineMap getWeightsMap() { return getIndexingMapsArray()[1]; } + AffineMap getOffsetsMap() { return getIndexingMapsArray()[2]; } + AffineMap getOutputMap() { return getIndexingMapsArray()[3]; } + + int64_t getIterationDomainRank() { return getLhsMap().getNumDims(); } + + /* Iteration domain dim positions. */ + static constexpr int64_t kExpertDim = 0; + static constexpr int64_t kMDim = 1; + static constexpr int64_t kNDim = 2; + static constexpr int64_t kKDim = 3; + }]; +} + +#endif // LINALGX_OPS diff --git a/include/gc/Dialect/Linalgx/LinalgxStructuredOps.td b/include/gc/Dialect/Linalgx/LinalgxStructuredOps.td deleted file mode 100644 index 5e24a05b6..000000000 --- a/include/gc/Dialect/Linalgx/LinalgxStructuredOps.td +++ /dev/null @@ -1,107 +0,0 @@ -//===- LinalgxStructuredOps.td - linalgx dialect ops -------*- tablegen -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef LINALGX_STRUCTURED_OPS -#define LINALGX_STRUCTURED_OPS - -include "LinalgxDialect.td" - -include "mlir/Dialect/Utils/StructuredOpsUtils.td" -include "mlir/Dialect/Linalg/IR/LinalgEnums.td" -include "mlir/IR/EnumAttr.td" -include "mlir/IR/OpBase.td" - -include "mlir/Dialect/Linalg/IR/LinalgBase.td" -include "mlir/Dialect/Linalg/IR/LinalgInterfaces.td" -include "mlir/Interfaces/DestinationStyleOpInterface.td" -include "mlir/Interfaces/InferTypeOpInterface.td" -include "mlir/Interfaces/SideEffectInterfaces.td" -include "mlir/IR/OpAsmInterface.td" - -// Base Tablegen class for Linalg ops. -// Linalg ops that correspond to library calls operate on ShapedType as their -// first operands. These may be optionally followed by non-view operands -// depending on the specific Linalg op. -class LinalgxStructuredBase_Op props> - : Op, - DeclareOpInterfaceMethods, - DestinationStyleOpInterface, - LinalgStructuredInterface, - ReifyRankedShapedTypeOpInterface], props)> { - code structuredOpsBaseDecls = [{ - // Return whether the op accesses the iteration indices. - bool hasIndexSemantics() { - return !this->getBody()->getOps().empty(); - } - - LogicalResult reifyResultShapes(OpBuilder &b, - ReifiedRankedShapedTypeDims &reifiedReturnShapes) { - return llvm::cast(getOperation()).reifyResultShapes(b, - reifiedReturnShapes); - } - }]; -} - -def TensorOrMemref : - AnyTypeOf<[AnyMemRef, AnyRankedTensor], "", "::mlir::ShapedType">; - -//===----------------------------------------------------------------------===// -// Structured linalgx operation definition -//===----------------------------------------------------------------------===// - -def Linalgx_SigmoidOp : LinalgxStructuredBase_Op<"sigmoid", - [AttrSizedOperandSegments]> { - let summary = [{Applies sigmoid(x) elementwise.}]; - let description = [{No numeric casting is performed on the input operand.}]; - let arguments = (ins - Variadic:$inputs, - Variadic:$outputs); - let results = (outs Variadic:$results); - let regions = (region AnyRegion:$region); - - let skipDefaultBuilders = 1; - let builders = [ - OpBuilder< - (ins - "TypeRange":$resultTensorTypes, - "ValueRange":$inputs, - "ValueRange":$outputs, - CArg<"ArrayRef", "{}">:$attributes), - [{ - buildStructuredOp($_builder, $_state, resultTensorTypes, - inputs, outputs, attributes, SigmoidOp::getRegionBuilder()); - }]> - ]; - - let hasCustomAssemblyFormat = 1; - let hasFolder = 1; - - let extraClassDeclaration = structuredOpsBaseDecls # [{ - // Declare functions necessary for LinalgStructuredInterface. - SmallVector getIteratorTypesArray(); - ArrayAttr getIndexingMaps(); - static unsigned getNumRegionArgs() { return 2; } - std::string getLibraryCallName() { - return "op_has_no_registered_library_name"; - } - - // Implement functions necessary for DestinationStyleOpInterface. - MutableOperandRange getDpsInitsMutable() { return getOutputsMutable(); } - - static void regionBuilder(ImplicitLocOpBuilder &b, - Block &block, ArrayRef attrs); - static std::function)> - getRegionBuilder() { - return regionBuilder; - } - }]; -} - -#endif // LINALGX_STRUCTURED_OPS \ No newline at end of file diff --git a/include/gc/Dialect/Linalgx/Utils.h b/include/gc/Dialect/Linalgx/Utils.h deleted file mode 100644 index 5bc83b449..000000000 --- a/include/gc/Dialect/Linalgx/Utils.h +++ /dev/null @@ -1,78 +0,0 @@ -//===- Utils.h - linalgx utils ----------------------------------*- C++ -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef GC_DIALECTS_LINALGX_UTILS_H -#define GC_DIALECTS_LINALGX_UTILS_H - -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/TypeUtilities.h" -#include "mlir/IR/Types.h" - -namespace mlir { -namespace linalgx { - -/// @brief enum of type of matmul packing -enum class PackingType : int { - MM4D = 0, // MKmk x NKkn - VNNI_MM2D, // MK x NKknV - VNNI_MM4D, // MKmk x NKknV - VNNI_BRMM3D, // BMK x BKNV - NUM_TYPES, -}; - -/// @brief make a generic packed matmul Op based on PackingType -/// @param builder builder -/// @param loc location -/// @param opType the PackingType -/// @param inputs matmul A, B -/// @param outputs matmul C -/// @return the generic packed matmul Op -FailureOr -makeGenericPackedMatmulOp(OpBuilder &builder, Location loc, PackingType opType, - ValueRange inputs, ValueRange outputs); - -/// @brief identify a generic packed matmul Op based on PackingType -/// @param op the op -/// @param opType the PackingType -/// @return true if op is a generic packed matmul Op -bool isGenericPackedMatmulOp(Operation *op, PackingType opType); - -template -inline bool isGenericPackedMatmulOp(Operation *op, PackingType first, - Args... args) { - return isGenericPackedMatmulOp(op, first) || - isGenericPackedMatmulOp(op, args...); -} - -/// @brief identify a generic packed matmul Op based on any PackingType -/// @param op the op -/// @return true if op is a generic packed matmul Op -template inline bool isAnyGenericPackedMatmulOp(Operation *op) { - return isGenericPackedMatmulOp(op, (PackingType)N) || - isAnyGenericPackedMatmulOp(op); -} -constexpr int NUM_ALL_TYPES = (int)PackingType::NUM_TYPES; -template <> -inline bool -isAnyGenericPackedMatmulOp(Operation *op) { - return false; -} -inline bool isAnyGenericPackedMatmulOp(Operation *op) { - return isAnyGenericPackedMatmulOp<0, NUM_ALL_TYPES>(op); -} - -/// @brief identify a matmul Op based on ContractionOp and PackingType -/// @param op the op -/// @return true if op is a matmul Op -bool isMatmulOp(Operation *op); - -} // namespace linalgx -} // namespace mlir - -#endif // GC_DIALECTS_LINALGX_UTILS_H diff --git a/include/gc/Dialect/Microkernel/CMakeLists.txt b/include/gc/Dialect/Microkernel/CMakeLists.txt deleted file mode 100644 index d9046919c..000000000 --- a/include/gc/Dialect/Microkernel/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -set(LLVM_TARGET_DEFINITIONS MicrokernelEnum.td) -mlir_tablegen(MicrokernelEnum.h.inc -gen-enum-decls) -mlir_tablegen(MicrokernelEnum.cpp.inc -gen-enum-defs) -add_public_tablegen_target(MLIRMicrokernelAttrDefIncGen) - -add_mlir_dialect(MicrokernelOps microkernel) -add_mlir_doc(MicrokernelOps MicrokernelOps gc/Dialect/Microkernel/ -gen-op-doc) -add_mlir_doc(MicrokernelDialect MicrokernelDialect gc/Dialect/Microkernel/ -gen-dialect-doc) diff --git a/include/gc/Dialect/Microkernel/MicrokernelDialect.h b/include/gc/Dialect/Microkernel/MicrokernelDialect.h deleted file mode 100644 index 35390ab6f..000000000 --- a/include/gc/Dialect/Microkernel/MicrokernelDialect.h +++ /dev/null @@ -1,17 +0,0 @@ -//===- MicrokernelDialect.h - microkernel dialect ---------------*- C++ -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef GC_DIALECTS_MICROKERNELDIALECT_H -#define GC_DIALECTS_MICROKERNELDIALECT_H - -#include "mlir/IR/Dialect.h" -#include "mlir/Interfaces/DestinationStyleOpInterface.h" - -#include "gc/Dialect/Microkernel/MicrokernelOpsDialect.h.inc" - -#endif // GC_DIALECTS_MICROKERNELDIALECT_H diff --git a/include/gc/Dialect/Microkernel/MicrokernelDialect.td b/include/gc/Dialect/Microkernel/MicrokernelDialect.td deleted file mode 100644 index ef8e544df..000000000 --- a/include/gc/Dialect/Microkernel/MicrokernelDialect.td +++ /dev/null @@ -1,34 +0,0 @@ -//===- MicrokernelDialect.td - microkernel dialect ---------*- tablegen -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef MICROKERNEL_DIALECT -#define MICROKERNEL_DIALECT - -include "mlir/IR/OpBase.td" - -//===----------------------------------------------------------------------===// -// Microkernel dialect definition. -//===----------------------------------------------------------------------===// - -def Microkernel_Dialect : Dialect { - let name = "microkernel"; - let summary = "A dialect for microkernel abstraction."; - let description = [{ - This dialect contains wrappers for microkernel primitives like BRGEMM. - }]; - let cppNamespace = "::mlir::microkernel"; -} - -//===----------------------------------------------------------------------===// -// Base microkernel operation definition. -//===----------------------------------------------------------------------===// - -class Microkernel_Op traits = []> : - Op; - -#endif // MICROKERNEL_DIALECT diff --git a/include/gc/Dialect/Microkernel/MicrokernelEnum.h b/include/gc/Dialect/Microkernel/MicrokernelEnum.h deleted file mode 100644 index ba6505b67..000000000 --- a/include/gc/Dialect/Microkernel/MicrokernelEnum.h +++ /dev/null @@ -1,18 +0,0 @@ -//===- MicrokernelEnum.h - microkernel dialect enums ------------*- C++ -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef GC_DIALECTS_MICROKERNELENUM_H -#define GC_DIALECTS_MICROKERNELENUM_H - -#include "mlir/IR/Attributes.h" -#include "mlir/IR/DialectImplementation.h" - -#define GET_ATTRDEF_CLASSES -#include "gc/Dialect/Microkernel/MicrokernelEnum.h.inc" - -#endif // GC_DIALECTS_MICROKERNELENUM_H diff --git a/include/gc/Dialect/Microkernel/MicrokernelEnum.td b/include/gc/Dialect/Microkernel/MicrokernelEnum.td deleted file mode 100644 index fc51dfce9..000000000 --- a/include/gc/Dialect/Microkernel/MicrokernelEnum.td +++ /dev/null @@ -1,26 +0,0 @@ -//===- MicrokernelEnum.td - microkernel dialect enum -------*- tablegen -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef MICROKERNEL_ENUM -#define MICROKERNEL_ENUM - -include "mlir/IR/EnumAttr.td" -include "gc/Dialect/Microkernel/MicrokernelDialect.td" - -def Microkernel_BrgemmFlags : I64EnumAttr< - "BrgemmFlags", "Flags for indicating optional behaviours of Brgemm", - [ - I64EnumAttrCase<"NONE", 0, "none">, - I64EnumAttrCase<"BETA_0", 1, "beta_0">, - I64EnumAttrCase<"STRIDE", 2, "stride">, - I64EnumAttrCase<"LIST", 4, "list"> - ]> { - let cppNamespace = "::mlir::microkernel"; -} - -#endif // MICROKERNEL_ENUM diff --git a/include/gc/Dialect/Microkernel/MicrokernelOps.h b/include/gc/Dialect/Microkernel/MicrokernelOps.h deleted file mode 100644 index 4ff6dd358..000000000 --- a/include/gc/Dialect/Microkernel/MicrokernelOps.h +++ /dev/null @@ -1,28 +0,0 @@ -//===- MicrokernelOps.h - microkernel dialect ops ---------------*- C++ -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef GC_DIALECTS_MICROKERNELOPS_H -#define GC_DIALECTS_MICROKERNELOPS_H - -#include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/OpenMP/OpenMPDialect.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/Dialect.h" -#include "mlir/IR/OpDefinition.h" -#include "mlir/Interfaces/DestinationStyleOpInterface.h" -#include "mlir/Interfaces/SideEffectInterfaces.h" - -#include "gc/Dialect/Microkernel/MicrokernelDialect.h" -#include "gc/Dialect/Microkernel/MicrokernelEnum.h" - -#define GET_OP_CLASSES -#include "gc/Dialect/Microkernel/MicrokernelOps.h.inc" - -#endif // GC_DIALECTS_MICROKERNELOPS_H diff --git a/include/gc/Dialect/Microkernel/MicrokernelOps.td b/include/gc/Dialect/Microkernel/MicrokernelOps.td deleted file mode 100644 index fdd4b7347..000000000 --- a/include/gc/Dialect/Microkernel/MicrokernelOps.td +++ /dev/null @@ -1,181 +0,0 @@ -//===- MicrokernelOps.td - microkernel dialect ops ---------*- tablegen -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef MICROKERNEL_OPS -#define MICROKERNEL_OPS - -include "MicrokernelDialect.td" -include "gc/Dialect/Microkernel/MicrokernelEnum.td" -include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.td" -include "mlir/Interfaces/DestinationStyleOpInterface.td" -include "mlir/Interfaces/SideEffectInterfaces.td" - -class StaticTensorRankOf allowedTypes, list ranks> : - Type.predicate, - HasAnyRankOfPred, HasStaticShapePred]>, - !interleave(!foreach(rank, ranks, rank # "D"), "/") # " static " # - TensorOf.summary, "::mlir::TensorType">; - -class StaticMemRefRankOf allowedTypes, list ranks> : - Type.predicate, - HasAnyRankOfPred, HasStaticShapePred]>, - !interleave(!foreach(rank, ranks, rank # "D"), "/") # " static " # - MemRefOf.summary, "::mlir::MemRefType">; - -def BrgemmTensor : StaticTensorRankOf<[F32, BF16, SI32, SI8, UI8], [2, 3, 4]>; - -def BrgemmTensorOrMemRef : AnyTypeOf<[StaticTensorRankOf<[F32, BF16, SI32, SI8, UI8], [2, 3, 4]>, - StaticMemRefRankOf<[F32, BF16, SI32, SI8, UI8], [2, 3, 4]>]>; - -def Microkernel_BrgemmOp : Microkernel_Op<"brgemm", - [DeclareOpInterfaceMethods, - BufferizableOpInterface, - DestinationStyleOpInterface]> { - let summary = "Abstract Op that execute brgemm kernel on tensors."; - let description = [{ - The operation has the following arguments: - 1) Tensors or MemRefs of operand A/B; - 2) The batch dims and leading dims of operand A/B; - And has the following outputs: - 1) Tensor of operand C; - }]; - - let arguments = (ins Variadic:$inputs, - BrgemmTensorOrMemRef:$init, - ConfinedAttr]>:$batchDims, - ConfinedAttr]>:$leadingDims, - TypedArrayAttrBase:$flags); - let results = (outs Variadic:$output); - - let extraClassDeclaration = [{ - Value getOperandA() { return getInputs()[0]; } - Value getOperandB() { return getInputs()[1]; } - Value getOperandC() { return getInit(); } - - int64_t getBatchDimA() { return getBatchDims()[0]; } - int64_t getLeadingDimA() { return getLeadingDims()[0]; } - - int64_t getBatchDimB() { return getBatchDims()[1]; } - int64_t getLeadingDimB() { return getLeadingDims()[1]; } - - MutableOperandRange getDpsInitsMutable() { return getInitMutable(); } - - bool bufferizesToMemoryRead(OpOperand &, - const bufferization::AnalysisState &); - bool bufferizesToMemoryWrite(OpOperand &, - const bufferization::AnalysisState &); - bool bufferizesToElementwiseAccess(const bufferization::AnalysisState &, - ArrayRef); - bufferization::AliasingValueList getAliasingValues(OpOperand &opOperand, - const bufferization::AnalysisState &state); - LogicalResult bufferize(RewriterBase &, - const bufferization::BufferizationOptions &); - }]; - - let hasVerifier = 1; - let hasCustomAssemblyFormat = 1; - let hasFolder = 1; -} - -def Microkernel_BrgemmDispatchOp : Microkernel_Op<"brgemm.dispatch", [Pure]> { - let summary = "JIT the brgemm microkernel given the parameters"; - let description = [{ - The operation has the following arguments: 1) m, n, k, lda, ldb, ldc, stride_a and stride_b. - Inputs is a dense attribute of I64 elements. 2) flags carry information on - the different flags that can be used for brgemm like whether beta == 0 or strided batch. For - more details, see: `Microkernel_BrgemmFlags`. 3) data_types of operand A & B. - Outpus is the id of JITed kernel. - }]; - - let arguments = (ins - ConfinedAttr]>:$inputs, - TypedArrayAttrBase:$flags, - TypedArrayAttrBase:$data_type); - - let results = (outs I64:$results); - let hasCustomAssemblyFormat = 1; - let hasVerifier = 1; -} - -def Microkernel_BrgemmPrologueOp : Microkernel_Op<"brgemm.prologue"> { - let summary = "Prologue before executing the JITed brgemm " - "microkernel, and the context is considered core-level"; - let description = [{ - The operation has the following arguments: Input is the id of JITed kernel. - There is no output. - }]; - - let arguments = (ins I64:$inputs); - - let assemblyFormat = [{ - `(` $inputs `)` - attr-dict `:` functional-type($inputs, results) - }]; -} - -def Microkernel_BrgemmEpilogueOp : Microkernel_Op<"brgemm.epilogue"> { - let summary = "Epilogue after executing the JITed brgemm microkernel"; - let description = [{ - The operation has the following arguments: Input is the id of JITed kernel. - There is no output. - }]; - - let arguments = (ins I64:$inputs); - - let assemblyFormat = [{ - `(` $inputs `)` - attr-dict `:` functional-type($inputs, results) - }]; -} - -/* A generic input type of Microkernel_BrgemmOp, allowing for `BrgemmMemRef` and I64. - * The `BrgemmMemRef` should be a static MemRef, and for each operand its shape should be: - * Operand A: StaticMemRefRankOf<[F32, BF16, SI8, UI8], [3]>; - * Operand B (none-VNNI): StaticMemRefRankOf<[F32], [3]>; - * Operand B (VNNI): StaticMemRefRankOf<[BF16, SI8, UI8], [4]>; - * Operand C: StaticMemRefRankOf<[F32, SI32], [2]>; - */ -def BrgemmMemRefOrI64 : AnyTypeOf<[StaticMemRefRankOf<[F32, BF16, SI32, SI8, UI8], [2, 3, 4]>, I64]>; - -def Microkernel_BrgemmExecuteOp : Microkernel_Op<"brgemm.execute"> { - let summary = "execute the JITed brgemm kernel."; - let description = [{ - The operation has the following arguments: - 1) For stride mode, id of JITed kernel, MemRef of operand A/B/C, and the batch size; - 2) For addr mode, plus the length of addr list at the end. - There is no output. - }]; - - let arguments = (ins Variadic:$inputs); - - let assemblyFormat = [{ - `(` $inputs `)` - attr-dict `:` functional-type($inputs, results) - }]; - - let extraClassDeclaration = [{ - Value getDispatch() { return getInputs()[0]; } - - Value getOperandA() { return getInputs()[1]; } - - Value getOperandB() { return getInputs()[2]; } - - Value getOutput() { return getInputs()[3]; } - - Value getBatch() { return getInputs()[4]; } - - Value getAddrLen() { return getInputs()[5]; } - }]; - - let hasVerifier = 1; -} - -#endif // MICROKERNEL_OPS diff --git a/include/gc/Dialect/OneDNNGraph/CMakeLists.txt b/include/gc/Dialect/OneDNNGraph/CMakeLists.txt deleted file mode 100644 index 08bad8334..000000000 --- a/include/gc/Dialect/OneDNNGraph/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -if (NOT GC_ENABLE_DNNL_API) - message(STATUS "OneDNNGraphDialect is not enabled.") - return() -endif () - -add_mlir_dialect(OneDNNGraphOps onednn_graph) -add_mlir_doc(OneDNNGraphOps OneDNNGraphOps gc/Dialect/OneDNNGraph/ -gen-op-doc) -add_mlir_doc(OneDNNGraphDialect OneDNNGraphDialect gc/Dialect/OneDNNGraph/ -gen-dialect-doc) diff --git a/include/gc/Dialect/OneDNNGraph/OneDNNGraphDialect.h b/include/gc/Dialect/OneDNNGraph/OneDNNGraphDialect.h deleted file mode 100644 index bae992507..000000000 --- a/include/gc/Dialect/OneDNNGraph/OneDNNGraphDialect.h +++ /dev/null @@ -1,19 +0,0 @@ -//===- OneDNNGraphDialect.h - OneDNN input dialect --------------*- C++ -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef GC_DIALECTS_ONEDNNGRAPHDIALECT_H -#define GC_DIALECTS_ONEDNNGRAPHDIALECT_H - -#include "mlir/IR/Dialect.h" -#include "mlir/IR/OpDefinition.h" -#include "mlir/IR/OpImplementation.h" - -#define GET_OP_CLASSES -#include "gc/Dialect/OneDNNGraph/OneDNNGraphOpsDialect.h.inc" - -#endif // GC_DIALECTS_ONEDNNGRAPHDIALECT_H diff --git a/include/gc/Dialect/OneDNNGraph/OneDNNGraphDialect.td b/include/gc/Dialect/OneDNNGraph/OneDNNGraphDialect.td deleted file mode 100644 index 6e1eaceca..000000000 --- a/include/gc/Dialect/OneDNNGraph/OneDNNGraphDialect.td +++ /dev/null @@ -1,27 +0,0 @@ -//===- OneDNNGraphDialect.td - OneDNN input dialect --------*- tablegen -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef ONEDNNGRAPH_DIALECT -#define ONEDNNGRAPH_DIALECT - -include "mlir/IR/OpBase.td" - -//===----------------------------------------------------------------------===// -// OneDNNGraph dialect definition. -//===----------------------------------------------------------------------===// - -def OneDNNGraphDialect : Dialect { - let name = "onednn_graph"; - let summary = "A dialect for oneDNN Graph."; - let description = [{ - This dialect follows oneDNN Graph Specification. - }]; - let cppNamespace = "::mlir::onednn_graph"; -} - -#endif // ONEDNNGRAPH_DIALECT diff --git a/include/gc/Dialect/OneDNNGraph/OneDNNGraphOps.h b/include/gc/Dialect/OneDNNGraph/OneDNNGraphOps.h deleted file mode 100644 index ec571e6ca..000000000 --- a/include/gc/Dialect/OneDNNGraph/OneDNNGraphOps.h +++ /dev/null @@ -1,24 +0,0 @@ -//===- OneDNNGraphOps.h - OneDNN input dialect ops --------------*- C++ -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef GC_DIALECTS_ONEDNNGRAPHOPS_H -#define GC_DIALECTS_ONEDNNGRAPHOPS_H - -#include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h" -#include "mlir/Dialect/Tensor/IR/Tensor.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/Dialect.h" -#include "mlir/IR/OpDefinition.h" -#include "mlir/Interfaces/InferTypeOpInterface.h" -#include "mlir/Interfaces/SideEffectInterfaces.h" -#include "mlir/Dialect/Traits.h" - -#define GET_OP_CLASSES -#include "gc/Dialect/OneDNNGraph/OneDNNGraphOps.h.inc" - -#endif // GC_DIALECTS_ONEDNNGRAPHOPS_H diff --git a/include/gc/Dialect/OneDNNGraph/OneDNNGraphOps.td b/include/gc/Dialect/OneDNNGraph/OneDNNGraphOps.td deleted file mode 100644 index e6d7ef015..000000000 --- a/include/gc/Dialect/OneDNNGraph/OneDNNGraphOps.td +++ /dev/null @@ -1,205 +0,0 @@ -//===- OneDNNGraphOps.td - OneDNN input dialect ops --------*- tablegen -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef ONEDNNGRAPH_OPS -#define ONEDNNGRAPH_OPS - -include "mlir/Interfaces/InferTypeOpInterface.td" -include "mlir/Interfaces/SideEffectInterfaces.td" -include "mlir/Interfaces/DestinationStyleOpInterface.td" -include "mlir/IR/AttrTypeBase.td" -include "mlir/IR/OpBase.td" -include "gc/Dialect/OneDNNGraph/OneDNNGraphDialect.td" -include "gc/Dialect/OneDNNGraph/OneDNNGraphTypes.td" - -//===----------------------------------------------------------------------===// -// Base OneDNNGraph operation definition. -//===----------------------------------------------------------------------===// - -class OneDNNGraph_Op traits = []> : - Op; - -class OneDNNGraph_ElemwiseUnaryOp traits = []> : - OneDNNGraph_Op { - let arguments = (ins OneDNNGraph_FloatTensor:$operand); - let results = (outs OneDNNGraph_FloatTensor:$result); - - let assemblyFormat = - "operands attr-dict `:` functional-type(operands, results)"; -} - -class OneDNNGraph_ElemwiseBinaryOp traits = []> : - OneDNNGraph_Op { - let arguments = (ins OneDNNGraph_FloatTensor:$input_a, - OneDNNGraph_FloatTensor:$input_b, - DefaultValuedOptionalAttr:$auto_broadcast); - let results = (outs OneDNNGraph_FloatTensor:$result); - - let assemblyFormat = - "operands attr-dict `:` functional-type(operands, results)"; -} - -class OneDNNGraph_ReduceOp traits = []> : - OneDNNGraph_Op { - let arguments = (ins OneDNNGraph_FloatTensor:$operand, - DefaultValuedOptionalAttr:$axes, - DefaultValuedOptionalAttr:$keep_dims); - let results = (outs OneDNNGraph_FloatTensor:$result); - - let hasVerifier = 1; - let hasCanonicalizer = 1; - let assemblyFormat = - "operands attr-dict `:` functional-type(operands, results)"; -} - -//===----------------------------------------------------------------------===// -// OneDNNGraph op definitions -//===----------------------------------------------------------------------===// - -// Matmul - -def OneDNNGraph_MatMulOp : - OneDNNGraph_Op<"matmul", - [SameOperandsAndResultElementType, InferTensorTypeAdaptor]> { - let summary = [{ - MatMul operation computes the product of two tensors with optional bias addition. - }]; - let description = [{ - `https://oneapi-src.github.io/oneDNN/dev_guide_op_matmul.html` - }]; - - let arguments = (ins OneDNNGraph_FloatTensor:$input_a, - OneDNNGraph_FloatTensor:$input_b, - Optional:$bias, - DefaultValuedOptionalAttr:$transpose_a, - DefaultValuedOptionalAttr:$transpose_b); - let results = (outs OneDNNGraph_FloatTensor:$result); - - let assemblyFormat = - "operands attr-dict `:` functional-type(operands, results)"; -} - -// Common Unary - -def OneDNNGraph_ReLUOp : OneDNNGraph_ElemwiseUnaryOp<"relu"> { - let summary = "element-wise relu"; - let description = [{ - `https://oneapi-src.github.io/oneDNN/dev_guide_op_relu.html` - }]; -} - -def OneDNNGraph_SigmoidOp : OneDNNGraph_ElemwiseUnaryOp<"sigmoid"> { - let summary = "element-wise sigmoid"; - let description = [{ - `https://oneapi-src.github.io/oneDNN/dev_guide_op_sigmoid.html` - }]; -} - -// Special Unary - -def OneDNNGraph_TypeCastOp : OneDNNGraph_Op<"type_cast", [SameOperandsAndResultShape]> { - let summary = [{ - TypeCast operation performs element-wise cast from input data type - to the data type given by output tensor. - }]; - let description = [{ - `https://oneapi-src.github.io/oneDNN/dev_guide_op_typecast.html` - }]; - - let arguments = (ins OneDNNGraph_FloatTensor:$operand); - let results = (outs OneDNNGraph_FloatTensor:$result); - - let assemblyFormat = - "operands attr-dict `:` functional-type(operands, results)"; -} - -def OneDNNGraph_PowOp : OneDNNGraph_Op<"pow", [SameOperandsAndResultType]> { - let summary = [{ - Pow operation performs an element-wise power operation on a given input - tensor with a single value attribute beta as its exponent. - }]; - let description = [{ - `https://oneapi-src.github.io/oneDNN/dev_guide_op_pow.html` - }]; - - let arguments = (ins OneDNNGraph_FloatTensor:$operand, - F32Attr:$beta); - let results = (outs OneDNNGraph_FloatTensor:$result); - - let assemblyFormat = - "operands attr-dict `:` functional-type(operands, results)"; -} - -// Common Binary - -def OneDNNGraph_AddOp : OneDNNGraph_ElemwiseBinaryOp<"add", [Commutative]> { - let summary = [{ - Add operation performs element-wise addition operation with two - given tensors applying multi-directional broadcast rules. - }]; - let description = [{ - `https://oneapi-src.github.io/oneDNN/dev_guide_op_add.html` - }]; -} - -def OneDNNGraph_MulOp : OneDNNGraph_ElemwiseBinaryOp<"mul", [Commutative]> { - let summary = [{ - Multiply operation performs element-wise multiply operation with two - given tensors applying multi-directional broadcast rules. - }]; - let description = [{ - `https://oneapi-src.github.io/oneDNN/dev_guide_op_multiply.html` - }]; -} - -def OneDNNGraph_SubOp : OneDNNGraph_ElemwiseBinaryOp<"sub"> { - let summary = [{ - Subtract operation performs element-wise subtraction operation with - two given tensors applying multi-directional broadcast rules. - }]; - let description = [{ - `https://oneapi-src.github.io/oneDNN/dev_guide_op_subtract.html` - }]; -} - -def OneDNNGraph_DivOp : OneDNNGraph_ElemwiseBinaryOp<"div"> { - let summary = [{ - Divide operation performs element-wise division operation with two - given tensors applying multi-directional broadcast rules. - }]; - let description = [{ - `https://oneapi-src.github.io/oneDNN/dev_guide_op_divide.html` - }]; -} - -// Common Reduce - -def OneDNNGraph_ReduceSumOp : OneDNNGraph_ReduceOp<"reduce_sum"> { - let summary = [{ - ReduceSum operation performs the reduction with addition on a given - src data along dimensions specified by axes. - }]; - let description = [{ - `https://oneapi-src.github.io/oneDNN/dev_guide_op_reducesum.html` - }]; -} - -def OneDNNGraph_ReduceMeanOp : OneDNNGraph_ReduceOp<"reduce_mean"> { - let summary = [{ - ReduceMean operation performs the reduction with finding the arithmetic - mean on a given src data along dimensions specified by axes. - }]; - let description = [{ - `https://oneapi-src.github.io/oneDNN/dev_guide_op_reducemean.html` - }]; -} - -#endif // ONEDNNGRAPH_OPS \ No newline at end of file diff --git a/include/gc/Dialect/OneDNNGraph/OneDNNGraphTypes.h b/include/gc/Dialect/OneDNNGraph/OneDNNGraphTypes.h deleted file mode 100644 index c897fb8be..000000000 --- a/include/gc/Dialect/OneDNNGraph/OneDNNGraphTypes.h +++ /dev/null @@ -1,17 +0,0 @@ -//===- OneDNNGraphTypes.h - OneDNN input dialect types ----------*- C++ -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef ONEDNNGRAPH_ONEDNNGRAPHTYPES_H -#define ONEDNNGRAPH_ONEDNNGRAPHTYPES_H - -#include "mlir/IR/BuiltinTypes.h" - -#define GET_TYPEDEF_CLASSES -#include "gc/Dialect/OneDNNGraph/OneDNNGraphOpsTypes.h.inc" - -#endif // ONEDNNGRAPH_ONEDNNGRAPHTYPES_H diff --git a/include/gc/Dialect/OneDNNGraph/OneDNNGraphTypes.td b/include/gc/Dialect/OneDNNGraph/OneDNNGraphTypes.td deleted file mode 100644 index 45e819b0d..000000000 --- a/include/gc/Dialect/OneDNNGraph/OneDNNGraphTypes.td +++ /dev/null @@ -1,42 +0,0 @@ -//===-- OneDNNGraphTypes.td - DESC -------------------------*- tablegen -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef ONEDNNGRAPH_TYPES -#define ONEDNNGRAPH_TYPES - -include "mlir/IR/BuiltinTypes.td" -include "mlir/IR/AttrTypeBase.td" -include "gc/Dialect/OneDNNGraph/OneDNNGraphDialect.td" - -//===----------------------------------------------------------------------===// -// OneDNNGraph type definitions -//===----------------------------------------------------------------------===// - -//===----------------------------------------------------------------------===// -// Floating-point types. -//===----------------------------------------------------------------------===// -def OneDNNGraph_Float : AnyTypeOf<[F32, - F16, - BF16]>; - -//===----------------------------------------------------------------------===// -// Integer types. -//===----------------------------------------------------------------------===// - -def OneDNNGraph_Int : AnyTypeOf<[SI<8>, - UI<8>]>; - -def OneDNNGraph_DataType : AnyTypeOf<[ - OneDNNGraph_Float, - OneDNNGraph_Int - ]>; - -def OneDNNGraph_LogicalTensor : TensorOf<[OneDNNGraph_DataType]>; -def OneDNNGraph_FloatTensor : TensorOf<[OneDNNGraph_Float]>; - -#endif // ONEDNNGRAPH_TYPES diff --git a/include/gc/ExecutionEngine/CPURuntime/Microkernel/BrgemmInterface.h b/include/gc/ExecutionEngine/CPURuntime/Microkernel/BrgemmInterface.h deleted file mode 100644 index 21641144e..000000000 --- a/include/gc/ExecutionEngine/CPURuntime/Microkernel/BrgemmInterface.h +++ /dev/null @@ -1,112 +0,0 @@ -//===-- BrgemmInterface.h - The interfaces of runtime Brgemm ----*- C++ -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef GC_EXECUTIONENGINE_CPURUNTIME_MICROKERNEL_BRGEMMINTERFACE_H -#define GC_EXECUTIONENGINE_CPURUNTIME_MICROKERNEL_BRGEMMINTERFACE_H - -#include - -#include "gc/Utils.h" - -extern "C" { -// Runtime interfaces - -/** - * Dispatch (JIT) the Brgemm kernel based on given parameters using DNNL - * Inputs: - * M, N, K: The size of Brgemm dims, given in element size; - * LDA, LDB, LDC: The stride of leading dim of - * each Brgemm matrix, given in element size; - * stride_a, stride_b: The stride of batch of Brgemm - * input A & B, given in element size; - * dtypeA, dtypeB: The dtype of Brgemm input A and B, - * given in dnnl type value. - * Output: A handle of dispatched kernel. - */ -GC_DLL_EXPORT int64_t dnnl_brgemm_dispatch(int64_t M, int64_t N, int64_t K, - int64_t LDA, int64_t LDB, - int64_t LDC, int64_t stride_a, - int64_t stride_b, float beta, - int64_t dtypeA, int64_t dtypeB); - -/** - * Config the AMX tile context for given kernel. - * Inputs: A handle of dispatched kernel. - * Output: None. - */ -GC_DLL_EXPORT void dnnl_brgemm_tileconfig(int64_t kernel); - -/** - * Release the current AMX tile context. - * Inputs: None. - * Output: None. - */ -GC_DLL_EXPORT void dnnl_brgemm_tilerelease(); - -/** - * Execute the given kernel with given parameters. - * Inputs: - * kernel: A handle of dispatched kernel; - * A, A_offset, B, B_offset, C, C_offset: - * Pointers and starting offset of each Brgemm matrix; - * num: Batch size of Brgemm. - * Output: None. - */ -GC_DLL_EXPORT void dnnl_brgemm_execute(int64_t kernel, void *A, - uint64_t A_offset, void *B, - uint64_t B_offset, void *C, - uint64_t C_offset, int num); -} - -struct bf16_t { - uint16_t storage_; - union caster_t { - uint32_t vl; - float vf; - }; - operator float() const { - caster_t val; - val.vl = uint32_t(storage_) << 16; - return val.vf; - } - bool operator==(const bf16_t &compare_to) const { - return storage_ == compare_to.storage_; - } - bool operator!=(const bf16_t &compare_to) const { - return storage_ != compare_to.storage_; - } - bf16_t(float v) { - if (std::isnan(v)) { - storage_ = UINT32_C(0x7FC0); - } else { - caster_t caster; - caster.vf = v; - uint32_t rounding_bias = ((caster.vl >> 16) & 1) + UINT32_C(0x7FFF); - storage_ = static_cast((caster.vl + rounding_bias) >> 16); - } - } - bf16_t() : storage_(0) {} - inline static bf16_t from_storage(uint16_t v) { - bf16_t ret; - ret.storage_ = v; - return ret; - } -}; - -// Naive implementation of `dnnl_brgemm_dispatch` -int64_t dnnl_brgemm_dispatch_naive(int64_t M, int64_t N, int64_t K, int64_t LDA, - int64_t LDB, int64_t LDC, int64_t stride_a, - int64_t stride_b, float beta, int64_t dtypeA, - int64_t dtypeB); - -// Naive implementation of `dnnl_brgemm_execute` -void dnnl_brgemm_execute_naive(int64_t kernel, void *A, uint64_t A_offset, - void *B, uint64_t B_offset, void *C, - uint64_t C_offset, int num); - -#endif diff --git a/include/gc/ExecutionEngine/Driver/Driver.h b/include/gc/ExecutionEngine/Driver/Driver.h deleted file mode 100644 index 7ae185f42..000000000 --- a/include/gc/ExecutionEngine/Driver/Driver.h +++ /dev/null @@ -1,69 +0,0 @@ -//===-- Driver.h - The top-level MLIR compiler driver -----------*- C++ -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef GC_EXECUTIONENGINE_DRIVER_DRIVER_H -#define GC_EXECUTIONENGINE_DRIVER_DRIVER_H - -#include "mlir/ExecutionEngine/CRunnerUtils.h" -#include "mlir/ExecutionEngine/ExecutionEngine.h" -#include -#include - -namespace mlir { -class DialectRegistry; -namespace gc { - -DialectRegistry &initCompilerAndGetDialects(); - -// the pointers to XXXMemRefType -using GeneralMemrefPtr = void *; -using JitModuleFuncT = void (*)(void **); - -struct DriverOptions { - /// the optimization level for the LLVM-JIT - llvm::CodeGenOptLevel jitCodeGenOptLevel = llvm::CodeGenOptLevel::Aggressive; - /// whether to run the MLIR transformation passes - bool runTransforms = true; - /// todo: target machine, etc. -}; - -class JitModule { -public: - static llvm::Expected> - create(Operation *op, const DriverOptions &options = {}); - - /// args should be an array of XXXMemrefType* - void call(GeneralMemrefPtr *args, std::size_t numArgs) { - // Silly code, MLIR execution engine requires pointers of real args as - // inputs - llvm::SmallVector realargs; - realargs.reserve(numArgs); - for (size_t i = 0; i < numArgs; i++) { - realargs.push_back(&args[i]); - } - compute(realargs.data()); - } - - /// directly call compute(). args should be an array of void*. args[i] should - /// be a pointer to the real data. For passing memref, users need to 1) create - /// a pointer to XXXMemrefType 2) store the pointer to pointer to - /// XXXMemrefType in args[i] - void callRaw(void **args) { compute(args); } - - JitModule(std::unique_ptr engine, JitModuleFuncT compute); - ~JitModule(); - -private: - std::unique_ptr engine; - JitModuleFuncT compute; -}; - -} // namespace gc -} // namespace mlir - -#endif \ No newline at end of file diff --git a/include/gc/ExecutionEngine/GPURuntime/GpuOclRuntime.h b/include/gc/ExecutionEngine/GPURuntime/GpuOclRuntime.h deleted file mode 100644 index 459d60993..000000000 --- a/include/gc/ExecutionEngine/GPURuntime/GpuOclRuntime.h +++ /dev/null @@ -1,469 +0,0 @@ -//===-- GpuOclRuntime.h - GPU OpenCL runtime --------------------*- C++ -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef GC_GPUOCLRUNTIME_H -#define GC_GPUOCLRUNTIME_H - -namespace mlir::gc::gpu { -constexpr char GPU_OCL_MALLOC_DEV[] = "gcGpuOclMallocDev"; -constexpr char GPU_OCL_MALLOC_SHARED[] = "gcGpuOclMallocShared"; -constexpr char GPU_OCL_DEALLOC[] = "gcGpuOclDealloc"; -constexpr char GPU_OCL_MEMCPY[] = "gcGpuOclMemcpy"; -constexpr char GPU_OCL_KERNEL_CREATE[] = "gcGpuOclKernelCreate"; -constexpr char GPU_OCL_KERNEL_DESTROY[] = "gcGpuOclKernelDestroy"; -constexpr char GPU_OCL_KERNEL_LAUNCH[] = "gcGpuOclKernelLaunch"; -constexpr char GPU_OCL_FINISH[] = "gcGpuOclFinish"; -constexpr char GPU_OCL_MOD_DESTRUCTOR[] = "gcGpuOclModuleDestructor"; -} // namespace mlir::gc::gpu - -#ifndef GC_GPU_OCL_CONST_ONLY -#include -#include -#include - -#include - -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/ExecutionEngine/ExecutionEngine.h" -#include "mlir/IR/BuiltinOps.h" -#include "mlir/Pass/PassManager.h" - -namespace mlir::gc::gpu { -struct OclDevCtxPair { - cl_device_id device; - cl_context context; - explicit OclDevCtxPair(cl_device_id device, cl_context context) - : device(device), context(context) {} - - bool operator==(const OclDevCtxPair &other) const { - return device == other.device && context == other.context; - } -}; -} // namespace mlir::gc::gpu -template <> struct std::hash { - std::size_t - operator()(const mlir::gc::gpu::OclDevCtxPair &pair) const noexcept { - return std::hash()(pair.device) ^ - std::hash()(pair.context); - } -}; // namespace std -namespace mlir::gc::gpu { -struct OclModule; -struct OclContext; -struct OclModuleBuilder; - -struct OclRuntime { - // Returns the available Intel GPU device ids. - [[nodiscard]] static llvm::Expected> - gcIntelDevices(size_t max = std::numeric_limits::max()); - - [[nodiscard]] static llvm::Expected get(); - - [[nodiscard]] static llvm::Expected get(cl_device_id device); - - [[nodiscard]] static llvm::Expected get(cl_command_queue queue); - - [[nodiscard]] static llvm::Expected get(cl_device_id device, - cl_context context); - - static bool isOutOfOrder(cl_command_queue queue); - - [[nodiscard]] cl_context getContext() const; - - [[nodiscard]] cl_device_id getDevice() const; - - [[nodiscard]] llvm::Expected - createQueue(bool outOfOrder = false) const; - - [[nodiscard]] static llvm::Expected - releaseQueue(cl_command_queue queue); - - [[nodiscard]] llvm::Expected usmAllocDev(size_t size) const; - - [[nodiscard]] llvm::Expected usmAllocShared(size_t size) const; - - [[nodiscard]] llvm::Expected usmFree(const void *ptr) const; - - [[nodiscard]] llvm::Expected usmCpy(OclContext &ctx, const void *src, - void *dst, size_t size) const; - - template - [[nodiscard]] llvm::Expected usmNewDev(size_t size) const { - auto expected = usmAllocDev(size * sizeof(T)); - if (expected) { - return static_cast(*expected); - } - return expected.takeError(); - } - - template - [[nodiscard]] llvm::Expected usmNewShared(size_t size) const { - auto expected = usmAllocShared(size * sizeof(T)); - if (expected) { - return static_cast(*expected); - } - return expected.takeError(); - } - - template - [[nodiscard]] llvm::Expected usmCpy(OclContext &ctx, const T *src, - T *dst, size_t size) const { - return usmCpy(ctx, static_cast(src), static_cast(dst), - size * sizeof(T)); - } - - // Use with caution! This is safe to check validity of USM, but may be false - // positive for any other kinds. - bool isUsm(const void *ptr) const; - - bool operator==(const OclRuntime &other) const { - return getDevice() == other.getDevice() && - getContext() == other.getContext(); - } - -private: - struct Ext; - struct Exports; - friend OclContext; - friend OclModuleBuilder; - template friend struct DynamicExecutor; - template friend struct StaticExecutor; - explicit OclRuntime(const Ext &ext); - const Ext &ext; - -#ifndef NDEBUG - static void debug(const char *file, int line, const char *msg); -#endif -}; - -static constexpr int64_t ZERO = 0; -static constexpr auto ZERO_PTR = const_cast(&ZERO); - -// NOTE: The context is mutable and not thread-safe! It's expected to be used in -// a single thread only. -struct OclContext { - const OclRuntime &runtime; - const cl_command_queue queue; - // Create 'cl_event' object, for each enqueued command, that can be used to - // query or wait for the command to complete. This is required in case of - // out-of-order execution (CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE), but can - // also be used to get the last event. When the execution is completed, the - // 'lastEvent' field contains the event of the last enqueued command. If this - // field is false, 'waitList' is ignored. - const bool createEvents; - cl_uint waitListLen; - cl_event *waitList; - cl_event lastEvent; - - explicit OclContext(const OclRuntime &runtime, cl_command_queue queue) - : OclContext(runtime, queue, OclRuntime::isOutOfOrder(queue)) {} - - explicit OclContext(const OclRuntime &runtime, cl_command_queue queue, - bool createEvents, cl_uint waitListLen = 0, - cl_event *waitList = nullptr); - - OclContext(const OclContext &) = delete; - OclContext &operator=(const OclContext &) = delete; - ~OclContext(); - - [[nodiscard]] llvm::Expected finish(); - -private: - friend OclRuntime; - friend OclRuntime::Exports; - template friend struct DynamicExecutor; - template friend struct StaticExecutor; - std::unordered_set *clPtrs; - - void setLastEvent(cl_event event); -}; - -struct OclModule { - const OclRuntime runtime; - // If all the function arguments have static shapes, then this field is true - // and main.staticMain is used. Otherwise, main.wrappedMain is used. - const bool isStatic; - - ~OclModule(); - OclModule(const OclModule &) = delete; - OclModule &operator=(const OclModule &) = delete; - OclModule(const OclModule &&) = delete; - OclModule &operator=(const OclModule &&) = delete; - - void dumpToObjectFile(StringRef filename) const { - engine->dumpToObjectFile(filename); - } - -private: - friend OclModuleBuilder; - template friend struct DynamicExecutor; - template friend struct OclModuleExecutorBase; - template friend struct StaticExecutor; - // This function is only created when all args are memrefs with static shape. - using StaticMainFunc = void (*)(OclContext *, void **); - // Wrapper, generated by the engine. The arguments are pointers to the values. - using WrappedMainFunc = void (*)(void **); - union MainFunc { - StaticMainFunc staticMain; - WrappedMainFunc wrappedMain; - }; - const MainFunc main; - const ArrayRef argTypes; - std::unique_ptr engine; - - explicit OclModule(const OclRuntime &runtime, const bool isStatic, - const MainFunc main, const ArrayRef argTypes, - std::unique_ptr engine) - : runtime(runtime), isStatic(isStatic), main(main), argTypes(argTypes), - engine(std::move(engine)) {} -}; - -struct OclModuleBuilderOpts { - StringRef funcName = {}; - bool printIr = false; - bool spirvDump = false; - bool enableObjectDump = false; - ArrayRef sharedLibPaths = {}; - void (*pipeline)(OpPassManager &) = nullptr; -}; - -struct OclModuleBuilder { - friend OclRuntime; - explicit OclModuleBuilder(ModuleOp module, - const OclModuleBuilderOpts &opts = {}); - explicit OclModuleBuilder(OwningOpRef &module, - const OclModuleBuilderOpts &opts = {}) - : OclModuleBuilder(module.release(), opts) {} - explicit OclModuleBuilder(OwningOpRef &&module, - const OclModuleBuilderOpts &opts = {}) - : OclModuleBuilder(module.release(), opts) {} - - llvm::Expected> - build(const OclRuntime &runtime); - - llvm::Expected> - build(cl_command_queue queue); - - llvm::Expected> build(cl_device_id device, - cl_context context); - -private: - ModuleOp mlirModule; - const bool printIr; - const bool spirvDump; - const bool enableObjectDump; - const ArrayRef sharedLibPaths; - void (*const pipeline)(OpPassManager &); - const StringRef funcName; - const ArrayRef argTypes; - std::shared_mutex mux; - std::unordered_map> - cache; - - llvm::Expected> - build(const OclRuntime::Ext &ext); -}; - -// NOTE: This class is mutable and not thread-safe! -template struct OclModuleExecutorBase { - - void reset() { - args.clear(); - clPtrs.clear(); - argCounter = 0; - } - - Type getArgType(unsigned idx) const { return mod->argTypes[idx]; } - - [[nodiscard]] bool isSmall() const { return args.small(); } - -protected: - struct Args : SmallVector { - [[nodiscard]] bool small() const { return this->isSmall(); } - }; - - const std::shared_ptr &mod; - // Contains the pointers of all non-USM arguments. It's expected, that the - // arguments are either USM or CL pointers and most probably are USM, thus, - // in most cases, this set will be empty. - std::unordered_set clPtrs; - Args args; - unsigned argCounter = 0; - - explicit OclModuleExecutorBase(std::shared_ptr &mod) - : mod(mod) {} - -#ifndef NDEBUG - void checkExec(const OclContext &ctx) const { - auto rt = OclRuntime::get(ctx.queue); - assert(rt); - assert(*rt == mod->runtime); - assert(argCounter == mod->argTypes.size()); - } - - void checkArg(const void *alignedPtr, bool isUsm = true) const { - assert(!isUsm || mod->runtime.isUsm(alignedPtr)); - } -#endif -}; - -// NOTE: This executor can only be used if mod->isStatic == true! -template struct StaticExecutor : OclModuleExecutorBase { - explicit StaticExecutor(std::shared_ptr &mod) - : OclModuleExecutorBase(mod) { - assert(this->mod->isStatic); - } - - void exec(OclContext &ctx) { -#ifndef NDEBUG - this->checkExec(ctx); -#endif - ctx.clPtrs = &this->clPtrs; - this->mod->main.staticMain(&ctx, this->args.data()); - } - - void arg(void *alignedPtr, bool isUsm = true) { -#ifndef NDEBUG - this->checkArg(alignedPtr, isUsm); - std::ostringstream oss; - oss << "Arg" << this->argCounter << ": alignedPtr=" << alignedPtr - << ", isUsm=" << (isUsm ? "true" : "false"); - OclRuntime::debug(__FILE__, __LINE__, oss.str().c_str()); -#endif - ++this->argCounter; - this->args.emplace_back(alignedPtr); - if (!isUsm) { - this->clPtrs.insert(alignedPtr); - } - } - - template void arg(T *alignedPtr, bool isUsm = true) { - arg(reinterpret_cast(alignedPtr), isUsm); - } - - void operator()(OclContext &ctx) { exec(ctx); } - - template void operator()(OclContext &ctx, T *ptr1, ...) { - { - this->reset(); - arg(reinterpret_cast(ptr1)); - va_list args; - va_start(args, ptr1); - for (unsigned i = 0, n = this->mod->argTypes.size() - 1; i < n; i++) { - arg(va_arg(args, void *)); - } - va_end(args); - exec(ctx); - } - } -}; - -// The main function arguments are added in the following format - -// https://mlir.llvm.org/docs/TargetLLVMIR/#c-compatible-wrapper-emission. -// NOTE: This executor can only be used if mod->isStatic != true! -template struct DynamicExecutor : OclModuleExecutorBase { - explicit DynamicExecutor(std::shared_ptr &mod) - : OclModuleExecutorBase(mod) { - assert(!this->mod->isStatic); - } - - void exec(OclContext &ctx) { -#ifndef NDEBUG - this->checkExec(ctx); -#endif - auto size = this->args.size(); - auto ctxPtr = &ctx; - this->args.emplace_back(&ctxPtr); - this->args.emplace_back(&ctxPtr); - this->args.emplace_back(ZERO_PTR); - this->mod->main.wrappedMain(this->args.data()); - this->args.truncate(size); - } - - void arg(void *&alignedPtr, size_t rank, const int64_t *shape, - const int64_t *strides, bool isUsm = true) { - arg(alignedPtr, alignedPtr, ZERO, rank, shape, strides, isUsm); - } - - // NOTE: The argument values are not copied, only the pointers are stored! - void arg(void *&allocatedPtr, void *&alignedPtr, const int64_t &offset, - size_t rank, const int64_t *shape, const int64_t *strides, - bool isUsm = true) { -#ifndef NDEBUG - this->checkArg(alignedPtr, isUsm); - if (auto type = - llvm::dyn_cast(this->getArgType(this->argCounter))) { - if (type.hasStaticShape()) { - auto size = type.getShape(); - assert(rank == size.size()); - for (size_t i = 0; i < rank; i++) { - assert(shape[i] == size[i]); - } - - SmallVector expectedStrides; - if (int64_t expectedOffset; !failed( - getStridesAndOffset(type, expectedStrides, expectedOffset))) { - assert(expectedOffset == offset); - for (size_t i = 0; i < rank; i++) { - assert(expectedStrides[i] == strides[i]); - } - } - } - } - - std::ostringstream oss; - oss << "Arg" << this->argCounter << ": ptr=" << allocatedPtr - << ", alignedPtr=" << alignedPtr - << ", isUsm=" << (isUsm ? "true" : "false") << ", offset=" << offset - << ", shape=["; - for (unsigned i = 0; i < rank; i++) { - oss << shape[i] << (i + 1 < rank ? ", " : "]"); - } - oss << ", strides=["; - for (unsigned i = 0; i < rank; i++) { - oss << strides[i] << (i + 1 < rank ? ", " : "]"); - } - OclRuntime::debug(__FILE__, __LINE__, oss.str().c_str()); -#endif - - ++this->argCounter; - this->args.emplace_back(&allocatedPtr); - this->args.emplace_back(&alignedPtr); - this->args.emplace_back(const_cast(&offset)); - for (size_t i = 0; i < rank; i++) { - this->args.emplace_back(const_cast(&shape[i])); - } - for (size_t i = 0; i < rank; i++) { - this->args.emplace_back(const_cast(&strides[i])); - } - if (!isUsm) { - this->clPtrs.insert(alignedPtr); - } - } - - template - void arg(T *&alignedPtr, size_t rank, const int64_t *shape, - const int64_t *strides, bool isUsm = true) { - arg(reinterpret_cast(alignedPtr), rank, shape, strides, isUsm); - } - - template - void arg(T *&allocatedPtr, T *&alignedPtr, const int64_t &offset, size_t rank, - const int64_t *shape, const int64_t *strides, bool isUsm = true) { - arg(reinterpret_cast(allocatedPtr), - reinterpret_cast(alignedPtr), offset, rank, shape, strides, - isUsm); - } - - void operator()(OclContext &ctx) { exec(ctx); } -}; -} // namespace mlir::gc::gpu -#else -#undef GC_GPU_OCL_CONST_ONLY -#endif -#endif diff --git a/include/gc/GpuCompiler.h b/include/gc/GpuCompiler.h new file mode 100644 index 000000000..3b57ea024 --- /dev/null +++ b/include/gc/GpuCompiler.h @@ -0,0 +1,270 @@ +//===-- GpuCompiler.h - GPU Compiler interface -----------------*- C++ -*-===// +// +// This file is licensed under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gc/Runtime/GpuRuntime.h" + +namespace mlir { +class ModuleOp; +template class OwningOpRef; +} // namespace mlir + +namespace gc { +struct JitEngine; + +namespace gpu { +struct Program; +using FutureProgram = std::shared_future>; + +struct CompileOptions { + CompileOptions() = default; + CompileOptions(unsigned id) : arch(DeviceInfo::idToArch(id)) {} + CompileOptions(const std::string &arch) : arch(arch) {} + std::string arch; + std::string igcOpts; + bool wait{true}; + bool dump{false}; +}; + +struct GpuCompiler { + using Options = CompileOptions; + + GpuCompiler(unsigned concurrency = 0); + ~GpuCompiler(); + GpuCompiler(const GpuCompiler &) = delete; + GpuCompiler &operator=(const GpuCompiler &) = delete; + GpuCompiler(GpuCompiler &&) = delete; + GpuCompiler &operator=(GpuCompiler &&) = delete; + + // Returns a platform-wide shared instance of the GPU compiler. + static GpuCompiler &get(); + + // The program is linked with Rt::symbols(). Src is either + // mlir::OwningOpRef or a string. + template + FutureProgram compile(Src &&src, Options opts = {}) { + return compile(std::forward(src), std::move(opts), + Rt::template symbols()); + } + + // The program is linked with RtSymbols. Takes ownership of src and runs the + // pipeline in its context; the caller must have loaded the required dialects. + FutureProgram compile(mlir::OwningOpRef src, Options opts, + const RtSymbols &syms); + // Prevents implicit OwningOpRef construction, erasing the caller's module. + FutureProgram compile(mlir::ModuleOp src, Options opts, + const RtSymbols &syms) = delete; + FutureProgram compile(std::string_view src, Options opts, + const RtSymbols &syms); +}; + +struct Program { + struct Symbols { +#define GC_PROG_SYM(name, type) \ + static constexpr char name##Name[] = #name; \ + using name##Type = type; \ + name##Type name + + GC_PROG_SYM(main, void (*)(void **)); + // The number of events returned by the main function. If > 0, the last + // main argument points to an array of `eventNum` Event*, filled by main. + GC_PROG_SYM(eventNum, uint32_t (*)()); + GC_PROG_SYM(destructor, void (*)()); +#undef GC_PROG_SYM + } const sym; + + uint32_t eventNum() const { return evtNum; } + + // Launch the program with the specified arguments. The args list contains + // pointers to the actual values. The list must be built with + // `ArgsBuilder::build()`. It can be used for multiple non-concurrent + // launches, if not modified. If `eventNum()` is > 0, the function returns the + // pointer to the first event, otherwise - returns nullptr. The returned + // events must be consumed by the caller after each call. Rt must be the + // runtime the program was compiled for. + template > + typename Rt::Event **main(Vector &args, Rt *runtime, + typename Rt::Event *const *deps = nullptr, + uint32_t nDeps = 0) { + assert(args.size() >= argsTailSize()); + void **data = args.data(); + auto pos = args.size() - argsTailSize(); + data[pos] = &runtime; + data[pos + 1] = &deps; + data[pos + 2] = &nDeps; + sym.main(data); + // data[pos + 3] is a pointer to data[pos + 4] which points to the array + // of events, if eventNum() > 0. + return eventNum() ? static_cast(data[pos + 4]) + : nullptr; + } + + Program(std::unique_ptr<::gc::JitEngine> jit); + ~Program(); + +private: + uint32_t const evtNum; + std::unique_ptr<::gc::JitEngine> jit; + + Program() = delete; + Program(const Program &) = delete; + Program &operator=(const Program &) = delete; + Program(Program &&) = delete; + Program &operator=(Program &&) = delete; + + template friend struct ArgsBuilder; + size_t argsTailSize() const { return 3 + (evtNum ? evtNum + 2 : 0); } +}; + +union ArgStorage { + void *p; + int64_t i; + ArgStorage() = default; + explicit ArgStorage(void *ptr) : p(ptr) {} + explicit ArgStorage(int64_t val) : i(val) {} +}; +// Storage::push_back must not invalidate references to existing elements. +template , + typename Storage = std::deque> +struct ArgsBuilder { + ArgsBuilder(Args &args, Storage &storage) : args(args), storage(storage) { + reset(); + } + // The value must remain alive and stable until the program returns. + template void append(T &value) { args.push_back(&value); } + // All the pointers must remain alive and stable until the program returns. + void appendMemref(void *&ptr, int64_t &offset, unsigned rank, int64_t *shape, + int64_t *strides) { + append(ptr); + append(ptr); + append(offset); + for (unsigned i = 0; i < rank; ++i) append(shape[i]); + for (unsigned i = 0; i < rank; ++i) append(strides[i]); + } + + void store(ArgStorage value) { + storage.push_back(value); + append(storage.back()); + } + void storeMemref(void *ptr, unsigned rank = 0, int64_t *shape = nullptr, + int64_t *strides = nullptr, int64_t offset = 0) { + assert(rank == 0 || (shape != nullptr && strides != nullptr)); + storeMemref( + ptr, rank, [&](unsigned i) { return shape[i]; }, + [&](unsigned i) { return strides[i]; }, offset); + } + template + void storeMemref(void *ptr, unsigned rank, GetDim getDim, GetStride getStride, + int64_t offset = 0) { + store(ArgStorage(ptr)); + args.push_back(args.back()); // Aligned pointer shares the allocated one. + store(ArgStorage(offset)); + for (unsigned i = 0; i < rank; ++i) store(ArgStorage(getDim(i))); + for (unsigned i = 0; i < rank; ++i) store(ArgStorage(getStride(i))); + } + + // Build the argument list for the program. The events, returned by the + // program (if any), are stored in the tail of the argument list. The list + // can be used to launch the same program multiple times, but only if it's + // not copied or modified and not concurrently. + Args &build(const Program &prog) { + auto pos = args.size(); + args.resize(pos + prog.argsTailSize()); + if (prog.eventNum()) { + auto data = args.data(); + data[pos + 3] = &data[pos + 4]; + data[pos + 4] = &data[pos + 5]; + } + return args; + } + + void reset() { + args.clear(); + storage.clear(); + } + +private: + Args &args; + Storage &storage; +}; +// Inline storage for N elements. On overflow, a Stable storage keeps the +// inline elements in place and appends the rest to the Heap, preserving +// references. Otherwise, all the elements move to the Heap, staying +// contiguous. +template , std::vector>> +struct SmallStorage { + void push_back(const T &value) { + if ((Stable || !heap) && sz < N) buf[sz++] = value; + else spill(2 * N).push_back(value); + } + T &back() { return heap && !heap->empty() ? heap->back() : buf[sz - 1]; } + void resize(size_t n) { + static_assert(!Stable, "Unsupported!"); + if (heap || n > N) return spill(n).resize(n); + if (n > sz) std::fill(buf + sz, buf + n, T{}); + sz = n; + } + T *data() { + static_assert(!Stable, "Unsupported!"); + return heap ? heap->data() : buf; + } + size_t size() const { return sz + (heap ? heap->size() : 0); } + void clear() { + sz = 0; + if (heap) heap->clear(); + } + +private: + T buf[N]; + std::optional heap; + size_t sz = 0; + + Heap &spill(size_t cap) { + if (heap) return *heap; + heap.emplace(); + if constexpr (!Stable) { + heap->reserve(std::max(cap, sz)); + heap->assign(buf, buf + sz); + sz = 0; + } + return *heap; + } +}; +template struct SmallArgs { + SmallStorage argsBuf; + SmallStorage valuesBuf; +}; +// SmallArgs is the first base to be constructed before ArgsBuilder binds it. +template +struct SmallBuilder final + : private SmallArgs, + ArgsBuilder, SmallStorage> { + using Base = + ArgsBuilder, SmallStorage>; + SmallBuilder() : Base(this->argsBuf, this->valuesBuf) {} + SmallBuilder(const SmallBuilder &) = delete; +}; +} // namespace gpu +} // namespace gc diff --git a/include/gc/Runtime/GpuRuntime.h b/include/gc/Runtime/GpuRuntime.h new file mode 100644 index 000000000..f96740680 --- /dev/null +++ b/include/gc/Runtime/GpuRuntime.h @@ -0,0 +1,220 @@ +//===-- GpuRuntime.h - GPU runtime interface --------------------*- C++ -*-===// +// +// This file is licensed under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace gc::gpu { + +struct DeviceInfo { + size_t id{0}; + std::string name; + std::string arch; + uint32_t maxWgSize{0}; + + DeviceInfo() = default; + DeviceInfo(size_t id) : id(id), arch(idToArch(id)) {} + + static const char *idToArch(int id) { + // Using device IDs from this source - + // https://github.com/intel/compute-runtime/blob/master/shared/source/dll/devices/devices_base.inl + switch (id) { + case 0x674C: + case 0x674D: + case 0x674E: + case 0x674F: + case 0x6750: + return "cri"; + case 0xE202: + case 0xE209: + case 0xE20B: + case 0xE20C: + case 0xE20D: + case 0xE210: + case 0xE211: + case 0xE212: + case 0xE215: + case 0xE216: + case 0xE220: + case 0xE221: + case 0xE222: + case 0xE223: + return "bmg"; + case 0x0BD0: + case 0x0BD5: + case 0x0BD6: + case 0x0BD7: + case 0x0BD8: + case 0x0BD9: + case 0x0BDA: + case 0x0BDB: + case 0x0B69: + case 0x0B6E: + case 0x0BD4: + return "pvc"; + default: + return ""; + } + } +}; + +enum class ArgType : uint8_t { + I8, + UI8, + I16, + UI16, + I32, + UI32, + I64, + UI64, + F32, + F64, + PTR, +}; + +constexpr uint8_t argSize(ArgType type) noexcept { + switch (type) { + case ArgType::I8: + case ArgType::UI8: + return sizeof(uint8_t); + case ArgType::I16: + case ArgType::UI16: + return sizeof(uint16_t); + case ArgType::I32: + case ArgType::UI32: + return sizeof(uint32_t); + case ArgType::I64: + case ArgType::UI64: + return sizeof(uint64_t); + case ArgType::F32: + return sizeof(float); + case ArgType::F64: + return sizeof(double); + case ArgType::PTR: + return sizeof(void *); + } + return 0; +} + +// Functions called by the generated code. +namespace sym { +#define GC_RT_SYM(name) inline constexpr char name##Name[] = #name +GC_RT_SYM(gcGpuCreateKernel); +GC_RT_SYM(gcGpuDestroyKernel); +GC_RT_SYM(gcGpuLaunch); +GC_RT_SYM(gcGpuAllocDevice); +GC_RT_SYM(gcGpuAllocShared); +GC_RT_SYM(gcGpuAllocHost); +GC_RT_SYM(gcGpuFree); +GC_RT_SYM(gcGpuMemcpy); +GC_RT_SYM(gcGpuWait); +#undef GC_RT_SYM +} // namespace sym + +using SymbolAddr = std::pair; +using RtSymbols = std::array; + +template struct GpuRuntime { + using Kernel = K; + using Event = E; + using Memory = M; + + GpuRuntime() = default; + virtual ~GpuRuntime() = default; + GpuRuntime(const GpuRuntime &) = delete; + GpuRuntime &operator=(const GpuRuntime &) = delete; + GpuRuntime(GpuRuntime &&) = delete; + GpuRuntime &operator=(GpuRuntime &&) = delete; + + virtual const DeviceInfo &getDeviceInfo() const = 0; + + // The caller is responsible for the kernel destruction before the runtime, + // via the static Rt::destroyKernel(Kernel *) of the implementation. + virtual Kernel *createKernel(const uint8_t *data, size_t size, + const char *name, uint32_t nArgs, + ArgType *argTypes) = 0; + // **args is an array of pointers to the kernel arguments. + // The caller is responsible for releasing the events by calling wait(). + virtual Event *launch(Kernel *kernel, void **args, uint32_t nArgs, + uint32_t gridX, uint32_t gridY, uint32_t gridZ, + uint32_t blockX, uint32_t blockY, uint32_t blockZ, + Event *const *deps, uint32_t nDeps) = 0; + virtual Memory *allocDevice(size_t size, size_t alignment) = 0; + virtual Memory *allocShared(size_t size, size_t alignment) = 0; + virtual Memory *allocHost(size_t size, size_t alignment) = 0; + virtual void free(Memory *ptr) = 0; + virtual Event *memcpy(const Memory *src, Memory *dst, size_t size, + Event *const *deps, uint32_t nDeps) = 0; + // The events ownership is transferred back to the runtime. + virtual void wait(Event *const *events, uint32_t nEvents) = 0; + + // The following symbols are registered in the ExecutionEngine. + template static const RtSymbols &symbols() { + static_assert(std::is_base_of_v); + auto entry = [](const char *name, auto *fn) { + return SymbolAddr{name, reinterpret_cast(fn)}; + }; + static const std::array syms{ + entry( + sym::gcGpuCreateKernelName, + +[](Rt *rt, const uint8_t *data, size_t size, const char *name, + uint32_t nArgs, ArgType *argTypes) { + return rt->createKernel(data, size, name, nArgs, argTypes); + }), + entry( + sym::gcGpuDestroyKernelName, + +[](Kernel *kernel) { Rt::destroyKernel(kernel); }), + entry( + sym::gcGpuLaunchName, + +[](Rt *rt, Kernel *kernel, void **args, uint32_t nArgs, + uint32_t gridX, uint32_t gridY, uint32_t gridZ, uint32_t blockX, + uint32_t blockY, uint32_t blockZ, Event *const *deps, + uint32_t nDeps) { + return rt->launch(kernel, args, nArgs, gridX, gridY, gridZ, + blockX, blockY, blockZ, deps, nDeps); + }), + entry( + sym::gcGpuAllocDeviceName, + +[](Rt *rt, size_t size, size_t alignment) { + return rt->allocDevice(size, alignment); + }), + entry( + sym::gcGpuAllocSharedName, + +[](Rt *rt, size_t size, size_t alignment) { + return rt->allocShared(size, alignment); + }), + entry( + sym::gcGpuAllocHostName, + +[](Rt *rt, size_t size, size_t alignment) { + return rt->allocHost(size, alignment); + }), + entry( + sym::gcGpuFreeName, +[](Rt *rt, Memory *ptr) { rt->free(ptr); }), + entry( + sym::gcGpuMemcpyName, + +[](Rt *rt, const Memory *src, Memory *dst, size_t size, + Event *const *deps, uint32_t nDeps) { + return rt->memcpy(src, dst, size, deps, nDeps); + }), + entry( + sym::gcGpuWaitName, + +[](Rt *rt, Event *const *events, uint32_t nEvents) { + rt->wait(events, nEvents); + }), + }; + return syms; + } +}; + +} // namespace gc::gpu diff --git a/include/gc/Runtime/ZeGpuRuntime.h b/include/gc/Runtime/ZeGpuRuntime.h new file mode 100644 index 000000000..8b2c976b1 --- /dev/null +++ b/include/gc/Runtime/ZeGpuRuntime.h @@ -0,0 +1,59 @@ +//===-- ZeGpuRuntime.h - Level Zero GpuRuntime ------------------*- C++ -*-===// +// +// This file is licensed under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include + +#include "gc/Runtime/GpuRuntime.h" + +// Level Zero handles, identical to the ze_api.h declarations. +struct _ze_event_handle_t; +typedef struct _ze_driver_handle_t *ze_driver_handle_t; +typedef struct _ze_device_handle_t *ze_device_handle_t; +typedef struct _ze_context_handle_t *ze_context_handle_t; + +namespace gc::gpu { +struct ZeKernel; + +class ZeGpuRuntime final + : public GpuRuntime { +public: + // Returns a platform-wide shared instance of the runtime or nullptr + // if no suitable device found. + static ZeGpuRuntime *get(); + static std::shared_ptr create(); + static std::shared_ptr create(ze_device_handle_t device, + ze_driver_handle_t driver, + ze_context_handle_t context); + ~ZeGpuRuntime() override; + + const DeviceInfo &getDeviceInfo() const override; + Kernel *createKernel(const uint8_t *data, size_t size, const char *name, + uint32_t nArgs, ArgType *argTypes) override; + static void destroyKernel(Kernel *kernel); + Event *launch(Kernel *kernel, void **args, uint32_t nArgs, uint32_t gridX, + uint32_t gridY, uint32_t gridZ, uint32_t blockX, + uint32_t blockY, uint32_t blockZ, Event *const *deps, + uint32_t nDeps) override; + void *allocDevice(size_t size, size_t alignment) override; + void *allocShared(size_t size, size_t alignment) override; + void *allocHost(size_t size, size_t alignment) override; + void free(void *ptr) override; + Event *memcpy(const void *src, void *dst, size_t size, Event *const *deps, + uint32_t nDeps) override; + void wait(Event *const *events, uint32_t nEvents) override; + +private: + struct Impl; + + explicit ZeGpuRuntime(std::unique_ptr impl); + + std::unique_ptr impl; +}; +} // namespace gc::gpu diff --git a/include/gc/Target/LLVM/XeVM/Target.h b/include/gc/Target/LLVM/XeVM/Target.h deleted file mode 100644 index c0ecafd88..000000000 --- a/include/gc/Target/LLVM/XeVM/Target.h +++ /dev/null @@ -1,30 +0,0 @@ -//===-- Target.h - MLIR XeVM target registration ----------------*- C++ -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// This provides registration calls for attaching the XeVM target interface. -// -//===----------------------------------------------------------------------===// - -#ifndef MLIR_TARGET_XEVM_TARGET_H -#define MLIR_TARGET_XEVM_TARGET_H - -namespace mlir { -class DialectRegistry; -class MLIRContext; -namespace xevm { -/// Registers the `TargetAttrInterface` for the `#xevm.target` attribute in -/// the given registry. -void registerXeVMTargetInterfaceExternalModels(DialectRegistry ®istry); - -/// Registers the `TargetAttrInterface` for the `#xevm.target` attribute in -/// the registry associated with the given context. -void registerXeVMTargetInterfaceExternalModels(MLIRContext &context); -} // namespace xevm -} // namespace mlir - -#endif // MLIR_TARGET_XEVM_TARGET_H diff --git a/include/gc/Target/LLVM/XeVM/Utils.h b/include/gc/Target/LLVM/XeVM/Utils.h deleted file mode 100644 index f1a39e48b..000000000 --- a/include/gc/Target/LLVM/XeVM/Utils.h +++ /dev/null @@ -1,39 +0,0 @@ -//===-- Utils.h - MLIR XeVM target utils ------------------------*- C++ -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// This files declares XeVM target related utility classes and functions. -// -//===----------------------------------------------------------------------===// - -#ifndef MLIR_TARGET_LLVM_XEVM_UTILS_H -#define MLIR_TARGET_LLVM_XEVM_UTILS_H - -#include "gc/Dialect/LLVMIR/XeVMDialect.h" -#include "mlir/Dialect/GPU/IR/CompilationInterfaces.h" -#include "mlir/Target/LLVM/ModuleToObject.h" - -namespace mlir { -namespace xevm { - -/// Base class for all XeVM serializations from GPU modules into binary strings. -/// By default this class serializes into LLVM bitcode. -class SerializeGPUModuleBase : public LLVM::ModuleToObject { -public: - SerializeGPUModuleBase(Operation &module, XeVMTargetAttr target, - const gpu::TargetOptions &targetOptions = {}); - - static void init(); - XeVMTargetAttr getTarget() const; - -protected: - XeVMTargetAttr target; -}; -} // namespace xevm -} // namespace mlir - -#endif // MLIR_TARGET_LLVM_XEVM_UTILS_H diff --git a/include/gc/Target/LLVMIR/Dialect/XeVM/XeVMToLLVMIRTranslation.h b/include/gc/Target/LLVMIR/Dialect/XeVM/XeVMToLLVMIRTranslation.h deleted file mode 100644 index b679656a4..000000000 --- a/include/gc/Target/LLVMIR/Dialect/XeVM/XeVMToLLVMIRTranslation.h +++ /dev/null @@ -1,31 +0,0 @@ -//===-- XeVMToLLVMIRTranslation.h - XeVM to LLVM IR -------------*- C++ -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// This provides registration calls for XeVM dialect to LLVM IR translation. -// -//===----------------------------------------------------------------------===// - -#ifndef MLIR_TARGET_LLVMIR_DIALECT_XEVM_XEVMTOLLVMIRTRANSLATION_H -#define MLIR_TARGET_LLVMIR_DIALECT_XEVM_XEVMTOLLVMIRTRANSLATION_H - -namespace mlir { - -class DialectRegistry; -class MLIRContext; - -/// Register the XeVM dialect and the translation from it to the LLVM IR in the -/// given registry; -void registerXeVMDialectTranslation(DialectRegistry ®istry); - -/// Register the XeVM dialect and the translation from it in the registry -/// associated with the given context. -void registerXeVMDialectTranslation(MLIRContext &context); - -} // namespace mlir - -#endif // MLIR_TARGET_LLVMIR_DIALECT_XEVM_XEVMTOLLVMIRTRANSLATION_H diff --git a/include/gc/Transforms/CMakeLists.txt b/include/gc/Transforms/CMakeLists.txt index b8ddfc622..9272cd406 100644 --- a/include/gc/Transforms/CMakeLists.txt +++ b/include/gc/Transforms/CMakeLists.txt @@ -1,18 +1,12 @@ -if(GC_ENABLE_DNNL_API) - list(APPEND TABLEGEN_MACROS -DGC_HAS_ONEDNN_DIALECT) -endif() -if(GC_ENABLE_GPU) - list(APPEND TABLEGEN_MACROS -DGC_USE_GPU) -endif() -if(GC_ENABLE_IMEX) - list(APPEND TABLEGEN_MACROS -DGC_USE_IMEX -DGC_USE_GPU) -endif() - set(LLVM_TARGET_DEFINITIONS Passes.td) -mlir_tablegen(Passes.h.inc -gen-pass-decls -name GraphCompiler ${TABLEGEN_MACROS}) +mlir_tablegen(Passes.h.inc -gen-pass-decls -name GraphCompiler + ${TABLEGEN_MACROS}) mlir_tablegen(Passes.capi.h.inc -gen-pass-capi-header --prefix GraphCompiler) mlir_tablegen(Passes.capi.cpp.inc -gen-pass-capi-impl --prefix GraphCompiler) add_public_tablegen_target(GraphCompilerPassIncGen) add_mlir_doc(Passes GraphCompilerPasses ./ -gen-pass-doc) -add_subdirectory(Microkernel) +set(LLVM_TARGET_DEFINITIONS TensorMaskingOpInterface.td) +mlir_tablegen(TensorMaskingOpInterface.h.inc -gen-op-interface-decls) +mlir_tablegen(TensorMaskingOpInterface.cpp.inc -gen-op-interface-defs) +add_public_tablegen_target(GcTensorMaskingOpInterfaceIncGen) diff --git a/include/gc/Transforms/GPU/AttentionOptimizations/Utils.h b/include/gc/Transforms/GPU/AttentionOptimizations/Utils.h new file mode 100644 index 000000000..0fb2a38d3 --- /dev/null +++ b/include/gc/Transforms/GPU/AttentionOptimizations/Utils.h @@ -0,0 +1,44 @@ +//===--- AttentionUtils.h - Common utilities for attention optimizations --===// +// +// This file is licensed under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef GC_TRANSFORMS_GPU_ATTENTIONOPTIMIZATIONS_UTILS_H +#define GC_TRANSFORMS_GPU_ATTENTIONOPTIMIZATIONS_UTILS_H + +#include "mlir/Dialect/SCF/IR/SCF.h" +#include "mlir/Dialect/XeGPU/IR/XeGPU.h" + +namespace mlir::gc::attention { + +/// Collect all xegpu.dpas operations in the loop body. +SmallVector collectDpasOps(scf::ForOp forOp); + +/// Recursively collect all operations inside `region` that `value` +/// transitively depends on. Returns them in topological order +/// (dependencies first). +void collectDepsInRegion(Value value, Region *region, + SmallVectorImpl &deps, + DenseSet &visited); + +/// Check if any operation in `deps` uses `value` as an operand. +bool usesValue(const SmallVectorImpl &deps, Value value); + +/// Compute sg_data = shape / sg_layout element-wise. +/// Returns failure if any dimension is not evenly divisible. +FailureOr> computeSgData(ArrayRef shape, + ArrayRef sgLayout); + +/// Build an xegpu::LayoutAttr from the given components. +/// Empty arrays for instData/order are treated as "not set" (nullptr). +xegpu::LayoutAttr makeLayout(MLIRContext *ctx, ArrayRef sgLayout, + ArrayRef sgData, + ArrayRef instData = {}, + ArrayRef order = {}); + +} // namespace mlir::gc::attention + +#endif // GC_TRANSFORMS_GPU_ATTENTIONOPTIMIZATIONS_UTILS_H diff --git a/include/gc/Transforms/MergeAllocTickBased.h b/include/gc/Transforms/MergeAllocTickBased.h deleted file mode 100644 index 522c483c9..000000000 --- a/include/gc/Transforms/MergeAllocTickBased.h +++ /dev/null @@ -1,187 +0,0 @@ -//===-- MergeAllocTickBased.h - Tick-based merge alloc interfac -*- C++ -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef GC_TRANSFORMS_MERGEALLOCTICKBASED_H -#define GC_TRANSFORMS_MERGEALLOCTICKBASED_H - -#include "mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h" -#include "mlir/IR/Builders.h" -#include "mlir/IR/Operation.h" -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/SmallPtrSet.h" -#include "llvm/ADT/SmallVector.h" -#include -#include "gc/Transforms/Passes.h" -#include "gc/Transforms/StaticMemoryPlanning.h" - -namespace mlir { -class BufferViewFlowAnalysis; -class ViewLikeOpInterface; -namespace gc { - -/// Usually ticks should be non-negative numbers. There are two special ticks -/// defined here. -namespace special_ticks { -/// the memref is not yet accessed -static constexpr int64_t NO_ACCESS = -1; -/// untraceable access happens on this memref, like func.return -static constexpr int64_t UNTRACEABLE_ACCESS = -2; -} // namespace special_ticks - -/// the collected tick [first, last] for a memref allocation -struct Tick { - /// The tick when the buffer is allocated. allocTick is only used to stablize - /// the sorting results of the buffers when ticks of different buffers are the - /// same - int64_t allocTick = special_ticks::NO_ACCESS; - int64_t firstAccess = special_ticks::NO_ACCESS; - int64_t lastAccess = special_ticks::NO_ACCESS; - - /// access the memref at the tick, will update firstAccess and lastAccess - /// based on the tick - void update(int64_t tick); -}; - -/// A complex scope object is addition info for a RegionBranchOpInterface or -/// LoopLikeOpInterface. It contains the scope itself, and the referenced alloc -/// ops inside this scope. It is used in TickCollecter and TickCollecterStates -/// internally to track which buffers this scope accesses. These buffers must -/// have overlapped lifetime -struct ComplexScope { - Operation *scope; - int64_t startTick; - llvm::SmallPtrSet operations; - ComplexScope(Operation *scope, int64_t startTick) - : scope{scope}, startTick{startTick} {} -}; - -/// the top-level collected lifetime trace for merge-alloc pass -struct TickTraceResult : public LifetimeTrace { - memoryplan::Traces traces; - Block *block; - Attribute memorySpace; - TickTraceResult(Block *block, Attribute memorySpace) - : LifetimeTrace{TK_TICK}, block{block}, memorySpace{memorySpace} {} - Block *getAllocScope() const override { return block; } - Attribute getMemorySpace() const override { return memorySpace; } - static bool classof(const LifetimeTrace *S) { - return S->getKind() == TK_TICK; - } -}; - -/// the internal states for TickCollecter analysis for a function -struct TickCollecterStates { - /// the alias analysis result for the function - const mlir::BufferViewFlowAnalysis &aliasAnaly; - const MergeAllocationOptions &opt; - /// the current tick for the current callback of walk(). It will be by default - /// incremented by 1 for each visited op - int64_t curTick = 0; - /// the currently collected AllocOp -> [start, end] map - llvm::DenseMap allocTicks; - /// the stack of ComplexScopes for the current visited position in the IR - llvm::SmallVector complexScopeStack; - TickCollecterStates(const mlir::BufferViewFlowAnalysis &aliasAnaly, - const MergeAllocationOptions &opt) - : aliasAnaly{aliasAnaly}, opt{opt} {} -}; - -/// the tick-based memory lifetime collector. This class overrides operator() so -/// that a TickCollecter object can be passed to a TraceCollectorFunc. This -/// collector assigns a monotonically increasing "tick" for each operation, by -/// Operaion::walk() order. It collects the first reference -/// tick and the last reference tick for each `memref.alloc` operation as the -/// lifetime trace stored in `TickTraceResult` -struct TickCollecter { - TickCollecter() = default; - /// called on each operation before calling onXXXX() below. If may call - /// onPopComplexScope internally when walk() runs out of a ComplexScope - virtual LogicalResult popScopeIfNecessary(TickCollecterStates *s, - Operation *op) const; - - virtual void forwardTick(TickCollecterStates *s) const; - - virtual void accessValue(TickCollecterStates *s, Value v, bool complex) const; - - virtual void onMemrefViews(TickCollecterStates *s, - ViewLikeOpInterface op) const; - - virtual void onReturnOp(TickCollecterStates *s, Operation *op) const; - - virtual void onAllocOp(TickCollecterStates *s, Operation *op) const; - - // called on Ops that extracts pointers from a memref - virtual void onExtractPointerOp(TickCollecterStates *s, Operation *op) const; - - virtual void onGeneralOp(TickCollecterStates *s, Operation *op) const; - - virtual void pushComplexScope(TickCollecterStates *s, Operation *op) const; - - /// called when walk() runs outside of the scope - LogicalResult onPopComplexScope(TickCollecterStates *s, - int64_t endTick) const; - - /// returns true of an allocation either is not defined in the scope, or the - /// allocation escapes from the scope - virtual bool needsResetTick(TickCollecterStates *s, Operation *scope, - Operation *allocation) const; - /// returns true if the memref.alloc op is "merge-able". If false is returned, - /// this memref.alloc will be untouched by merge-alloc - virtual bool isMergeableAlloc(TickCollecterStates *s, Operation *op, - int64_t tick) const; - - /// gets the "AllocScope" op of a memref.alloc op. The default implementation - /// finds the closest surrounding parent operation with - /// AutomaticAllocationScope trait, and is not scf.for - virtual Operation *getAllocScope(TickCollecterStates *s, Operation *op) const; - /// returns the allocation size in bytes of a memref.alloc op. May fail when - /// the shape is not static - virtual FailureOr getAllocSize(TickCollecterStates *s, - Operation *op) const; - // consolidate the traces for each buffer in each scope, recorded in - // TickCollecterStates. It is called after walk() in operator() - virtual FailureOr getTrace(TickCollecterStates *s) const; - - /// the top-level entry for the collector. It creates a TickCollecterStates - /// and applies Operaion::walk() on the root function Op. - /// And it finally calls getTrace() to collect the `TickTraceResult` for each - /// scope in MemoryTraceScopes. - virtual FailureOr - operator()(Operation *root, const mlir::BufferViewFlowAnalysis &aliasAnaly, - const MergeAllocationOptions &option) const; - - virtual ~TickCollecter() = default; -}; - -// the default merge-alloc IR mutator. It can be passes to a -// MemoryMergeMutatorFunc as argument of merge-alloc -struct MergeAllocDefaultMutator { - /// builds an memory alloc op at the scope with given size and alignment in - /// bytes - virtual Value buildAlloc(OpBuilder &build, Block *block, int64_t size, - int64_t alignment, Attribute memorySpace) const; - /// builds an memory view op for original memref.alloc op (origAllocOp) and - /// the merged single allocation (mergedAlloc) - virtual Value buildView(OpBuilder &build, Block *scope, - Operation *origAllocOp, Value mergedAlloc, - int64_t byteOffset) const; - virtual LogicalResult operator()(Operation *op, Block *scope, - const MemorySchedule &schedule, - const MergeAllocationOptions &o) const; - MergeAllocDefaultMutator() = default; - virtual ~MergeAllocDefaultMutator() = default; -}; - -FailureOr tickBasedPlanMemory(Operation *op, - const LifetimeTrace &tr, - const MergeAllocationOptions &o); - -} // namespace gc -} // namespace mlir - -#endif \ No newline at end of file diff --git a/include/gc/Transforms/Microkernel/BrgemmRuntimeUtils.h b/include/gc/Transforms/Microkernel/BrgemmRuntimeUtils.h deleted file mode 100644 index 0c92458ed..000000000 --- a/include/gc/Transforms/Microkernel/BrgemmRuntimeUtils.h +++ /dev/null @@ -1,53 +0,0 @@ -//===-- BrgemmRuntimeUtils.h - Utils for Brgemm Runtime ---------*- C++ -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef GC_MICROKERNEL_BRGEMMRUNTIMEUTILS_H -#define GC_MICROKERNEL_BRGEMMRUNTIMEUTILS_H - -#include "mlir/IR/Attributes.h" -#include "mlir/IR/PatternMatch.h" -#include "oneapi/dnnl/dnnl_types.h" - -namespace mlir::microkernel { - -// these strings contain symbols for BRGEMM interfaces used in mlir pass -static const std::string DNNL_BRGEMM_DISPATCH_NAME = "dnnl_brgemm_dispatch"; -static const std::string DNNL_BRGEMM_TILECFG_NAME = "dnnl_brgemm_tileconfig"; -static const std::string DNNL_BRGEMM_TILERELEASE_NAME = - "dnnl_brgemm_tilerelease"; -static const std::string DNNL_BRGEMM_EXECUTE_NAME = "dnnl_brgemm_execute"; - -static inline int64_t getDnnlDataTypeVal(RewriterBase &rewriter, - Attribute attr) { - auto context = rewriter.getContext(); - auto tattr = dyn_cast_or_null(attr); - assert(tattr); - if (tattr == TypeAttr::get(FloatType::getF32(context))) { - return static_cast(dnnl_f32); - } else if (tattr == TypeAttr::get(FloatType::getF64(context))) { - return static_cast(dnnl_f64); - } else if (tattr == TypeAttr::get(FloatType::getBF16(context))) { - return static_cast(dnnl_bf16); - } else if (tattr == TypeAttr::get(FloatType::getF16(context))) { - return static_cast(dnnl_f16); - } else if (tattr == TypeAttr::get( - IntegerType::get(context, 32, IntegerType::Signed))) { - return static_cast(dnnl_s32); - } else if (tattr == - TypeAttr::get(IntegerType::get(context, 8, IntegerType::Signed))) { - return static_cast(dnnl_s8); - } else if (tattr == TypeAttr::get(IntegerType::get(context, 8, - IntegerType::Unsigned))) { - return static_cast(dnnl_u8); - } - return static_cast(dnnl_data_type_undef); -} - -}; // namespace mlir::microkernel - -#endif // GC_MICROKERNEL_BRGEMMRUNTIMEUTILS_H diff --git a/include/gc/Transforms/Microkernel/CMakeLists.txt b/include/gc/Transforms/Microkernel/CMakeLists.txt deleted file mode 100644 index 2e345775c..000000000 --- a/include/gc/Transforms/Microkernel/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -set(LLVM_TARGET_DEFINITIONS MicrokernelPasses.td) -mlir_tablegen(MicrokernelPasses.h.inc --gen-pass-decls -name Microkernel) -mlir_tablegen(MicrokernelPasses.capi.h.inc -gen-pass-capi-header --prefix Microkernel) -mlir_tablegen(MicrokernelPasses.capi.cpp.inc -gen-pass-capi-impl --prefix Microkernel) -add_public_tablegen_target(MLIRMicrokernelPassesIncGen) -add_mlir_doc(MicrokernelPasses GraphCompilerMicrokernelPasses ./ -gen-pass-doc) diff --git a/include/gc/Transforms/Microkernel/MicrokernelPasses.h b/include/gc/Transforms/Microkernel/MicrokernelPasses.h deleted file mode 100644 index ee9da8a4e..000000000 --- a/include/gc/Transforms/Microkernel/MicrokernelPasses.h +++ /dev/null @@ -1,28 +0,0 @@ -//===- MicrokernelPasses.h - Graph Compiler microkerenl passes --*- C++ -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef GC_MICROKERNELPASSES_H -#define GC_MICROKERNELPASSES_H - -#include "gc/Dialect/Linalgx/LinalgxDialect.h" -#include "gc/Dialect/Microkernel/MicrokernelDialect.h" -#include "gc/Dialect/Microkernel/MicrokernelOps.h" -#include "mlir/Pass/Pass.h" -#include - -namespace mlir { -namespace microkernel { -#define GEN_PASS_DECL -#include "gc/Transforms/Microkernel/MicrokernelPasses.h.inc" - -#define GEN_PASS_REGISTRATION -#include "gc/Transforms/Microkernel/MicrokernelPasses.h.inc" -} // namespace microkernel -} // namespace mlir - -#endif // GC_MICROKERNELPASSES_H diff --git a/include/gc/Transforms/Microkernel/MicrokernelPasses.td b/include/gc/Transforms/Microkernel/MicrokernelPasses.td deleted file mode 100644 index 41a3fd070..000000000 --- a/include/gc/Transforms/Microkernel/MicrokernelPasses.td +++ /dev/null @@ -1,110 +0,0 @@ -//===-- MicrokernelPasses.td - microkernel passes ----------*- tablegen -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef GC_DIALECT_MICROKERNELPASSES -#define GC_DIALECT_MICROKERNELPASSES - -include "mlir/Pass/PassBase.td" - -def ConvertLinalgToMicrokernel: Pass<"convert-linalg-to-microkernel", "::mlir::func::FuncOp"> { - let summary = "Lower eligible linalg ops to microkernels"; - let description = [{ - Convert eligible linalg ops to microkernel dialects based on pattern matching. - For example: - ``` - scf.forall { - linalg.fill ins(...) outs(...) -> tensor<...> - linalg.batch_reduce_matmul ins(...) outs(...) -> tensor<...> - } - ``` - Will be changed into - ``` - scf.forall { - linalg.fill ins(...) outs(...) -> tensor<...> - microkernel.brgemm ins(...) outs(...) -> tensor<...> - } - ``` - }]; - let dependentDialects = ["func::FuncDialect", - "tensor::TensorDialect", - "memref::MemRefDialect", - "linalg::LinalgDialect", - "linalgx::LinalgxDialect", - "microkernel::MicrokernelDialect"]; -} - -def ExpandMicrokernel: Pass<"expand-microkernel", "::mlir::func::FuncOp"> { - let summary = "Expand abstract microkernels into detailed execution phases"; - let description = [{ - Expand abstract microkernels into detailed execution phases - For example: - ``` - scf.forall { - linalg.fill ins(...) outs(...) -> tensor<...> - microkernel.brgemm ins(...) outs(...) -> tensor<...> - } - ``` - Will be changed into - ``` - scf.forall { - linalg.fill ins(...) outs(...) -> tensor<...> - %0 = microkernel.brgemm.dispatch(...) - microkernel.brgemm.prologue(%0) - microkernel.brgemm.execute(%0, ...) - microkernel.brgemm.epilogue(%0) - } - ``` - }]; - let dependentDialects = ["func::FuncDialect", - "memref::MemRefDialect", - "microkernel::MicrokernelDialect"]; -} - -def ConvertMicrokernelToDnnlFunc: Pass<"convert-microkernel-to-dnnl-func", "::mlir::ModuleOp"> { - let summary = "Lower microkernel dialects to dnnl func call"; - let description = [{ - Convert microkernel dialects to runtime function call to oneDNN library. - }]; - let dependentDialects = ["func::FuncDialect", - "memref::MemRefDialect", - "LLVM::LLVMDialect", - "microkernel::MicrokernelDialect"]; -} - -def EarlyDispatchMicrokernel: Pass<"early-dispatch-microkernel", "::mlir::ModuleOp"> { - let summary = "Early dispatch microkernel during compile time"; - let description = [{ - Early dispatch microkernel during compile time. - }]; - let dependentDialects = ["func::FuncDialect", - "memref::MemRefDialect", - "LLVM::LLVMDialect", - "microkernel::MicrokernelDialect"]; -} - -def MergeBranchMicrokernelContext: Pass<"merge-branch-microkernel-context", "::mlir::ModuleOp"> { - let summary = "Find and merge identical microkernel context operations in branches into one"; - let description = [{ - Find and merge identical microkernel context operations in branches into one. - }]; - let dependentDialects = ["func::FuncDialect", - "memref::MemRefDialect"]; -} - -def MicrokernelInvariantCodeMotion: Pass<"microkernel-invariant-code-motion", "::mlir::ModuleOp"> { - let summary = "Hoist invariant microkernel code to avoid redundant execution"; - let description = [{ - Hoist invariant microkernel code to avoid redundant execution. - }]; - let dependentDialects = ["func::FuncDialect", - "memref::MemRefDialect", - "LLVM::LLVMDialect", - "microkernel::MicrokernelDialect"]; -} - -#endif // GC_DIALECT_MICROKERNELPASSES diff --git a/include/gc/Transforms/Passes.h b/include/gc/Transforms/Passes.h index 19a0def9a..0bb3cda16 100644 --- a/include/gc/Transforms/Passes.h +++ b/include/gc/Transforms/Passes.h @@ -1,4 +1,4 @@ -//===- Passes.h - Graph Compiler passes -------------------------*- C++ -*-===// +//===- Passes.h - Graph Compiler passes ----------------------*- C++ -*-===// // // This file is licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -9,135 +9,51 @@ #ifndef GC_PASSES_H #define GC_PASSES_H -#include "mlir/Pass/Pass.h" -#include "mlir/Pass/PassOptions.h" +#include +#include namespace mlir { class OpBuilder; -class SymbolTable; class ModuleOp; namespace func { class FuncOp; -} // namespace func - -namespace LLVM { -class LLVMDialect; -} - -namespace scf { -class SCFDialect; -} - -namespace openmp { -class OpenMPDialect; -} - -namespace linalg { -class LinalgDialect; -} -namespace linalgx { -class LinalgxDialect; -} - -namespace MemRef { -class MemRefDialect; -} - -class PassManager; -namespace xegpu { -class XeGPUDialect; } class OpPassManager; -class BufferViewFlowAnalysis; namespace gc { - -/// abstract base class for lifetime of buffers in the same "allocation scope". -/// It should hold the lifetime informantion of buffers that are to be merged in -/// the same allocation in an "allocation scope". TraceCollectorFunc decides -/// which buffers are put into which "allocation scope". -class LifetimeTrace { -public: - enum TraceKind { TK_TICK }; - virtual ~LifetimeTrace() = default; - LifetimeTrace(TraceKind kind) : kind{kind} {} - TraceKind getKind() const { return kind; } - virtual Block *getAllocScope() const = 0; - virtual Attribute getMemorySpace() const = 0; - -private: - TraceKind kind; -}; - -/// top level memory trace info for multiple scopes. Each element of scopeTraces -/// should contain an "allocation scope" and the implementation-defined lifetime -/// data -struct MemoryTraceScopes { - llvm::SmallVector> scopeTraces; - MemoryTraceScopes() = default; -}; - -/// the memory scheduling result for allocations in the same allocation scope. -/// allocation => offset map. All Operation* in the map should be -/// memref::AllocOp which are in the same LifetimeTrace. -struct MemorySchedule { - size_t totalSize; - Attribute memorySpace; - llvm::DenseMap allocToOffset; - MemorySchedule() : totalSize{0} {} -}; - -struct MergeAllocationOptions; -using TraceCollectorFunc = std::function( - Operation *, const BufferViewFlowAnalysis &, - const MergeAllocationOptions &)>; -using MemoryPlannerFunc = std::function( - Operation *, const LifetimeTrace &, const MergeAllocationOptions &)>; -using MemoryMergeMutatorFunc = std::function; - -struct MergeAllocationOptions { - bool checkOnly = false; - std::string plannerOptions; - int64_t alignment = 64; - TraceCollectorFunc tracer; - MemoryPlannerFunc planner; - MemoryMergeMutatorFunc mutator; -}; - -/// Creates an operation pass to merge the local memref allocations -std::unique_ptr createMergeAllocPass(const MergeAllocationOptions &o); -std::unique_ptr createMergeAllocPass(); - -void populateFrontendPasses(mlir::OpPassManager &); -void populateCPUPipeline(mlir::OpPassManager &); - +struct GpuDevicePropsOptions; struct GPUPipelineOptions : PassPipelineOptions { - Option isUsmArgs{ - *this, "is-usm-args", - llvm::cl::desc("Whether to use USM(unified shared memory) func args, in " - "which the host and device could access the same buffer " - "and there is no need to add memcpy explicitly."), + Option truncate{ + *this, "truncate-bin", + llvm::cl::desc("Truncate large binary blobs in the dumps."), llvm::cl::init(true)}; - Option useGpuRuntime{ - *this, "use-gpu-ocl", - llvm::cl::desc("Use the GpuToGpuOcl path, that converts the GPU " - "operations to GpuOclRuntime calls."), - llvm::cl::init(true)}; - Option callFinish{ - *this, "call-finish", - llvm::cl::desc( - "Call finish() after each GPU kernel launch. This option is passed " - "to the GpuToGpuOcl path, if use-gpu-ocl is true."), + Option enableAttentionPrefetch{ + *this, "enable-attention-prefetch", + llvm::cl::desc("Enable the SetAttentionPrefetch pass."), llvm::cl::init(false)}; + Option igcCmdOptions{ + *this, "igc-cmd-options", + llvm::cl::desc("Commandline options to pass to IGC compiler."), + llvm::cl::init("")}; + Option wait{ + *this, "wait", + llvm::cl::desc("Wait for all the GPU events at the end of the main " + "function instead of returning them to the caller."), + llvm::cl::init(true)}; + Option dump{*this, "dump", + llvm::cl::desc("Dump the IR after each phase."), + llvm::cl::init(false)}; + llvm::raw_ostream *dumpStream = nullptr; + const GpuDevicePropsOptions *deviceProps = nullptr; }; + +DialectRegistry &getDialectRegistry(); + void populateGPUPipeline(mlir::OpPassManager &, const GPUPipelineOptions &); -#ifdef GC_USE_IMEX -void populateIMEXPipeline(mlir::OpPassManager &, const GPUPipelineOptions &); -#endif + +void registerGPUPipeline(); #define GEN_PASS_DECL #include "gc/Transforms/Passes.h.inc" diff --git a/include/gc/Transforms/Passes.td b/include/gc/Transforms/Passes.td index 6d23cee80..190635e5e 100644 --- a/include/gc/Transforms/Passes.td +++ b/include/gc/Transforms/Passes.td @@ -1,4 +1,4 @@ -//===- Passes.td - Graph Compiler passes -------------------*- tablegen -*-===// +//===- Passes.td - Graph Compiler passes ----------------*- tablegen -*-===// // // This file is licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -11,103 +11,53 @@ include "mlir/Pass/PassBase.td" -def MergeAlloc : Pass<"gc-merge-alloc", "func::FuncOp"> { - let summary = "Merge multiple memref.alloc and reuse the buffer"; +def GpuDeviceProps : Pass<"gpu-dev-props", "ModuleOp"> { + let summary = "Set GPU device properties."; let description = [{ - The pass merges the "mergeable" memref.alloc allocations into a single - memref.alloc in its ancestor "allocation scope", to enhance memory - reuse and cache locality. A memref.alloc is "mergeable" if it is owned - by the current function and it is statically shaped and has identity layout. - An "allocation scope" is the nearest ancestor surrounding operation - of memref.alloc, which extends AutomaticAllocationScope trait and is not - scf.for. The function top-level block or the body of parallel-loop are - examples of "allocation scope". If there are nested AutomaticAllocationScope, - each level of the AutomaticAllocationScope is a different "allocation scope". - A "mergeable" memref.alloc will be replaced by a memref.view on the "merged" - buffer, with an offset. The "merged" buffer will be located at the begining - of the block of the "allocation scope". - The offset of each merged buffer is decided by this pass, by considering the - lifetime of the original memref before merging. This pass schedules the - offsets to 1) make sure the offsets and address ranges do not overlap if - two "mergeable" allocations have overlapped lifetime, and 2) reuse the - address ranges that are considered "hot" in cache for an later allocation. + Set module wide GPU device attributes. }]; let options = [ - Option<"optionAnalysisOnly", "analysis-only", "bool", - /*default=*/"false", - "Skip the mutation of the IR and only mark the lifetime and scope on the" - " attr of operations. Useful for debugging and testing.">, - Option<"plannerOptions", "planner-options", "std::string", - /*default=*/"\"\"", - "The options for the memory-planner. `cost-model` for using a cost-" - "model considering both cache locality and memory size. `size-first`" - " may generate allocations with smaller total size without considering" - " cache locality. By default `cost-model` is used.">, - Option<"optionAlignment", "alignment", "int64_t", - /*default=*/"64", - "The alignment of the merged allocations">, - ]; - let dependentDialects = ["memref::MemRefDialect", "arith::ArithDialect"]; - let constructor = "mlir::gc::createMergeAllocPass()"; + Option<"id", "id", "size_t", + /*default=*/"0", "Device id.">, + Option<"name", "name", "std::string", + /*default=*/"", "Device name.">, + Option<"arch", "arch", "std::string", + /*default=*/"\"bmg\"", "GPU architecture.">, + Option<"maxWgSize", "max-wg-size", "size_t", + /*default=*/"1024", "Maximum workgroup size.">, + ListOption<"sgSizes", "sg-sizes", "size_t", + "Subgroup sizes supported by the device."> + ]; } - -#ifdef GC_HAS_ONEDNN_DIALECT -def ConvertOneDNNGraphToLinalg : Pass<"convert-onednn-graph-to-linalg"> { - let summary = - "Lower the operations from the oneDNN Graph dialect into Linalg"; - let description = [{Lowers the `onednn_graph` ops to `linalg` ops.}]; - let dependentDialects = [ - "func::FuncDialect", "math::MathDialect", "arith::ArithDialect", - "tensor::TensorDialect", "linalg::LinalgDialect", "linalgx::LinalgxDialect" - ]; +def TileContraction : Pass<"tile-contract", "func::FuncOp"> { + let summary = "Tile contraction operations."; + let description = [{ + Tile contraction operations and fuse their producers. + }]; } -#endif -def ConvertMemRefToCPURuntime : Pass<"convert-memref-to-cpuruntime", "func::FuncOp"> { - let summary = "Lower the allocation / deallocation operations from the MemRef dialect into CPURuntime"; +def TileGroupedMatmul : Pass<"tile-grouped-matmul", "func::FuncOp"> { + let summary = "Tile grouped matmul operations."; let description = [{ - Lowers the `memref` allocation / deallocation ops to `CPURuntime` ops. + Tile 'linalgx.grouped_matmul' ops (work-group level) and the contractions + they decompose into (sub-group level). Must run after 'tile-contract'. }]; - let dependentDialects = [ - "memref::MemRefDialect", - "cpuruntime::CPURuntimeDialect" - ]; } -#ifdef GC_USE_IMEX -def LinalgToXeGPU : Pass<"linalg-to-xegpu", "func::FuncOp"> { - let summary = "Convert linalg dialect to XeGPU dialect."; - let description = [{Lower linalg ops to XeGPU dialect.}]; - let dependentDialects = [ - "linalg::LinalgDialect", "gpu::GPUDialect", "xegpu::XeGPUDialect", - "scf::SCFDialect", "memref::MemRefDialect", "arith::ArithDialect", - "math::MathDialect", "vector::VectorDialect" - ]; - let options = [ - Option<"kTile", "k-tile", "int64_t", - /*default=*/"32", "GEMM tile size for reduction dimension.">, - Option<"stages", "stages", "int64_t", - /*default=*/"1", "Number of cooperative prefetch stages.">, - ListOption<"dpasTile", "dpas-tile", "int64_t", - "DPAS register block sizes MxNxK">, - ]; +def TileAttention : Pass<"tile-attention", "func::FuncOp"> { + let summary = "Tile attention operations."; + let description = [{ + Tile 'linalgx.attention' ops. + }]; } -#endif -#ifdef GC_USE_GPU -def GpuToGpuOcl : Pass<"gpu-to-gpuocl", "ModuleOp"> { - let summary = "Convert the GPU operations to GpuOclRuntime calls."; +def TileParallel : Pass<"tile-parallel", "func::FuncOp"> { + let summary = "Tile parallel operations."; let description = [{ - Convert the gpu alloc, dealloc, memcpy and launch operations to GpuOclRuntime calls. + Tile parallel operations and fuse their producers. }]; - let options = [ - Option<"callFinish", "call-finish", "bool", - /*default=*/"false", - "Call finish() after each kernel launch."> - ]; } -#endif // GC_USE_GPU def AddContextArg : Pass<"add-ctx-arg", "func::FuncOp"> { let summary = "Add a context argument."; @@ -116,187 +66,205 @@ def AddContextArg : Pass<"add-ctx-arg", "func::FuncOp"> { }]; } -def AllocsToSLM : Pass<"allocs-to-slm", "func::FuncOp"> { - let summary = "Add 'shared' memory space to memrefs allocated inside a gpu.block."; - let description = [{Add 'shared' memory space to memrefs allocated inside a gpu.block.}]; - let dependentDialects = [ - "gpu::GPUDialect", "memref::MemRefDialect" - ]; +def Decomposition : Pass<"decomposition", "func::FuncOp"> { + let summary = "Decompose complex operations."; + let description = [{ + Decompose complex operations into simpler ones for easier optimization and code generation. + }]; } +def SetGpuFastMath : Pass<"set-gpu-fast-math"> { + let summary = "Set fastmath attribute on math.exp ops in GPU modules."; + let description = [{ + Walks GPU modules and sets the fastmath=fast attribute on all math.exp + operations. + }]; +} -def GpuTilingAndFusion : Pass<"gpu-tiling", "func::FuncOp"> { - let summary = "GPU tiling and fusion path."; +def SetAttentionLayouts : Pass<"set-attention-layouts"> { + let summary = "Set XeGPU layouts on attention-pattern dpas and store ops."; let description = [{ - This pass tiles linalg operations and creates two nested scf.forall loops. When converting to gpu.launch, - the inner loop is mapped to the block sizes and the outer - to grid sizes. The tiles calculation is based - on the GPU device properties, retrieved from the DLTI attributes. If the DLTI attributes are not specified, - defaults to the pass options. + Walks the module to find scf.for loops containing + at least two xegpu.dpas ops, and sets layout attributes on those dpas ops + and on any downstream xegpu.store_nd ops. }]; - let options = [ - Option<"numEus", "num-eus", "size_t", - /*default=*/"448", - "Number of Execution Units.">, - Option<"numEusPerSlice", "num-eus-per-slice", "size_t", - /*default=*/"8", - "Number of Execution Units per slice.">, - Option<"numThreadsPerEu", "num-threads-per-eu", "size_t", - /*default=*/"8", - "Number of threads per Execution Unit.">, - Option<"localMemSize", "local-mem-size", "size_t", - /*default=*/"131072", - "The size of the local memory, shared across a work-group.">, - Option<"vectorWidth", "vector-width", "size_t", - /*default=*/"512", - "The maximum width of EU's vector registers.">, - Option<"workGroupSize", "work-group-size", "size_t", - /*default=*/"64", - "The maximum workgroup size."> - ]; } -def GpuXeVMAttachTarget: Pass<"xevm-attach-target", ""> { - let summary = "Attaches a XeVM target attribute to a GPU Module."; +def SetAttentionPrefetch : Pass<"set-attention-prefetch"> { + let summary = "Set XeGPU prefetches on attention-pattern dpas and store ops."; let description = [{ - This pass searches for all GPU Modules in the immediate regions and attaches - a XeVM target if the module matches the name specified by the `module` argument. - - Example: - ``` - // File: in.mlir: - gpu.module @nvvm_module_1 {...} - gpu.module @rocdl_module_2 {...} - gpu.module @xevm_module_3 {...} - // mlir-opt --xevm-attach-target="module=xevm.* chip=pvc" in.mlir - gpu.module @nvvm_module_1 {...} - gpu.module @rocdl_module_2 {...} - gpu.module @xevm_module_3 [#xevm.target] {...} - ``` + Walks the module to find scf.for loops containing + at least two xegpu.dpas ops, and sets prefetch attributes on those dpas ops + and on any downstream xegpu.store_nd ops. }]; - let options = [ - Option<"moduleMatcher", "module", "std::string", - /*default=*/ [{""}], - "Regex used to identify the modules to attach the target to.">, - Option<"triple", "triple", "std::string", - /*default=*/ "\"spirv64-unknown-unknown\"", - "Target triple.">, - Option<"chip", "chip", "std::string", - /*default=*/"\"pvc\"", - "Target chip.">, - Option<"optLevel", "O", "unsigned", - /*default=*/"2", - "Optimization level."> - ]; } +def HoistAttentionVLoad : Pass<"hoist-attention-v-load"> { + let summary = "Hoist V-load to the beginning of flash-attention loop."; + let description = [{ + Experimental pass that matches flash-attention patterns at the XeGPU level + and moves the V-load (xegpu.load_nd before the second xegpu.dpas) to right + after the K-load at the beginning of the scf.for loop. + }]; +} -def IterativeTilingAndFusion : Pass<"iterative-tiling-and-fusion", - "func::FuncOp"> { - let summary = "Iterative tiling and fusion for any tilable operation"; +def Vectorize : Pass<"vectorize", "func::FuncOp"> { + let summary = "Vectorize structured operations."; let description = [{ - The pass tries to fuse any MLIR operation which can be tiled. Moreover, this pass aims to support: - 1. Matmul fusion with element-wise/reduce/broadcast ops. - 2. Producer and consumer fusion. - 3. Arbitrary topology, including residual pattern with multiple consumers . - 4. Nest loops structure with multiple level candidates. - 5. Flexible option to control the boundary of iterative process. - 6. Default tiling when no op is tiled before fusion. - 7. Cost-model to determine whether to fuse or not. + Vectorize structured operations that have vectorization support and apply + vector transformation patterns. }]; - let dependentDialects = ["func::FuncDialect", "linalg::LinalgDialect", "scf::SCFDialect", - "tensor::TensorDialect"]; +} - let options = [ - Option<"useCostModel", "use-cost-model", "bool", - /*default=*/"false", - "Decide if enable cost model to control iterative fusion.">, - Option<"defaultNDTile", "default-nd-tile", "unsigned", - /*default=*/"2", - "Set default amount of non-one dimensions in TileSize, such as 1, 2[default, a.k.a. 2D-Tile], etc.">, - ListOption<"defaultTileSize", "default-tile-size", "std::string", - "Set default TileSize for the certain type of op, saying `matmul:{32,32}`.">, - ]; +def DropUnitDims : Pass<"drop-unit-dims", "func::FuncOp"> { + let summary = "Drop unit dims."; + let description = [{ + Drop unit dims to allow some operations, like vector transfers, be properly lowered to XeGPU. + }]; + let dependentDialects = ["memref::MemRefDialect", "vector::VectorDialect"]; } -def DeepTileContractionOp - : Pass<"deep-tile-contraction-op", "func::FuncOp"> { - let summary = "Tile linalg contraction operation deeply"; - let description = - [{The pass tries to tile the linalg contraction op deeply.}]; - let dependentDialects = [ - "func::FuncDialect", - "arith::ArithDialect", - "tensor::TensorDialect", - "linalg::LinalgDialect", - "linalgx::LinalgxDialect", - ]; + +def RemoveAllocs : Pass<"remove-allocs", "func::FuncOp"> { + let summary = "Remove unnecessry allocs."; + let description = [{ + Remove unnecessry memref.alloc operations. + }]; + let dependentDialects = ["gpu::GPUDialect", "memref::MemRefDialect", + "scf::SCFDialect"]; } -def VerifyTargetDescription : Pass<"verify-target-description", "ModuleOp"> { - let summary = "Verify the target description from ModuleOp DLTI attribute."; +def TensorReshapeToLinalg : Pass<"tensor-reshape-to-linalg", "func::FuncOp"> { + let summary = "Convert tensor.expand_shape/collapse_shape to linalg ops, if possible."; let description = [{ - Verify the target description from ModuleOp DLTI attribute. Raise error for unexpected input(such as a negative number of num_threads), and raise warn for missing fields, and provide a default value(such as 32K for L1_cache_size). + Replaces tensor.expand_shape and tensor.collapse_shape with linalg.pack + and linalg.unpack or linalg.broadcast, if possible. }]; - let dependentDialects = ["DLTIDialect"]; - let options = [ - Option<"device", "device", "std::string", - /*default=*/"\"CPU\"", - "The device to verify. Supported device: CPU, ">, - ]; } -def DecomposeAggregatedOps : Pass<"decompose-aggregated-ops", "func::FuncOp"> { - let summary = "Decompose aggregated operations."; +def TensorConcatToLinalg : Pass<"tensor-concat-to-linalg", "func::FuncOp"> { + let summary = "Convert tensor.concat to linalg operations."; let description = [{ - Decompose operations that implement the `AggregatedOpInterface`. + Eliminate tensor.concat by routing producers into output subviews + or convert to linalg.copy. }]; } -def SinkOpIntoInnerLoop : Pass<"sink-op-into-inner-loop"> { - let summary = "Sink operations into inner loops"; - let description = [{The pass tries to sink operations into inner loops as deep as possible to maximize the chance for outer loop optimization. +def SetLayouts : Pass<"set-layouts"> { + let summary = "Set XeGPU layouts."; + let description = [{ + Sets layout attributes on anchor xegpu.* ops. + }]; +} + +def HoistForLoopTransferRead : Pass<"hoist-for-loop-transfer-read", "func::FuncOp"> { + let summary = "Hoist transfer read/write operations out of for loops."; + let description = [{ + Hoist transfer read/write operations out of for loops to improve memory access patterns. }]; - let dependentDialects = []; } -def MergeNestedForall : Pass<"merge-nested-forall"> { - let summary = "Merge nested scf.forall operations"; - let description = [{The pass tries to merge nested forall operations.}]; +def GpuKernelOutline : Pass<"gpu-kernel-outline", "ModuleOp"> { + let summary = "Outline GPU kernels."; + let description = [{ + Convert scf.forall operations into GPU kernels and attach the xevm target + to the outlined kernel modules. The target chip is taken from the device + attributes, see the GpuDeviceProps pass. + }]; + let dependentDialects = ["scf::SCFDialect", "gpu::GPUDialect", "xevm::XeVMDialect"]; + let options = [ + Option<"cmdOptions", "cmd-options", "std::string", + /*default=*/[{""}], + "Command line options to pass to the downstream compiler. These are " + "stored as the flags of the xevm target."> + ]; +} + +def SetKernelDepends : Pass<"set-kernel-depends", "func::FuncOp"> { + let summary = "Set dependencies between GPU kernels."; + let description = [{ + Collect dependencies between scf.forall operations annotated as GPU kernels. + }]; let dependentDialects = ["scf::SCFDialect"]; } -def FoldTensorOperation : Pass<"fold-tensor-operation"> { - let summary = "Fold some tensor operation"; +def MemrefCopyToGpu : Pass<"memref-copy-to-gpu", "ModuleOp"> { + let summary = "Convert memref.copy to gpu.memcpy."; + let dependentDialects = ["scf::SCFDialect", "gpu::GPUDialect", + "memref::MemRefDialect", "arith::ArithDialect"]; +} + +def GpuToGcGpu : Pass<"gpu-to-gc-gpu", "ModuleOp"> { + let summary = "Convert the GPU operations to GpuRuntime calls."; let description = [{ - Remove some useless tensor operations. + Convert the GPU alloc, dealloc, memcpy and launch operations to GpuRuntime calls. + It changes the function name to `main` and adds additional function arguments for + the GpuRuntime*, the external dependencies (Event* and uint32_t) and, if any + events are returned, an Event** array. The runtime and the dependencies are + passed into the function by the caller and the events, that were not consumed by + GpuRuntime::wait(), are returned back to the caller via the array. The new + function `eventNum()`, added by the Pass, returns the number of unconsumed events. + If it's > 0, the last argument of the main function points to an array of + `eventNum()` Event* elements, where the function stores the events. + }]; + let options = [ + Option<"wait", "wait", "bool", + /*default=*/"true", + "Wait for all the events at the end of the main function instead of " + "returning them to the caller."> + ]; +} + +def PeeledForToIf : Pass<"peeled-for-to-if", "func::FuncOp"> { + let summary = "Convert at-most-one-iteration peeled scf.for loops to scf.if."; + let description = [{ + After loop peeling, the remainder loop has at most one iteration (trip count + is 0 or 1). This pass detects such loops by matching the affine.apply + pattern produced by peeling (lb = ub - (ub - orig_lb) % step) and converts + them to scf.if, eliminating loop control-flow overhead on GPU. + }]; +} + +def FoldPadRoundtrip : Pass<"fold-pad-roundtrip", "func::FuncOp"> { + let summary = "Fold pad(extract_slice(src)) → src and pad(empty) → empty."; + let description = [{ + Folds two patterns that arise after padding and hoisting: + + Pattern 1 — roundtrip elimination: + %val = tensor.extract_slice %src [0,...][d,...][1,...] + %padded = tensor.pad %val low[0,...] high[S-d,...] → tensor + Folded to %src (valid when padded positions are not observed). + + Pattern 2 — padding of uninitialized tensor: + %empty = tensor.empty(%d) : tensor + %padded = tensor.pad %empty ... → tensor + Folded to tensor.empty() : tensor. }]; - let dependentDialects = [ - "tensor::TensorDialect", - ]; } -def DecomposeTensorOperation : Pass<"decompose-tensor-operation"> { - let summary = "decompose some tensor operation"; +def HoistStaticForInit : Pass<"hoist-static-for-init", "func::FuncOp"> { + let summary = "Hoist extract_slice out of scf.for/scf.if to make results static."; let description = [{ - Decompose some tensor operations(concat, gather) into linalg operation. + Hoists tensor.extract_slice operations out of region-based control flow + (scf.for, scf.if) so that iter_args/results carry static types. + + For scf.for: recognizes loops where iter_args are initialized by + linalg.fill(tensor.empty) or by extract_slice from a static tensor, + padded to static shape inside the body, and sliced back at the yield. + Transforms to carry static iter_args with a single extract_slice after. + + For scf.if: when all yield paths produce extract_slice from the same + static type (zero offsets, unit strides), rewrites the if to yield + static values and places a single extract_slice after the if. }]; - let dependentDialects = [ - "::mlir::tensor::TensorDialect" - ]; } -def LowerToTileVector : Pass<"lower-to-tile-vector", "func::FuncOp"> { - let summary = "Lower tensor to tile (virtual) vector"; +def ApplyPaddingLevel : Pass<"apply-padding-level", "func::FuncOp"> { + let summary = "Pad operations to align iteration domains to tile sizes."; let description = [{ - Lower operation operate on tensor to vector operation. + Pads operations implementing TensorMaskingOpInterface so that their + iteration dimensions become multiples of the tile sizes specified in + gc.tiling.wg_tile_sizes. }]; - let dependentDialects = [ - "::mlir::func::FuncDialect", - "::mlir::math::MathDialect", - "::mlir::arith::ArithDialect", - "::mlir::tensor::TensorDialect", - "::mlir::linalg::LinalgDialect", - "::mlir::vector::VectorDialect", - ]; } #endif // GC_DIALECT_GC_PASSES diff --git a/include/gc/Transforms/StaticMemoryPlanning.h b/include/gc/Transforms/StaticMemoryPlanning.h deleted file mode 100644 index 60fc3a35c..000000000 --- a/include/gc/Transforms/StaticMemoryPlanning.h +++ /dev/null @@ -1,71 +0,0 @@ -//===- StaticMemoryPlanning.h - Static memory planning ----------*- C++ -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef GC_TRANSFORMS_STATICMEMORYPLANNING_H -#define GC_TRANSFORMS_STATICMEMORYPLANNING_H - -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/SmallVector.h" -#include -#include -#include -#include -#include - -namespace mlir { -class Operation; -namespace gc { - -namespace memoryplan { -enum class InplaceKind { - ZERO_OFFSET, // this requires that the tensor share the same base - // pointer of the replaced tensor - FREE, // the tensor can freely choose any offset on this tensor -}; - -struct MemoryTrace { - // unique id of a buffer - uintptr_t bufferId; - // if > 0, size of the buffer allocation, if = 0, it is a deallocation trace - std::size_t size; - MemoryTrace(uintptr_t bufferId = 0, std::size_t size = 0) - : bufferId{bufferId}, size{size} {} -}; - -using Traces = llvm::SmallVector; -using InplaceInfo = std::pair; - -using InplaceInfoMap = - llvm::DenseMap>; - -/** - * Given a list of memory buffer alloc and free traces, try to use a large - * buffer to hold all allocated memory, and statically allocate each memory - * buffer from the large buffer for better memory reuse. - * @param traces the list of memory alloc and free traces, sorted by event time. - * @param alignment the alignment in number of elements - * @param hotFirst use the hot buffer first, instead of using best fit in size - * @param inplaceMap the map from the tensor to alloc into the candidate - * tensors that can be inplace reused for it. - * @param outSchedule the output schedule for each buffer: the location that - * the buffer should be in the large buffer (as an offset in number of elements) - * @param outInplaceSelection the output buffer id -> inplace buffer it reuses - * @return the size of the large buffer, in number of elements - * */ -std::size_t scheduleMemoryAllocations( - const Traces &traces, std::size_t alignment, bool hotFirst, - const InplaceInfoMap &inplaceMap, - std::unordered_map &outSchedule, - std::unordered_map> &outInplaceSelection); - -} // namespace memoryplan - -} // namespace gc -} // namespace mlir - -#endif diff --git a/include/gc/Transforms/TensorMaskingOpInterface.h b/include/gc/Transforms/TensorMaskingOpInterface.h new file mode 100644 index 000000000..5eddbd645 --- /dev/null +++ b/include/gc/Transforms/TensorMaskingOpInterface.h @@ -0,0 +1,26 @@ +//===- TensorMaskingOpInterface.h --------------------------------*- C++ +//-*-===// +// +// This file is licensed under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef GC_TRANSFORMS_TENSOR_MASKING_OP_INTERFACE_H +#define GC_TRANSFORMS_TENSOR_MASKING_OP_INTERFACE_H + +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/PatternMatch.h" + +// clang-format off +#include "gc/Transforms/TensorMaskingOpInterface.h.inc" +// clang-format on + +namespace mlir::gc { + +void registerTensorMaskingOpInterfaceForLinalg(DialectRegistry ®istry); + +} // namespace mlir::gc + +#endif // GC_TRANSFORMS_TENSOR_MASKING_OP_INTERFACE_H diff --git a/include/gc/Transforms/TensorMaskingOpInterface.td b/include/gc/Transforms/TensorMaskingOpInterface.td new file mode 100644 index 000000000..db90104c9 --- /dev/null +++ b/include/gc/Transforms/TensorMaskingOpInterface.td @@ -0,0 +1,40 @@ +//===- TensorMaskingOpInterface.td --------------------------------*- tablegen -*-===// +// +// This file is licensed under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef GC_TRANSFORMS_TENSOR_MASKING_OP_INTERFACE +#define GC_TRANSFORMS_TENSOR_MASKING_OP_INTERFACE + +include "mlir/IR/OpBase.td" + +def TensorMaskingOpInterface : OpInterface<"TensorMaskingOpInterface"> { + let cppNamespace = "::mlir::gc"; + + let description = [{ + Interface for operations that can produce a padded + masked version of + themselves. The `getMaskedImplementation` method pads operands to multiples + of the given sizes and inserts masking logic to preserve correctness. + }]; + + let methods = [ + InterfaceMethod< + /*desc=*/[{ + Returns a masked (padded) implementation of this operation. + `padMultiples` specifies the target alignment for each iteration + dimension. A value of 0 means no padding for that dimension. + Returns the replacement values for the original operation's results. + }], + /*retTy=*/"FailureOr>", + /*methodName=*/"getMaskedImplementation", + /*args=*/(ins + "OpBuilder &":$builder, + "ArrayRef":$padMultiples) + >, + ]; +} + +#endif // GC_TRANSFORMS_TENSOR_MASKING_OP_INTERFACE diff --git a/include/gc/Transforms/Utils/MatcherUtils.h b/include/gc/Transforms/Utils/MatcherUtils.h deleted file mode 100644 index c04530df1..000000000 --- a/include/gc/Transforms/Utils/MatcherUtils.h +++ /dev/null @@ -1,34 +0,0 @@ -//===- MatcherUtils.h - Matcher utils ---------------------------*- C++ -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef GC_MATCHERUTILS_H -#define GC_MATCHERUTILS_H - -namespace mlir { -class Value; -namespace linalg { -class LinalgOp; -class GenericOp; -} // namespace linalg -namespace structured_match { -namespace utils { - -// Returns true if the linalg operation is a 2d eltwsie floating point addition. -bool isTwoDAddOp(linalg::LinalgOp linalgOp, - SmallVectorImpl *capturedOperands = nullptr); - -// Returns true if the linalg.generic is a 2d eltwise floating point relu -// operation. -bool isTwoDReluOp(linalg::LinalgOp linalgOp, - SmallVectorImpl *capturedOperands = nullptr); - -} // namespace utils -} // namespace structured_match -} // namespace mlir - -#endif // GC_MATCHERUTILS_H diff --git a/include/gc/Transforms/Utils/StructuredOpMatcher.h b/include/gc/Transforms/Utils/StructuredOpMatcher.h deleted file mode 100644 index 66d398474..000000000 --- a/include/gc/Transforms/Utils/StructuredOpMatcher.h +++ /dev/null @@ -1,388 +0,0 @@ -//===- StructuredOpMatcher.h - Structured op matcher ------------*- C++ -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef GC_UTILS_STRUCTUREDOPMATCHER_H -#define GC_UTILS_STRUCTUREDOPMATCHER_H - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "llvm/ADT/SmallSet.h" -#include "llvm/ADT/SmallVector.h" -#include -#include - -namespace mlir { -class Operation; -namespace structured_match { - -// Base class for the matcher predicates selection tag. -struct MatchSelector { - MatchSelector() = delete; - size_t getLowerBound() const { return lowerBound; }; - size_t getUpperBound() const { return upperBound; }; - -protected: - explicit MatchSelector(size_t lowerBound, size_t upperBound) - : lowerBound(lowerBound), upperBound(upperBound) { - assert(upperBound > lowerBound); - } - -private: - const size_t lowerBound; - const size_t upperBound; -}; - -// Selector which specifies that predicate should apply on all values. -struct MatchAll : public MatchSelector { - MatchAll() : MatchSelector(0, std::numeric_limits::max()) {} -}; - -// Selector which specifies that predicate should apply only on one value at -// the position `idx`. -struct MatchOne : public MatchSelector { - MatchOne() = delete; - MatchOne(size_t idx) : MatchSelector(idx, idx + 1) {} -}; - -// Callable object to check if the number of loops in `op` satisfies `fun`. -struct NumOfLoops { - NumOfLoops() = delete; - explicit NumOfLoops(std::function fun) : fun(std::move(fun)){}; - - bool operator()(Operation *op) const { - if (auto linalgOp = dyn_cast_or_null(op)) { - auto numberOfLoops = linalgOp.getNumLoops(); - return fun(numberOfLoops); - } - return false; - } - std::function fun; -}; - -// Callable object to check if the `operand` of `op` has a map that satisfies -// `fun`. -struct HasMap { - HasMap() = delete; - explicit HasMap(std::function fun) : fun(std::move(fun)){}; - explicit HasMap(std::function fun, AffineMap *ptrMap) - : fun(std::move(fun)), ptrMap(ptrMap){}; - - bool operator()(OpOperand *operand, Operation *op) const { - if (auto linalgOp = dyn_cast_or_null(op)) { - auto map = linalgOp.getMatchingIndexingMap(operand); - assert(fun && "must be a callable target"); - if (!fun(map)) - return false; - if (ptrMap) - *ptrMap = std::move(map); - return true; - } - return false; - } - std::function fun; - AffineMap *ptrMap = nullptr; -}; - -// Callble object to verify if `map` is a broadcastable projected permutation -// map. We require the dimensions to be in sorted order this avoid filtering -// projected permutation without broadcasting semantics, for example -// affine_map<(d0, d1) -> (d1, d0)> is rejected. -struct BroadcastableProjectedPermutation { - BroadcastableProjectedPermutation() = default; - - bool operator()(AffineMap map) const { - if (map.getNumSymbols() > 0 || map.getNumResults() > map.getNumInputs()) - return false; - - SmallVector seen(map.getNumInputs(), false); - SmallVector pos; - for (auto expr : map.getResults()) { - if (auto dim = dyn_cast(expr)) { - if (seen[dim.getPosition()]) - return false; - seen[dim.getPosition()] = true; - pos.push_back(dim.getPosition()); - } else if (auto constExpr = dyn_cast(expr)) { - if (constExpr.getValue() != 0) - return false; - } else { - return false; - } - } - return llvm::is_sorted(pos); - } -}; - -// Callable object to verify if `map` is a projected permutation. -struct ProjectedPermutation { - ProjectedPermutation() = default; - - bool operator()(AffineMap map) const { return map.isProjectedPermutation(); } -}; - -// Callable object to verify if `map` is an identity map. -struct Identity { - Identity() = default; - - bool operator()(AffineMap map) const { return map.isIdentity(); } -}; - -// Callable object to verify if `operand` has static shape. -struct HasStaticShape { - HasStaticShape() = default; - HasStaticShape(SmallVectorImpl *shape) : shape(shape){}; - - bool operator()(OpOperand *operand, Operation *op) const { - auto operandType = operand->get().getType(); - if (auto shapedType = dyn_cast_or_null(operandType)) { - if (!shapedType.hasStaticShape()) - return false; - if (shape) { - for (int64_t shapeOnDim : shapedType.getShape()) - shape->push_back(shapeOnDim); - } - } - return true; - } - SmallVectorImpl *shape = nullptr; -}; - -// Callable object to verify if `operand` has static strides. -// If `operand` is a tensor type or a scalar, return true. -struct HasStaticStrides { - HasStaticStrides() = default; - HasStaticStrides(SmallVector *strides) : strides(strides){}; - - bool operator()(OpOperand *operand, Operation *op) const { - auto operandType = operand->get().getType(); - SmallVector strides; - if (auto memRefType = dyn_cast_or_null(operandType)) { - int64_t offset; - if (failed(getStridesAndOffset(memRefType, strides, offset))) - return false; - if (llvm::any_of(strides, [](int64_t stride) { - return stride == ShapedType::kDynamic; - })) { - return false; - } - if (this->strides) - this->strides->append(strides.begin(), strides.end()); - } - return true; - } - SmallVectorImpl *strides = nullptr; -}; - -// Callable object to verify `operand` to have a rank in `ranks`. -struct HasRank { - HasRank() = delete; - explicit HasRank(std::initializer_list ranks) : ranks(ranks){}; - - bool operator()(OpOperand *operand, Operation *op) const { - auto operandType = operand->get().getType(); - if (!isa(operandType)) - return llvm::is_contained(ranks, HasRank::SCALAR); - int64_t rank = cast(operandType).getRank(); - return llvm::any_of( - ranks, [=](int64_t expectedRank) { return expectedRank == rank; }); - } - - // There are multiple way to represent a scalar: f32, tensor. - // SCALAR means f32. - static constexpr int64_t SCALAR = -1; - std::vector ranks; -}; - -// Callable object to verify `operand` to have an element type `T`. -template struct HasElementType { - bool operator()(OpOperand *operand, Operation *op) const { - auto operandType = getElementTypeOrSelf(operand->get().getType()); - return isa(operandType); - } -}; - -// Callable object to check if the input is equal to specified `value`. -template struct EqualsTo { - EqualsTo() = delete; - explicit EqualsTo(T value) : value(value){}; - - const T value; - - bool operator()(T value) const { return value == this->value; } -}; -template EqualsTo(T) -> EqualsTo; - -// Callable object to check if the input is greater than or equal to specified -// `value`. -struct GreaterThanOrEqualTo { - GreaterThanOrEqualTo() = delete; - explicit GreaterThanOrEqualTo(size_t value) : value(value){}; - const size_t value; - - bool operator()(size_t value) const { return value >= this->value; } -}; - -// Callable object to validate number of init operands for `op`. -struct NumDpsInits { - NumDpsInits() = delete; - explicit NumDpsInits(std::function fun) : fun(std::move(fun)){}; - - bool operator()(Operation *op) const { - if (auto linalgOp = dyn_cast_or_null(op)) - return fun(linalgOp.getNumDpsInits()); - return false; - } - - std::function fun; -}; - -// Callable object to validate number of input operands for `op`. -struct NumDpsInputs { - NumDpsInputs() = delete; - explicit NumDpsInputs(std::function fun) : fun(std::move(fun)){}; - - bool operator()(Operation *op) { - if (auto linalgOp = dyn_cast_or_null(op)) - return fun(linalgOp.getNumDpsInputs()); - return false; - } - - std::function fun; -}; - -// Callable object to validate number of regions for `op`. -struct NumRegions { - NumRegions() = delete; - explicit NumRegions(std::function fun) : fun(std::move(fun)){}; - - bool operator()(Operation *op) const { - if (auto linalgOp = dyn_cast_or_null(op)) - return fun(linalgOp->getNumRegions()); - return false; - } - - std::function fun; -}; - -// Logical OR between two predicates. -struct _OR { - _OR() = delete; - _OR(std::function lhs, std::function rhs) - : lhs(std::move(lhs)), rhs(std::move(rhs)) {} - - bool operator()(size_t num) { return (lhs(num) || rhs(num)); } - - std::function lhs; - std::function rhs; -}; - -// Callable object to check if `op` adheres to a given property passed -// as an std::function object. -struct VerifyOpProperty { - VerifyOpProperty() = delete; - explicit VerifyOpProperty(std::function fun) - : fun(std::move(fun)){}; - - bool operator()(Operation *op) { - if (succeeded(fun(op))) - return true; - return false; - } - - std::function fun; -}; - -// Work-around for template specialization. -struct WithSingleOpImpl { - WithSingleOpImpl() = default; - - bool withSingleOpImpl(StringRef, Region *, Operation *, - SmallVectorImpl *); -}; - -// Callable object to check the `op` region for a single scalar operation OpTy. -template struct WithSingleOp { - WithSingleOp() : WithSingleOp(nullptr){}; - WithSingleOp(SmallVectorImpl *captures) : captures(captures){}; - - bool operator()(Region *region, Operation *op) { - return WithSingleOpImpl().withSingleOpImpl(OpTy::getOperationName(), region, - op, captures); - } - -private: - SmallVectorImpl *captures; -}; - -// Implemenation to allow definition in cpp file -using TypeCheckFunc = std::function; -bool withOpChainImpl(Region *region, Operation *op, SmallVectorImpl *, - SmallVectorImpl &); - -// Callable object to check the region for a chain of operations. -template struct WithOpChain { - WithOpChain() : WithOpChain(nullptr){}; - WithOpChain(SmallVectorImpl *captures) : captures(captures) { - (typeChecks.push_back([](Operation *op) { return isa(op); }), ...); - }; - - bool operator()(Region *region, Operation *op) { - return withOpChainImpl(region, op, captures, typeChecks); - } - -private: - SmallVectorImpl *captures; - SmallVector typeChecks; -}; - -class StructuredOpMatcher { - using PredicateFn = std::function; - -public: - StructuredOpMatcher() = default; - - StructuredOpMatcher(PredicateFn &&firstPredicate) { - predicates.push_back(std::move(firstPredicate)); - } - - template static StructuredOpMatcher make() { - return StructuredOpMatcher( - [](linalg::LinalgOp op) { return isa(op.getOperation()); }); - } - - // Match given `op` using stored predicates. - bool match(Operation *op); - - // Predicates on operation. - StructuredOpMatcher &operation(std::function); - - // Predicate on OpOperands. - StructuredOpMatcher &input(MatchSelector range, - std::function); - - // Predicates on OpOperands. - StructuredOpMatcher &output(MatchSelector range, - std::function); - - // Predicates on Iterators. - StructuredOpMatcher &dim(MatchSelector range, - SmallVector kinds); - StructuredOpMatcher &dim(MatchSelector range, mlir::utils::IteratorType kind); - - // Predicates on region. - StructuredOpMatcher ®ion(MatchSelector range, - std::function); - -private: - llvm::SmallVector predicates; -}; - -} // namespace structured_match -} // namespace mlir - -#endif // GC_UTILS_STRUCTUREDOPMATCHER_H diff --git a/include/gc/Transforms/Utils/ValueUtils.h b/include/gc/Transforms/Utils/ValueUtils.h deleted file mode 100644 index 1709078e8..000000000 --- a/include/gc/Transforms/Utils/ValueUtils.h +++ /dev/null @@ -1,81 +0,0 @@ -//===-- ValueUtils.h - Zero-checking utilities ------------------*- C++ -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef GC_TRANSFORMS_UTILS_VALUEUTILS_H -#define GC_TRANSFORMS_UTILS_VALUEUTILS_H - -namespace mlir { -class Value; -class OpBuilder; -namespace utils { - -// Returns true if the value is a constant float or integer. -bool isValConstZero(Value val); - -// Returns true if the op defining `val` represents a zero filled tensor. -bool isZeroTensor(Value val); - -// Returns the strides of `val`. The method returns something useful -// only if the `val` type is a strided memref. -FailureOr> getStrides(Value val); - -// Returns the strides of `val`. The method returns something useful -// only if the `val` type is a strided memref and the strides are statically -// known. -FailureOr> getStaticStrides(Value val); - -// Return the offset and ptr for `val`. Assert if `val` -// is not a memref. -std::pair getPtrAndOffset(OpBuilder &builder, Value operand); - -// Create a 'mlir::vector' constant from a list of values. -template -Value createTypedVector(PatternRewriter &rewriter, Location loc, - ArrayRef values, Type elementType) { - mlir::VectorType vectorType = - mlir::VectorType::get({static_cast(values.size())}, elementType); - mlir::DenseElementsAttr denseAttr = - mlir::DenseElementsAttr::get(vectorType, values); - auto vector = - rewriter.create(loc, vectorType, denseAttr) - .getResult(); - return vector; -} - -// Flatten a 2D memref to a 1D memref. -Value flattenMemref(PatternRewriter &rewriter, Location loc, Value srcMemref); - -// Return true if the memref has shared memory space. -bool hasSharedMemSpace(mlir::Value memref); - -// Go through all parent 'memref.subview' ops for the given `memref` -// and return the folded offsets of all subviews and the root memref. -void computeSubviewOffsets(PatternRewriter &rewriter, Location loc, - Value memref, SmallVector &resultOffsets, - Value &resultRootMemref); - -// Return the strides of the memref -SmallVector getMemrefStrides(PatternRewriter &rewriter, - Location loc, Value memref); - -// Squeeze the leading dimensions of a given memref up to 'maxDims'. -FailureOr reduceMemrefDims(PatternRewriter &rewriter, Location loc, - Value memref, size_t maxDims = 2); - -// Squeeze the leading dimensions of memref operands of a given 'linalgOp'. -LogicalResult maybeSqueezeDims(PatternRewriter &rewriter, - linalg::LinalgOp linalgOp, size_t maxDims = 2); - -// Return if a memref with the given shape can be squeezed to the shape of -// 'maxDims'. Only leading dimensions are considered squeezable. -bool canSqueezeDims(llvm::ArrayRef shape, size_t maxDims = 2); - -} // namespace utils -} // namespace mlir - -#endif // GC_TRANSFORMS_UTILS_VALUEUTILS_H diff --git a/include/gc/Utils.h b/include/gc/Utils.h deleted file mode 100644 index 6e2eb630d..000000000 --- a/include/gc/Utils.h +++ /dev/null @@ -1,18 +0,0 @@ -//===-- Utils.h - Common utility functions and macros -----------*- C++ -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef GC_UTILS_H -#define GC_UTILS_H - -#if defined _WIN32 || defined __CYGWIN__ -#define GC_DLL_EXPORT __declspec(dllexport) -#else -#define GC_DLL_EXPORT __attribute__((visibility("default"))) -#endif - -#endif diff --git a/include/gc_version.h b/include/gc_version.h deleted file mode 100644 index e2172a2e0..000000000 --- a/include/gc_version.h +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (C) 2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef GC_VERSION_H -#define GC_VERSION_H - -#if !defined(GC_VERSION_MAJOR) || !defined(GC_VERSION_MINOR) || !defined(GC_VERSION_PATCH) -// define an invalid version if it wasn't defined by CMake -#include -#define GC_VERSION_MAJOR std::numeric_limits::max() -#define GC_VERSION_MINOR std::numeric_limits::max() -#define GC_VERSION_PATCH std::numeric_limits::max() -#endif -#ifndef GC_VERSION_HASH -#define GC_VERSION_HASH "N/A" -#endif - -#endif diff --git a/legacy/core/CMakeLists.txt b/legacy/core/CMakeLists.txt deleted file mode 100644 index f5e7a84c2..000000000 --- a/legacy/core/CMakeLists.txt +++ /dev/null @@ -1,245 +0,0 @@ -#=============================================================================== -# Copyright 2020-2023 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#=============================================================================== - -### graph-compiler specical config ### - -set(SC_LIB_NAME dnnl_graph_backend_compiler_core) -SET(SC_MEMORY_LEAK_CHECK 0 CACHE STRING "memory leak checking. 0 = off, 1 = check numbers of instances, 2 = dump leaked objects") -set(SC_CPP_VER 11) - -if(NOT "${SC_CPU_RUNTIME}" MATCHES "^(OMP|TBB|SEQ|THREADPOOL|SYCL)$") - message(FATAL_ERROR "Unsupported CPU runtime: ${SC_CPU_RUNTIME}") -endif() - -if("${SC_CPU_RUNTIME}" STREQUAL "SYCL") - set(SC_CPU_RUNTIME "TBB") -endif() - -# SET(DNNL_CPU_RUNTIME ${SC_CPU_RUNTIME}) -# SET(DNNL_CPU_THREADING_RUNTIME ${SC_CPU_RUNTIME}) - -if(${SC_CPU_RUNTIME} STREQUAL "OMP") - set(SC_OMP_ENABLED ON) - add_definitions_with_host_compiler("-DSC_CPU_THREADPOOL=1") - set_property(GLOBAL APPEND PROPERTY GRAPH_COMPILER_DEFINITIONS "-DSC_CPU_THREADPOOL=1") -endif() - -if(${SC_CPU_RUNTIME} STREQUAL "TBB") - add_definitions_with_host_compiler("-DSC_CPU_THREADPOOL=2") - set_property(GLOBAL APPEND PROPERTY GRAPH_COMPILER_DEFINITIONS "-DSC_CPU_THREADPOOL=2") -endif() - -if(${SC_CPU_RUNTIME} STREQUAL "SEQ") - add_definitions_with_host_compiler("-DSC_CPU_THREADPOOL=0") - set_property(GLOBAL APPEND PROPERTY GRAPH_COMPILER_DEFINITIONS "-DSC_CPU_THREADPOOL=0") -endif() - -if(${SC_CPU_RUNTIME} STREQUAL "THREADPOOL") - add_definitions_with_host_compiler("-DSC_CPU_THREADPOOL=3") - set_property(GLOBAL APPEND PROPERTY GRAPH_COMPILER_DEFINITIONS "-DSC_CPU_THREADPOOL=3") -endif() - -set(DNNL_PATH "${CMAKE_SOURCE_DIR}/src") - -include_directories_with_host_compiler( - ${CMAKE_CURRENT_SOURCE_DIR}/src - ${PROJECT_SOURCE_DIR}/src/cpu/x64 -) - -file(GLOB_RECURSE SC_SOURCES - ${CMAKE_CURRENT_SOURCE_DIR}/src/*.c - ${CMAKE_CURRENT_SOURCE_DIR}/src/*.cc - ${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/src/*.hpp - ) - -if(NOT DNNL_ENABLE_JIT_PROFILING) - list(REMOVE_ITEM SC_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/src/compiler/jit/xbyak/debug/vtune/vtune.cpp) -else() - add_definitions_with_host_compiler(-DSC_PROFILING=1) -endif() - -if(WIN32 OR APPLE OR NOT SC_CFAKE_ENABLED) - list(REMOVE_ITEM SC_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/src/util/fdstream.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/src/util/fdstream.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/src/util/subprocess.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/src/compiler/jit/cfake/cfake_jit.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/src/compiler/jit/cfake/cfake_jit.hpp - ) - add_definitions_with_host_compiler("-DSC_CFAKE_JIT_ENABLED=0") - set_property(GLOBAL APPEND PROPERTY GRAPH_COMPILER_DEFINITIONS "-DSC_CFAKE_JIT_ENABLED=0") -else() - add_definitions_with_host_compiler("-DSC_CFAKE_JIT_ENABLED=1") - set_property(GLOBAL APPEND PROPERTY GRAPH_COMPILER_DEFINITIONS "-DSC_CFAKE_JIT_ENABLED=1") -endif() - - -if(NOT SC_BUILTIN_ENABLED) - file(GLOB_RECURSE SC_BUILTIN_JIT_SRC - ${CMAKE_CURRENT_SOURCE_DIR}/src/compiler/jit/xbyak/* - ) - list(APPEND SC_BUILTIN_JIT_SRC ${CMAKE_CURRENT_SOURCE_DIR}/dummy) - list(REMOVE_ITEM SC_SOURCES ${SC_BUILTIN_JIT_SRC}) - add_definitions_with_host_compiler("-DSC_BUILTIN_JIT_ENABLED=0") - set_property(GLOBAL APPEND PROPERTY GRAPH_COMPILER_DEFINITIONS "-DSC_BUILTIN_JIT_ENABLED=0") -else() - add_definitions_with_host_compiler("-DSC_BUILTIN_JIT_ENABLED=1") - set_property(GLOBAL APPEND PROPERTY GRAPH_COMPILER_DEFINITIONS "-DSC_BUILTIN_JIT_ENABLED=1") -endif() - -if(SC_LLVM_ENABLED) - if(${SC_LLVM_VERSION} LESS 9) - # default to c++11 - elseif(${SC_LLVM_VERSION} LESS 16) - set(SC_CPP_VER "14") - else() - set(SC_CPP_VER "17") - endif() - if(NOT MSVC) - include(CheckCXXCompilerFlag) - CHECK_CXX_COMPILER_FLAG(-std=c++${SC_CPP_VER} COMPILER_SUPPORTS_LLVM_CXX_STD) - if(NOT COMPILER_SUPPORTS_LLVM_CXX_STD) - message(WARNING "LLVM${SC_LLVM_VERSION} requires c++${SC_CPP_VER} which is not supported by your C++ compiler. LLVM backend is disabled") - set(SC_LLVM_ENABLED OFF) - set(SC_CPP_VER "11") - endif() - endif() -endif() - -if(NOT SC_LLVM_ENABLED) - file(GLOB_RECURSE SC_LLVM_SRC - ${CMAKE_CURRENT_SOURCE_DIR}/src/compiler/codegen/codegen_llvm.[ch]pp - ${CMAKE_CURRENT_SOURCE_DIR}/src/compiler/codegen/llvm/* - ${CMAKE_CURRENT_SOURCE_DIR}/src/compiler/jit/llvm/* - ) - list(APPEND SC_LLVM_SRC ${CMAKE_CURRENT_SOURCE_DIR}/dummy) - list(REMOVE_ITEM SC_SOURCES ${SC_LLVM_SRC}) -endif() - -if( SC_MEMORY_LEAK_CHECK GREATER -1 AND SC_MEMORY_LEAK_CHECK LESS 3 ) - add_definitions_with_host_compiler("-DSC_MEMORY_LEAK_CHECK=${SC_MEMORY_LEAK_CHECK}") - set_property(GLOBAL APPEND PROPERTY GRAPH_COMPILER_DEFINITIONS "-DSC_MEMORY_LEAK_CHECK=${SC_MEMORY_LEAK_CHECK}") -else() - message(FATAL_ERROR "SC_MEMORY_LEAK_CHECK should be in 0 to 2") -endif() - -#################### -# Configuring LLVM -#################### - -if(SC_LLVM_ENABLED) - message(STATUS "LLVM backend is ON. version ${SC_LLVM_VERSION}") - add_definitions_with_host_compiler("-DSC_LLVM_BACKEND=${SC_LLVM_VERSION}") - set_property(GLOBAL APPEND PROPERTY GRAPH_COMPILER_DEFINITIONS -DSC_LLVM_BACKEND=${SC_LLVM_VERSION}) - file(TO_CMAKE_PATH "${SC_LLVM_INCLUDE_PATH}" SC_LLVM_INCLUDE_PATH_CMAKE) # to convert "\" to cmake style "/" in windows path - include_directories_with_host_compiler(${SC_LLVM_INCLUDE_PATH_CMAKE}) # include LLVM headers -endif() -#################### -# Compiling objlib -#################### - -add_library(${SC_LIB_NAME} OBJECT ${SC_SOURCES}) -set_property(GLOBAL APPEND PROPERTY DNNL_LIB_DEPS - $) - -if(SC_LIBRARY_TYPE STREQUAL "SHARED") - if(MSVC) - set_property(TARGET dnnl_graph_backend_compiler PROPERTY COMPILE_FLAGS "/DSC_DLL_EXPORTS") - else() - set_property(TARGET dnnl_graph_backend_compiler PROPERTY COMPILE_FLAGS "-DSC_DLL_EXPORTS") - endif() - add_definitions_with_host_compiler("-DSC_DLL_EXPORTS") -endif() - -include_directories_with_host_compiler( - ${PROJECT_BINARY_DIR}/include -) - -################ -# Linking LLVM -################ - -if(SC_LLVM_ENABLED) - set(SC_LLVM_ABS_LIBPATH ${SC_LLVM_LIB_NAME}) - - #expose the LLVM libraries it depends on when cmake version > 3.0 - if(${CMAKE_VERSION} VERSION_GREATER "3.0.0") - add_library(dnnl_graphcompiler_llvm_lib INTERFACE IMPORTED GLOBAL) - set_property(TARGET dnnl_graphcompiler_llvm_lib PROPERTY INTERFACE_LINK_LIBRARIES ${SC_LLVM_LIB_NAME}) - add_library(dnnl_graphcompiler_llvm_lib_exclude_string INTERFACE IMPORTED GLOBAL) - set_property(TARGET dnnl_graphcompiler_llvm_lib_exclude_string PROPERTY INTERFACE_LINK_LIBRARIES ${SC_LLVM_LIB_EXCLUDE}) - endif() - - if(${SC_LLVM_CONFIG_RETURN_STATIC}) - set_property(GLOBAL APPEND PROPERTY DNNL_SUBDIR_EXTRA_STATIC_LIBS - ${SC_LLVM_ABS_LIBPATH}) - else() - set_property(GLOBAL APPEND PROPERTY DNNL_SUBDIR_EXTRA_SHARED_LIBS - ${SC_LLVM_ABS_LIBPATH}) - endif() - - if(NOT MSVC) - set_source_files_properties(${CMAKE_CURRENT_SOURCE_DIR}/src/compiler/jit/llvm/llvm_jit_resolver.cpp PROPERTIES COMPILE_FLAGS -fno-rtti) - endif() -endif() - -########################## -# Setting compiler flags -########################## - -if(CMAKE_BASE_NAME MATCHES "(icx|icpx)" AND ${SC_CPP_VER} LESS 14) - set(NEED_SET_CPP_STD OFF) -else() - set(NEED_SET_CPP_STD ON) -endif() - -if(NEED_SET_CPP_STD) - if(${CMAKE_VERSION} VERSION_LESS "3.1.0") - if(MSVC) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /std:c++${SC_CPP_VER}") - else() - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++${SC_CPP_VER}") - endif() - else() - set_property(TARGET ${SC_LIB_NAME} PROPERTY CXX_STANDARD ${SC_CPP_VER}) - endif() -endif() - -if(NOT MSVC OR CMAKE_BASE_NAME MATCHES "(icx|icpx)") - if(NOT MSVC) - - else() - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated-declarations") - append_host_compiler_options(CMAKE_CXX_FLAGS "-Wno-deprecated-declarations") - endif() - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-variable -Wno-unused-function -std=c99 -DSC_HOME=\"${CMAKE_CURRENT_SOURCE_DIR}\"") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-variable -Wno-unused-function") - append_host_compiler_options(CMAKE_CXX_FLAGS "-Wno-unused-variable -Wno-unused-function") - if(DNNL_DPCPP_HOST_COMPILER STREQUAL "DEFAULT") - append(CMAKE_CXX_FLAGS " -DSC_HOME=\"${CMAKE_CURRENT_SOURCE_DIR}\"") - endif() - if(NOT CMAKE_CXX_COMPILER_ID STREQUAL "Intel") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wmissing-field-initializers") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wmissing-field-initializers") - endif() -endif() - -if("${DNNL_CPU_RUNTIME}" STREQUAL "SYCL" AND WIN32 AND CMAKE_CXX_COMPILER_ID MATCHES "Clang") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated-declarations") - append_host_compiler_options(CMAKE_CXX_FLAGS "-Wno-deprecated-declarations") -endif() - -# SET(DNNL_CPU_RUNTIME ${SC_CPU_RUNTIME}) diff --git a/legacy/core/cmake/find_llvm.cmake b/legacy/core/cmake/find_llvm.cmake deleted file mode 100644 index b21e137d2..000000000 --- a/legacy/core/cmake/find_llvm.cmake +++ /dev/null @@ -1,159 +0,0 @@ -#=============================================================================== -# Copyright 2020-2023 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#=============================================================================== - -macro(exec_cmd_and_check CMD1 CMD2 CMD3 OUT) - execute_process(COMMAND ${CMD1} ${CMD2} ${CMD3} - RESULT_VARIABLE _sc_exit_code - OUTPUT_VARIABLE _sc_cmd_out) - if(NOT ${_sc_exit_code} STREQUAL 0) - message(FATAL_ERROR "Failed with exit code ${_sc_exit_code}: ${CMD}") - endif() - set(${OUT} ${_sc_cmd_out}) -endmacro() - -macro(find_llvm) - if(NOT ${SC_LLVM_VERSION} STREQUAL OFF) - # if LLVM environment is already set in parent environment, do nothing - elseif(LLVM_FOUND) - # if LLVM is found by cmake, use it - set(SC_LLVM_VERSION ${LLVM_VERSION_MAJOR}) - set(SC_LLVM_INCLUDE_PATH ${LLVM_INCLUDE_DIRS}) - set(SC_LLVM_LIB_NAME "") - foreach(__libname ${LLVM_AVAILABLE_LIBS}) - # skip dynamic LLVM libraries - if(NOT ${__libname} MATCHES "^(LLVM|LLVMRemarks|LLVMLTO|LTO|Remarks)$") - get_target_property(__interface_lib ${__libname} INTERFACE_LINK_LIBRARIES) - # skip targets that depend on dynamic LLVM libraries - list(FIND __interface_lib "LLVM" _index) - if (NOT ${_index} GREATER -1) - set(LIB_VAR "__LIB_${__libname}") - find_library(${LIB_VAR} NAMES ${__libname} HINTS ${LLVM_LIBRARY_DIRS}) - if(${LIB_VAR}) - list(APPEND SC_LLVM_LIB_NAME ${__libname}) - endif() - endif() - endif() - endforeach() - elseif(SC_LLVM_CONFIG STREQUAL "CMAKE") - find_package(LLVM CONFIG) - if(NOT LLVM_FOUND) - message(FATAL_ERROR "LLVM not found.") - else() - set(SC_LLVM_CONFIG_RETURN_STATIC ON) - # LLVM-10+ has a bug in llvm_map_components_to_libnames(LLVM_LIBS all) - # bug report at https://bugs.llvm.org/show_bug.cgi?id=47003 - if(SC_LLVM_TRY_DYN_LINK OR (NOT SC_LIBRARY_TYPE STREQUAL "STATIC")) - # find if LLVM has shared library built - list(FIND LLVM_AVAILABLE_LIBS "LLVM" _index) - if (${_index} GREATER -1) - set(LLVM_LIBS LLVM) - set(SC_LLVM_CONFIG_RETURN_STATIC OFF) - else() - set(LLVM_LIBS ${LLVM_AVAILABLE_LIBS}) - endif() - else() - set(LLVM_LIBS ${LLVM_AVAILABLE_LIBS}) - endif() - set(SC_LLVM_VERSION ${LLVM_VERSION_MAJOR}) - set(SC_LLVM_INCLUDE_PATH ${LLVM_INCLUDE_DIRS}) - set(SC_LLVM_LIB_NAME ${LLVM_LIBS}) - message(STATUS "Link static LLVM=${SC_LLVM_CONFIG_RETURN_STATIC}") - endif() - else() - if(SC_LLVM_CONFIG STREQUAL "AUTO") - foreach(__sc_llvm_ver RANGE 16 10 -1) - find_program(__sc_llvm_config llvm-config-${__sc_llvm_ver}) - if(__sc_llvm_config) - SET(SC_LLVM_CONFIG ${__sc_llvm_config}) - endif() - endforeach() - if(SC_LLVM_CONFIG STREQUAL "AUTO") - find_program(__sc_llvm_config llvm-config) - if(__sc_llvm_config) - SET(SC_LLVM_CONFIG ${__sc_llvm_config}) - endif() - if(SC_LLVM_CONFIG STREQUAL "AUTO") - message(FATAL_ERROR "Failed to find llvm-config in AUTO mode. Please specify an llvm-config executable.") - endif() - endif() - endif() - message(STATUS "Finding LLVM using ${SC_LLVM_CONFIG}") - set(__sc_llvm_link "--link-static") - set(SC_LLVM_CONFIG_RETURN_STATIC ON) - if(NOT MSVC) - if(SC_LLVM_TRY_DYN_LINK OR (NOT SC_LIBRARY_TYPE STREQUAL "STATIC")) - # try link with shared library - execute_process(COMMAND ${SC_LLVM_CONFIG} "--libfiles" "--link-shared" - RESULT_VARIABLE _sc_exit_code - OUTPUT_VARIABLE _sc_cmd_out ERROR_QUIET) - if(${_sc_exit_code} STREQUAL 0) - set(__sc_llvm_link "--link-shared") - set(SC_LLVM_CONFIG_RETURN_STATIC OFF) - endif() - else() - execute_process(COMMAND ${SC_LLVM_CONFIG} "--libfiles" "--link-static" - RESULT_VARIABLE _sc_exit_code - OUTPUT_VARIABLE _sc_cmd_out ERROR_QUIET) - if(NOT ${_sc_exit_code} STREQUAL 0) - set(__sc_llvm_link "--link-shared") - set(SC_LLVM_CONFIG_RETURN_STATIC OFF) - endif() - endif() - endif() - - message(STATUS "Decided to link LLVM with ${__sc_llvm_link}") - - exec_cmd_and_check(${SC_LLVM_CONFIG} "--system-libs" "${__sc_llvm_link}" - _sc_llvm_system_libs) - exec_cmd_and_check(${SC_LLVM_CONFIG} "--libfiles" "${__sc_llvm_link}" - _sc_llvm_libs) - - # libs to link - string(STRIP ${_sc_llvm_libs} _sc_llvm_libs) - string(STRIP ${_sc_llvm_system_libs} _sc_llvm_system_libs) - separate_arguments(_sc_llvm_libs) - if(SC_LLVM_CONFIG_RETURN_STATIC) - set(SC_LLVM_LIB_NAME "") - foreach(__libname ${_sc_llvm_libs}) - if(EXISTS ${__libname}) - list(APPEND SC_LLVM_LIB_NAME ${__libname}) - endif() - endforeach() - else() - set(SC_LLVM_LIB_NAME ${_sc_llvm_libs}) - endif() - set(SC_LLVM_LIB_NAME "${SC_LLVM_LIB_NAME} ${_sc_llvm_system_libs}") - separate_arguments(SC_LLVM_LIB_NAME) - - # version - exec_cmd_and_check(${SC_LLVM_CONFIG} "--version" "" - _sc_llvm_version) - string(REGEX MATCH "^([^.]+)\\.([^.])+\\.[^.]+.*$" _ ${_sc_llvm_version}) - set(SC_LLVM_VERSION ${CMAKE_MATCH_1}) - string(STRIP ${SC_LLVM_VERSION} SC_LLVM_VERSION) - - # include directory - exec_cmd_and_check(${SC_LLVM_CONFIG} "--includedir" "" SC_LLVM_INCLUDE_PATH) - endif() - if(SC_LLVM_CONFIG_RETURN_STATIC) - set(__sc_llvm_exclude "") - foreach(__sc_llvm_lib ${SC_LLVM_LIB_NAME}) - get_filename_component(__sc_llvm_file ${__sc_llvm_lib} NAME) - list(APPEND __sc_llvm_exclude ${__sc_llvm_file}) - endforeach() - string(REPLACE ";" ":" SC_LLVM_LIB_EXCLUDE "${__sc_llvm_exclude}") - endif() -endmacro() diff --git a/legacy/core/src/compiler/codegen/codegen_c.cpp b/legacy/core/src/compiler/codegen/codegen_c.cpp deleted file mode 100644 index 2b124e125..000000000 --- a/legacy/core/src/compiler/codegen/codegen_c.cpp +++ /dev/null @@ -1,1393 +0,0 @@ -/* - * Copyright (C) 2025 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions - * and limitations under the License. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "codegen_c.hpp" -#include "../ir/viewer.hpp" -#include "codegen_c_internal.hpp" -#include "precodegen_passes.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -static constexpr const char *wrapper_postfix = "_0wrapper"; - -static std::string get_closure_wrapper_name(const std::string &name) { - return name + "_0closurewrapper"; -} - -static void print_cpp_etype(ostream &os, sc_data_etype t) { - switch (t) { - case sc_data_etype::UNDEF: - assert(0 && "Met undef"); - break; - case sc_data_etype::F16: - os << "_Float16"; - break; - case sc_data_etype::BF16: - os << "uint16_t"; - break; - case sc_data_etype::U16: - os << "uint16_t"; - break; - case sc_data_etype::F32: - os << "float"; - break; - case sc_data_etype::S32: - os << "int32_t"; - break; - case sc_data_etype::U32: - os << "uint32_t"; - break; - case sc_data_etype::S8: - os << "int8_t"; - break; - case sc_data_etype::U8: - os << "uint8_t"; - break; - case sc_data_etype::INDEX: - os << "uint64_t"; - break; - case sc_data_etype::POINTER: - os << "void*"; - break; - case sc_data_etype::VOID_T: - os << "void"; - break; - case sc_data_etype::BOOLEAN: - os << "bool"; - break; - case sc_data_etype::GENERIC: - os << "generic_val"; - break; - default: - if (etypes::is_pointer(t)) { - print_cpp_etype(os, etypes::get_pointer_element(t)); - os << '*'; - return; - } - assert(0 && "Unknown type"); - } -} - -ostream &print_cpp_type(ostream &os, sc_data_type_t dtype) { - assert(dtype.lanes_ == 1 && "Only scalar types allowed here"); - print_cpp_etype(os, dtype.type_code_); - return os; -} - -ostream &codegen_c_vis::print_cpp_var_def(const var &v) { - print_type(v->dtype_); - *os << ' ' << v->name_; - return *os; -} - -ostream &codegen_c_vis::print_tensor_def(const tensor &v) { - if (v->attr_ && v->attr_->get_or_else("volatile", false)) { - *os << "volatile "; - } - print_type(v->elem_dtype_); - auto ali_info = alias_info::get_alias_info(*v); - if (ali_info && !ali_info->has_no_alias()) { - *os << "* "; - } else { - *os << "* __restrict__ "; - } - *os << v->name_; - return *os; -} - -ostream &codegen_c_vis::print_param(const expr &e) { - if (e.isa()) { - print_cpp_var_def(e.static_as()); - } else { - print_tensor_def(e.checked_as()); - } - return *os; -} - -ostream &codegen_c_vis::print_func_params(const func_c &v, bool with_type) { - if (!v->params_.empty()) { - for (size_t i = 0; i < v->params_.size() - 1; i++) { - expr e = v->params_[i]; - if (with_type) { - print_param(e); - } else { - if (e.isa()) { - *os << e.static_as()->name_; - } else { - *os << e.checked_as()->name_; - } - } - *os << ", "; - } - expr e = v->params_.back(); - if (with_type) { - print_param(e); - } else { - if (e.isa()) { - *os << e.static_as()->name_; - } else { - *os << e.checked_as()->name_; - } - } - } - return *os; -} - -void codegen_c_vis::print_type(sc_data_type_t dtype) { - if (dtype.lanes_ == 1) { - print_cpp_type(*os, dtype); - } else { - switch (dtype) { - case sc_data_type_t::bf16(4): - *os << "vec_u16x4"; - break; - case sc_data_type_t::bf16(8): - *os << "vec_u16x8"; - break; - case sc_data_type_t::bf16(16): - *os << "vec_u16x16"; - break; - case sc_data_type_t::bf16(32): - *os << "vec_u16x32"; - break; - - case sc_data_type_t::f16(4): - *os << "vec_f16x4"; - break; - case sc_data_type_t::f16(8): - *os << "vec_f16x8"; - break; - case sc_data_type_t::f16(16): - *os << "vec_f16x16"; - break; - case sc_data_type_t::f16(32): - *os << "vec_f16x32"; - break; - - case sc_data_type_t::f32(4): - *os << "vec_f32x4"; - break; - case sc_data_type_t::f32(8): - *os << "vec_f32x8"; - break; - case sc_data_type_t::f32(16): - *os << "vec_f32x16"; - break; - - case sc_data_type_t::s32(4): - *os << "vec_s32x4"; - break; - case sc_data_type_t::s32(8): - *os << "vec_s32x8"; - break; - case sc_data_type_t::s32(16): - *os << "vec_s32x16"; - break; - - case sc_data_type_t::u32(4): - *os << "vec_u32x4"; - break; - case sc_data_type_t::u32(8): - *os << "vec_u32x8"; - break; - case sc_data_type_t::u32(16): - *os << "vec_u32x16"; - break; - - case sc_data_type_t::index(2): - *os << "vec_u64x2"; - break; - case sc_data_type_t::index(4): - *os << "vec_u64x4"; - break; - case sc_data_type_t::index(8): - *os << "vec_u64x8"; - break; - - case sc_data_type_t::u16(8): - *os << "vec_u16x8"; - break; - case sc_data_type_t::u16(16): - *os << "vec_u16x16"; - break; - case sc_data_type_t::u16(32): - *os << "vec_u16x32"; - break; - - case sc_data_type_t::s8(8): - *os << "vec_s8x8"; - break; - case sc_data_type_t::s8(16): - *os << "vec_s8x16"; - break; - case sc_data_type_t::s8(32): - *os << "vec_s8x32"; - break; - case sc_data_type_t::s8(64): - *os << "vec_s8x64"; - break; - - case sc_data_type_t::u8(8): - *os << "vec_u8x8"; - break; - case sc_data_type_t::u8(16): - *os << "vec_u8x16"; - break; - case sc_data_type_t::u8(32): - *os << "vec_u8x32"; - break; - case sc_data_type_t::u8(64): - *os << "vec_u8x64"; - break; - - case sc_data_type_t::boolean(4): - *os << "uint8_t"; - break; - case sc_data_type_t::boolean(8): - *os << "uint8_t"; - break; - case sc_data_type_t::boolean(16): - *os << "uint16_t"; - break; - case sc_data_type_t::boolean(32): - *os << "uint32_t"; - break; - case sc_data_type_t::boolean(64): - *os << "uint64_t"; - break; - - default: - COMPILE_ASSERT(0, "Cannot generate vector type for C++: " << dtype); - break; - } - } -} - -codegen_c_vis::codegen_c_vis(ostream *os, bool prototype_only, bool is_static) - : os(os), prototype_only(prototype_only), is_static(is_static) {} - -stmt_c codegen_c_vis::dispatch(stmt_c v) { - if (v->attr_) { - if (auto comments = - v->attr_->get_or_null>("comments")) { - for (auto &str : *comments) { - *os << "// " << str << "\n"; - print_indents(*os, indents); - } - } - } - return ir_visitor_t::dispatch(std::move(v)); -} - -static const std::string &get_func_name(const func_c &v) { - if (v->attr_) { - if (auto pname = - v->attr_->get_or_null("temp.replace_func_name")) { - return *pname; - } - } - return v->name_; -} - -func_c codegen_c_vis::dispatch(func_c v) { - if (utils::string_startswith(v->name_, "_should_inline_")) { - return v; - } - if (prototype_only) { - print_func_comments(v, *os); - } - bool is_symbol_in_runtime = - !is_offline_ && default_external_symbol_resolve(v->name_); - if (!is_symbol_in_runtime) { - *os << (is_static ? "static " : "extern \"C\" "); - } - print_type(v->ret_type_); - const std::string &real_name = get_func_name(v); - if (is_symbol_in_runtime) { - *os << " (*" << real_name << "_fptr" << ')' << '('; - } else { - *os << " " << real_name << '('; - } - print_func_params(v); - *os << ") noexcept"; - if (prototype_only) { - if (v->attr_) { - if (v->attr_->get_or_else(function_attrs::pure, false)) { - if (is_symbol_in_runtime) { - // create a wrapper to symbol fptr for compiler function opt - *os << ";\n"; - print_type(v->ret_type_); - *os << " __" << real_name << wrapper_postfix << '('; - print_func_params(v); - *os << ") noexcept __attribute__((const));\n"; - - print_type(v->ret_type_); - *os << " __" << real_name << wrapper_postfix << '('; - print_func_params(v); - *os << ") noexcept { return " << real_name << "_fptr("; - print_func_params(v, false); - *os << "); }"; - } else { - *os << " __attribute__((const))"; - } - } - if (v->attr_->get_or_else(function_attrs::no_alias, false)) { - *os << " __attribute__((returns_nonnull)) "; - if (!is_offline_) { - *os << "/*"; - } - *os << "__attribute__((malloc))"; - if (!is_offline_) { - *os << "*/"; - } - } - } - std::string tensor_idx = " __attribute__((nonnull ("; - bool has_tensor = false; - for (size_t i = 0; i < v->params_.size(); i++) { - if (v->params_[i].isa()) { - if (has_tensor) { - tensor_idx += ','; - } - has_tensor = true; - tensor_idx += std::to_string(i + 1); - } - } - if (has_tensor) { - *os << tensor_idx << ")))"; - } - } - if (!prototype_only && v->body_.defined()) { - dispatch(v->body_); - } else { - *os << ";"; - } - return std::const_pointer_cast(v); -} - -void codegen_c_vis::view(constant_c v) { - if (v->is_vector()) { - print_type(v->dtype_); - v->to_string(*os); - } else if (v->dtype_ != datatypes::pointer && v->dtype_.is_pointer()) { - (*os) << '(' << '('; - print_type(v->dtype_); - (*os) << ')' << v->value_.at(0).u64 << ')'; - } else { - if (v->dtype_.is_etype(sc_data_etype::F16)) { - (*os) << "(_Float16)"; - v->to_string(*os); - } else { - v->to_string(*os); - } - } -} - -void codegen_c_vis::view(var_c v) { *os << v->name_; } - -void codegen_c_vis::view(cast_c v) { - // if it is T* => void* cast, C++ can auto cast it - if (v->in_->dtype_.is_pointer() && v->dtype_ == datatypes::pointer) { - dispatch(v->in_); - return; - } - // cast to generic_val - if (v->dtype_ == datatypes::generic) { - assert(v->in_->dtype_.lanes_ == 1 && - "casting to generic vector is not implemented"); - // we have constructors defined. C++ will handle the conversions - dispatch(v->in_); - return; - } - // cast from generic_val - if (v->in_->dtype_ == datatypes::generic) { - assert(v->in_->dtype_.lanes_ == 1 && - "casting from generic vector is not implemented"); - if (v->dtype_.is_etype_pointer()) { - // prints (float*)(XXX.v_ptr) - *os << '('; - print_cpp_etype(*os, v->dtype_.type_code_); - *os << ")("; - dispatch(v->in_); - *os << ".v_ptr)"; - } else { - // prints xxx.v_float - dispatch(v->in_); - *os << ".v_"; - print_cpp_etype(*os, v->dtype_.type_code_); - } - return; - } - - if (v->dtype_.lanes_ == 1) { - if (v->in_->dtype_.is_etype(sc_data_etype::F32) && - v->dtype_.is_etype(sc_data_etype::BF16)) { - *os << "tobf16("; - dispatch(v->in_); - *os << ')'; - } else if (v->in_->dtype_.is_etype(sc_data_etype::F16) && - v->dtype_.is_etype(sc_data_etype::F32)) { - *os << "(float)"; - dispatch(v->in_); - } else if (v->in_->dtype_.is_etype(sc_data_etype::F32) && - v->dtype_.is_etype(sc_data_etype::F16)) { - *os << "(_Float16)"; - dispatch(v->in_); - } else { - *os << '('; - print_cpp_type(*os, v->dtype_) << ')'; - dispatch(v->in_); - } - - } else { - COMPILE_ASSERT(v->dtype_.lanes_ == v->in_->dtype_.lanes_, - "Vector cast should have same lanes. Got " - << v->dtype_ << " vs " << v->in_->dtype_); - if (v->in_->dtype_.is_etype(sc_data_etype::F32) && - v->dtype_.is_etype(sc_data_etype::BF16)) { - *os << "tobf16("; - dispatch(v->in_); - *os << ')'; - } else { - *os << '('; - print_type(v->dtype_); - *os << ")("; - dispatch(v->in_); - *os << ')'; - } - } -} - -#define GEN_BINARY(TYPE) \ - void codegen_c_vis::print_binary(const TYPE &v, const char *op) { \ - *os << '('; \ - dispatch(v->l_); \ - *os << op; \ - dispatch(v->r_); \ - *os << ')'; \ - } - -GEN_BINARY(binary_c); -GEN_BINARY(logic_c); -GEN_BINARY(cmp_c); - -void codegen_c_vis::trinary_func_codegen_c(const std::vector &args, - const char *funcname) { - COMPILE_ASSERT(args.size() == 3, - "Invalid arg size: " << args.size() << ", should be 3"); - auto &os = *this->os; - os << funcname << "("; - dispatch(args[0]); - os << ", "; - dispatch(args[1]); - os << ", "; - dispatch(args[2]); - os << ')'; -} - -void codegen_c_vis::binary_func_codegen_c(const std::vector &args, - const char *funcname) { - COMPILE_ASSERT(args.size() == 2, - "Invalid arg size: " << args.size() << ", should be 3"); - auto &os = *this->os; - os << funcname << '('; - dispatch(args[0]); - os << ", "; - dispatch(args[1]); - os << ')'; -} - -void codegen_c_vis::unary_func_codegen_c(const expr &arg, - const char *funcname) { - auto &os = *this->os; - os << funcname << '('; - dispatch(arg); - os << ')'; -} - -void codegen_c_vis::view(func_addr_c v) { - if (!is_offline_ && default_external_symbol_resolve(v->func_->name_)) { - *os << "(void*)" << v->func_->name_ << "_fptr"; - } else { - *os << "(void*)&" << v->func_->name_; - } -} - -static const char *prefetch_names[] = {"_MM_HINT_T0", "_MM_HINT_T1", - "_MM_HINT_T2", "_MM_HINT_NTA"}; - -void codegen_c_vis::view(intrin_call_c v) { - switch (v->type_) { - case intrin_type::min: - binary_func_codegen_c(v->args_, "sc_min"); - break; - case intrin_type::max: - binary_func_codegen_c(v->args_, "sc_max"); - break; - case intrin_type::abs: - unary_func_codegen_c(v->args_[0], "sc_abs"); - break; - case intrin_type::round: - unary_func_codegen_c(v->args_[0], "sc_round"); - break; - case intrin_type::floor: - unary_func_codegen_c(v->args_[0], "sc_floor"); - break; - case intrin_type::ceil: - unary_func_codegen_c(v->args_[0], "sc_ceil"); - break; - case intrin_type::exp: - unary_func_codegen_c(v->args_[0], "sc_exp"); - break; - case intrin_type::sqrt: - unary_func_codegen_c(v->args_[0], "sc_sqrt"); - break; - case intrin_type::rsqrt: - unary_func_codegen_c(v->args_[0], "sc_rsqrt"); - break; - case intrin_type::reduce_add: - unary_func_codegen_c(v->args_[0], "sc_reduce_add"); - break; - case intrin_type::reduce_mul: - unary_func_codegen_c(v->args_[0], "sc_reduce_mul"); - break; - case intrin_type::reduce_max: - unary_func_codegen_c(v->args_[0], "sc_reduce_max"); - break; - case intrin_type::reduce_min: - unary_func_codegen_c(v->args_[0], "sc_reduce_min"); - break; - case intrin_type::fmadd: - trinary_func_codegen_c(v->args_, "sc_fmadd"); - break; - case intrin_type::unpack_low: - *os << "sc_unpack_low_"; - print_type(v->dtype_); - *os << "_"; - *os << v->intrin_attrs_->get("elem_bits"); - *os << "bits"; - *os << "("; - dispatch(v->args_[0]); - *os << ", "; - dispatch(v->args_[1]); - *os << ')'; - break; - case intrin_type::unpack_high: - *os << "sc_unpack_high_"; - print_type(v->dtype_); - *os << "_"; - *os << v->intrin_attrs_->get("elem_bits"); - *os << "bits"; - *os << "("; - dispatch(v->args_[0]); - *os << ", "; - dispatch(v->args_[1]); - *os << ')'; - break; - case intrin_type::shuffle: - *os << "sc_shuffle_"; - print_type(v->dtype_); - *os << "_"; - *os << v->intrin_attrs_->get("type_bits"); - *os << "bits"; - *os << "("; - dispatch(v->args_[0]); - *os << ", "; - dispatch(v->args_[1]); - *os << ", "; - *os << v->intrin_attrs_->get("shuffle_imm"); - *os << ')'; - break; - case intrin_type::permute: - *os << "sc_permute_"; - print_type(v->dtype_); - *os << "("; - dispatch(v->args_[0]); - *os << ", "; - dispatch(v->args_[1]); - *os << ", "; - *os << v->intrin_attrs_->get("permute_imm"); - *os << ')'; - break; - case intrin_type::prefetch: { - *os << "_mm_prefetch("; - dispatch(v->args_[0]); - auto locality = v->intrin_attrs_->get("locality"); - COMPILE_ASSERT(locality <= 3 && locality >= 0, "bad locality for prefetch"); - *os << ", " << prefetch_names[locality] << ')'; - break; - } - case intrin_type::gather: - *os << "sc_gather("; - dispatch(v->args_[0]); - *os << ", "; - dispatch(v->args_[1]); - *os << ")"; - break; - case intrin_type::broadcast: - print_type(v->dtype_); - *os << "("; - dispatch(v->args_[0]); - *os << ')'; - break; - case intrin_type::reinterpret: - *os << "sc_reinterpret<"; - print_type(v->dtype_); - *os << ">("; - dispatch(v->args_[0]); - *os << ')'; - break; - case intrin_type::isnan: - *os << "sc_isnan("; - dispatch(v->args_[0]); - *os << ')'; - break; - case intrin_type::saturated_cast: - *os << "sc_saturated_cast<"; - print_type(v->dtype_); - *os << ">("; - dispatch(v->args_[0]); - *os << ')'; - break; - case intrin_type::round_and_cast: - *os << "sc_round_and_cast<"; - print_type(v->dtype_); - *os << ">("; - dispatch(v->args_[0]); - *os << ')'; - break; - case intrin_type::int_and: - *os << '('; - dispatch(v->args_[0]); - *os << " & "; - dispatch(v->args_[1]); - *os << ')'; - break; - case intrin_type::int_or: - *os << '('; - dispatch(v->args_[0]); - *os << " | "; - dispatch(v->args_[1]); - *os << ')'; - break; - case intrin_type::int_xor: - *os << '('; - dispatch(v->args_[0]); - *os << " ^ "; - dispatch(v->args_[1]); - *os << ')'; - break; - case intrin_type::shl: - *os << '('; - dispatch(v->args_[0]); - *os << " << "; - dispatch(v->args_[1]); - *os << ')'; - break; - case intrin_type::shr: - *os << '('; - dispatch(v->args_[0]); - *os << " >> "; - dispatch(v->args_[1]); - *os << ')'; - break; - case intrin_type::permutex2var: - trinary_func_codegen_c(v->args_, "sc_permutex2var"); - break; - case intrin_type::permutexvar: - if (v->args_[0].isa()) { - // vpermq have two different invocation. If is imm, we need to - // do use these part. - int lanes = v->intrin_attrs_->get("lanes"); - int elem_bits = - utils::get_sizeof_etype(v->args_[1]->dtype_.type_code_) * 8 * lanes; - auto suffix = std::to_string(elem_bits) + "bits"; - *os << "sc_permutexvar_"; - print_type(v->args_[1]->dtype_); - *os << "_" + suffix; - *os << '('; - dispatch(v->args_[0]); - *os << ','; - dispatch(v->args_[1]); - *os << ')'; - } else { - binary_func_codegen_c(v->args_, "sc_permutexvar"); - } - break; - case intrin_type::insert: - *os << "sc_insert_"; - print_type(v->dtype_); - *os << "("; - dispatch(v->args_[0]); - *os << ", "; - dispatch(v->args_[1]); - *os << ", "; - *os << v->intrin_attrs_->get("insert_imm"); - *os << ')'; - break; - case intrin_type::extract: - *os << "sc_extract_"; - print_type(v->args_[0]->dtype_); - *os << "("; - dispatch(v->args_[0]); - *os << ", "; - *os << v->intrin_attrs_->get("extract_imm"); - *os << ')'; - break; - default: - assert(0 && "Unknown intrinsic!"); - break; - } -} - -void codegen_c_vis::view(add_c v) { print_binary(v, " + "); } -void codegen_c_vis::view(sub_c v) { print_binary(v, " - "); } -void codegen_c_vis::view(mul_c v) { print_binary(v, " * "); } -void codegen_c_vis::view(div_c v) { print_binary(v, " / "); } -void codegen_c_vis::view(mod_c v) { print_binary(v, " % "); } - -void codegen_c_vis::view(cmp_eq_c v) { print_binary(v, " == "); } -void codegen_c_vis::view(cmp_lt_c v) { print_binary(v, " < "); } -void codegen_c_vis::view(cmp_le_c v) { print_binary(v, " <= "); } -void codegen_c_vis::view(cmp_gt_c v) { print_binary(v, " > "); } -void codegen_c_vis::view(cmp_ge_c v) { print_binary(v, " >= "); } -void codegen_c_vis::view(cmp_ne_c v) { print_binary(v, " != "); } - -void codegen_c_vis::view(logic_and_c v) { print_binary(v, " && "); } -void codegen_c_vis::view(logic_or_c v) { print_binary(v, " || "); } - -void codegen_c_vis::view(logic_not_c v) { - *os << '!'; - dispatch(v->in_); -} - -void codegen_c_vis::view(select_c v) { - if (v->l_->dtype_.lanes_ > 1) { - *os << "sc_select("; - dispatch(v->cond_); - *os << ", "; - dispatch(v->l_); - *os << ", "; - dispatch(v->r_); - } else { - *os << "("; - dispatch(v->cond_); - *os << "?"; - dispatch(v->l_); - *os << ":"; - dispatch(v->r_); - } - *os << ")"; -} - -void codegen_c_vis::view(indexing_c v) { - if (v->dtype_.lanes_ > 1) { - if (v->mask_.defined()) { - print_type(v->dtype_); - *os << "::mask_load(&"; - dispatch(v->ptr_); - *os << '['; - assert(v->idx_.size() == 1); - dispatch(v->idx_.front()); - *os << "], "; - dispatch(v->mask_); - *os << ')'; - } else { - print_type(v->dtype_); - *os << "::load(&"; - dispatch(v->ptr_); - *os << '['; - assert(v->idx_.size() == 1); - dispatch(v->idx_.front()); - *os << ']' << ')'; - } - } else { - dispatch(v->ptr_); - *os << '['; - assert(v->idx_.size() == 1); - dispatch(v->idx_.front()); - *os << ']'; - } -} - -void codegen_c_vis::view(tensorptr_c v) { - *os << '&'; - dispatch(v->base_); -} - -void codegen_c_vis::view(call_c v) { - func_t the_func = std::dynamic_pointer_cast(v->func_); - expr the_expr; - if (!the_func) { - the_expr = expr(std::dynamic_pointer_cast(v->func_)); - assert(the_expr.defined()); - } - if (!v->para_attr_.empty()) { - assert(v->args_.size() == 1); - assert(v->para_attr_.size() == 1); - assert(the_func); - *os << "sc_parallel_call_cpu(" << the_func->name_ << ", "; - dispatch(v->para_attr_[0].begin_); - *os << ", "; - dispatch(v->para_attr_[0].end_); - *os << ", "; - dispatch(v->para_attr_[0].step_); - *os << ", "; - *os << v->args_[0] << ')'; - } else { - if (the_func) { - if (!is_offline_ && default_external_symbol_resolve(the_func->name_)) { - // if set gcc attributes, call func wrapper instead - if (!the_func->attr_ || - !the_func->attr_->get_or_else(function_attrs::pure, false)) { - *os << the_func->name_ << "_fptr"; - } else { - *os << "__" << the_func->name_ << wrapper_postfix; - } - } else { - *os << the_func->name_; - } - } else { - auto func = the_expr->attr().get_or_else("prototype", func_t()); - COMPILE_ASSERT(func, "Call node expects an expr with prototype"); - *os << '(' << '('; - print_type(func->ret_type_); - *os << "(*)("; - if (!func->params_.empty()) { - for (size_t i = 0; i < func->params_.size() - 1; i++) { - print_type(func->params_[i]->dtype_); - *os << ',' << ' '; - } - print_type(func->params_.back()->dtype_); - } - *os << ')' << ')'; - dispatch(the_expr); - *os << ')'; - } - *os << '('; - if (!v->args_.empty()) { - for (size_t i = 0; i < v->args_.size() - 1; i++) { - expr e = v->args_.at(i); - dispatch(e); - *os << ", "; - } - dispatch(v->args_.back()); - } - *os << ")"; - } -} - -void codegen_c_vis::view(tensor_c v) { *os << v->name_; } - -void codegen_c_vis::view(assign_c v) { - if (v->var_->dtype_.lanes_ > 1 && v->var_.isa()) { - if (v->var_.static_as()->mask_.defined()) { - print_type(v->var_->dtype_); - *os << "::mask_store("; - *os << "&"; - indexing dst = v->var_.static_as(); - dispatch(dst->ptr_); - *os << '['; - dispatch(dst->idx_.front()); - *os << "], "; - dispatch(dst->mask_); - *os << ", "; - dispatch(v->value_); - *os << ");"; - } else { - print_type(v->var_->dtype_); - *os << "::store("; - dispatch(v->value_); - *os << ", &"; - indexing dst = v->var_.static_as(); - dispatch(dst->ptr_); - *os << '['; - dispatch(dst->idx_.front()); - *os << "]);"; - } - } else { - dispatch(v->var_); - *os << " = "; - dispatch(v->value_); - *os << ';'; - } -} - -void codegen_c_vis::view(stmts_c v) { - *os << "{\n"; - indents++; - for (auto &s : v->seq_) { - print_indents(*os, indents); - dispatch(s); - *os << "\n"; - } - indents--; - print_indents(*os, indents); - *os << "}"; -} - -void codegen_c_vis::view(if_else_c v) { - *os << "if ("; - dispatch(v->condition_); - *os << ") "; - dispatch(v->then_case_); - if (v->else_case_.defined()) { - *os << " else "; - dispatch(v->else_case_); - } -} - -void codegen_c_vis::view(evaluate_c v) { - dispatch(v->value_); - *os << ';'; -} - -void codegen_c_vis::view(returns_c v) { - *os << "return "; - if (v->value_.defined()) { - dispatch(v->value_); - } - *os << ';'; -} - -void codegen_c_vis::view(define_c v) { - if (v->linkage_ == linkage::static_local || - v->linkage_ == linkage::private_global) { - *os << "static "; - } - if (v->var_.isa()) { - auto thevar = v->var_.static_as(); - if (v->var_->attr_ && - v->var_->attr_->has_key(attr_keys::module_global_offset)) { - // if it is a global variable that is lowered to local - print_type(thevar->dtype_); - *os << '&' << ' ' << thevar->name_; - auto &offset = v->var_->attr_->get_any(attr_keys::module_global_offset); - *os << " = *("; - print_type(thevar->dtype_); - if (auto ptr = offset.get_or_null()) { - *os << "*)(" << *ptr << ')'; - } else { - *os << "*)(__module_data + " << offset.get() << ')'; - } - } else { - print_cpp_var_def(thevar); - if (v->init_.defined()) { - *os << " = "; - dispatch(v->init_); - } - } - } else if (v->var_.isa()) { - tensor t = v->var_.static_as(); - // if it is a view of the rescheduled buffer/ local tensor on heap - if (v->init_.defined()) { - print_cpp_type(*os, t->elem_dtype_) << "* " << t->name_ << " = ("; - print_cpp_type(*os, t->elem_dtype_) << "*)"; - dispatch(v->init_); - *os << ';'; - return; - } - bool use_heap = false; - // if the tensor is defined as global, do not alloc it on heap - if (v->linkage_ != linkage::public_global && - v->linkage_ != linkage::private_global) { - if (t->dims_.front().isa()) { - auto dim = t->dims_.front().static_as(); - auto sz = get_const_as_int(dim); - if (sz >= 256) { - use_heap = true; - } - } else { - use_heap = true; - } - } - if (use_heap) { - const char *buffertype = "_buffer<"; - if (t->attr_ && t->attr_->has_key("is_thread_buffer") && - t->attr_->get("is_thread_buffer")) { - buffertype = "_thread_buffer<"; - } - *os << buffertype; - print_cpp_type(*os, t->elem_dtype_) - << "> _buf_" << t->name_ << "(__stream, "; - dispatch(t->dims_.front()); - *os << "); "; - print_cpp_type(*os, t->elem_dtype_) - << "* " << t->name_ << " = _buf_" << t->name_ << ".buf"; - } else { - // explicitly align tensor with cache line size, except that tensor - // is a scalar or bytes size < 64. - bool need_align = false; - // check condition. - if (t->dims_.size() == 1 && - get_const_as_int(t->dims_[0].checked_as()) == 1) { - // it is a scalar - } else { - size_t shape = 1; - for (auto &d : t->dims_) { - shape *= get_const_as_int(d.checked_as()); - } - size_t dtsize = utils::get_sizeof_etype(t->elem_dtype_.type_code_); - // check bytes size - if (shape * dtsize > 64) - need_align = true; - } - // cache line alignment - if (need_align) - *os << "alignas(64) "; - - print_cpp_type(*os, t->elem_dtype_) << ' ' << t->name_ << '['; - dispatch(t->dims_.front()); - *os << ']'; - } - } else { - assert(0 && "Bad var type"); - } - *os << ';'; -} - -void codegen_c_vis::view(for_loop_c v) { - if (v->kind_ == for_type::PARALLEL) { - *os << "parallel(("; - dispatch(v->iter_end_); - *os << " - "; - dispatch(v->iter_begin_); - *os << ") / "; - dispatch(v->step_); - *os << ", [&](const int __ithr, const int __nthr) {\n"; - indents++; - print_indents(*os, indents); - auto itrv = v->var_.checked_as(); - print_cpp_var_def(itrv); - *os << " = "; - dispatch(v->iter_begin_); - *os << " + __ithr * "; - dispatch(v->step_); - *os << ";\n"; - print_indents(*os, indents); - dispatch(v->body_); - indents--; - *os << "\n"; - print_indents(*os, indents); - *os << "});"; - } else { - *os << "for ("; - auto itrv = v->var_.checked_as(); - print_cpp_var_def(itrv); - *os << " = "; - dispatch(v->iter_begin_); - *os << "; " << itrv->name_ << " < "; - dispatch(v->iter_end_); - *os << "; " << itrv->name_ << " += "; - dispatch(v->step_); - *os << ") "; - dispatch(v->body_); - } -} - -static void prepare_include(std::ostream *source) { - *source << R"(#include - -)"; -} - -static bool is_main_func_wrapper(const func_c &f) { - return f->attr_ && f->attr_->get_or_else(function_attrs::is_main, false) && - utils::string_endswith(f->name_, wrapper_postfix); -} - -static bool is_func_static(const func_c &f, bool is_offline) { - auto attr = f->attr_.get(); - if (is_offline) { - // if is offline mode, hide all symbols except the main entry - if (is_main_func_wrapper(f)) { - return false; - } - if (f->name_ == "__sc_init__" || f->name_ == "memset") { - return false; - } - return !default_external_symbol_resolve(f->name_); - } - return attr && attr->get_or_else(function_attrs::private_, false); -} - -void write_cpp_prototype(std::ostream *source_, const func_c &f, - bool is_offline) { - codegen_c_vis vis(source_, true, is_func_static(f, is_offline)); - vis.is_offline_ = is_offline; - vis.dispatch(f); - *source_ << '\n'; -} - -void write_cpp_generic_wrapper(std::ostream *source_, const func_c &f, - bool is_parallel) { - *source_ << "extern \"C\" void "; - if (is_parallel) { - *source_ << get_closure_wrapper_name(f->name_) - << "(int64_t i, generic_val* args) {\n "; - } else { - *source_ << f->name_ << "_0wrapper(generic_val* args) {\n "; - } - *source_ << f->name_ << '('; - std::vector::const_iterator itr; - if (is_parallel) { - *source_ << "i"; - if (f->params_.size() > 1) { - *source_ << ", "; - } - assert(!f->params_.empty()); - // skip the first parameter, as it is given by "i" - itr = f->params_.begin() + 1; - } else { - itr = f->params_.begin(); - } - - int idx = 0; - for (; itr != f->params_.end(); ++itr) { - auto &arg = *itr; - if (arg.isa()) { - auto v = arg.static_as(); - *source_ << "args[" << idx << "].v_"; - if (v->dtype_.is_pointer()) { - *source_ << "ptr"; - } else { - print_cpp_type(*source_, v->dtype_); - } - } else { - auto v = arg.checked_as(); - *source_ << '('; - print_cpp_type(*source_, v->elem_dtype_) - << "*)(args[" << idx << "].v_ptr)"; - } - idx++; - if (itr + 1 != f->params_.end()) { - *source_ << ", "; - } - } - *source_ << ");\n}\n\n"; -} - -static func_c do_generate_c(func_c f, std::ostream &source, - const std::vector &globals, - bool gen_wrapper, bool is_static, bool is_offline) { - codegen_c_vis vis(&source, false, is_static); - vis.is_offline_ = is_offline; - vis.dispatch(f); - source << '\n' << '\n'; - return f; -} - -void c_generator_pass_t::operator()(func_t f) { - f = pre_passes_(ir_module_t::from_entry_func(context_, f))->get_entry_func(); - do_generate_c(f, source_, {}, gen_wrapper_, false, false); -} - -c_generator_pass_t::c_generator_pass_t(std::ostream &source, - const context_ptr &ctx, bool gen_wrapper, - c_generator_optional_out_t *optional_out) - : source_(source), context_(ctx), gen_wrapper_(gen_wrapper), - pre_passes_{get_default_precodegen_passes(ctx, gen_wrapper)}, - optional_out_(optional_out) { - prepare_include(&source_); - if (optional_out_) { - prepare_include(optional_out_->offline_source_); - } -} - -const_ir_module_ptr -preprocess_module_and_make_decl(const const_ir_module_ptr &mod, - module_pass_t &pre_passes, std::ostream &source, - c_generator_optional_out_t *optout) { - auto mod_cpy = run_precodegen_passes(pre_passes, mod); - auto timer = - SC_SCOPED_TIMER_INFO("pass.time.c_generator_pass.preprocess", ""); - // stage 1, find and print all function prototypes - std::vector dep; - std::unordered_set depset; - func_dependency_finder_t finder(dep); - for (auto &f : mod_cpy->get_contents()) { - finder(f, depset); - } - for (auto &v : mod_cpy->get_module_vars()) { - finder(v, depset); - } - for (auto &f : dep) { - if (!f->attr_ || !f->attr_->has_key("device_func")) { - write_cpp_prototype(&source, f, false); - } - } - source << '\n'; - source << '\n'; - - if (optout) { - auto mode = mod_cpy->attr_.get( - ir_module_t::attr_key_t::MANAGED_THREAD_POOL); - COMPILE_ASSERT(mode != thread_pool_mode_t::DYNAMIC, - "Dynamic threadpool offline mode is not yet supported."); - bool managed_thread_pool = mode == thread_pool_mode_t::MANAGED; - if (managed_thread_pool) { - (*optout->offline_source_) - << R"(#include -)"; - } - - (*optout->offline_source_) << R"(#include -#define sc_get_thread_id omp_get_thread_num -#define sc_parallel_call_cpu_with_env sc_parallel_call_cpu_with_env_impl -)"; - for (auto &f : dep) { - if (!f->attr_ || !f->attr_->has_key("device_func")) { - write_cpp_prototype(optout->offline_source_, f, true); - } - } - *(optout->offline_source_) << '\n' << '\n'; - } - return mod_cpy; -} - -static void generate_dumped_source(const const_ir_module_ptr &mod, - c_generator_optional_out_t *optional_out, - bool gen_wrapper) { - std::string module_name = "main_entry"; - for (auto &f : mod->get_contents()) { - if (f->attr_ && f->attr_->get_or_else(function_attrs::is_main, false) && - !utils::string_endswith(f->name_, wrapper_postfix)) { - module_name = f->name_; - break; - } - } - - thread_pool_mode_t mode = mod->attr_.get( - ir_module_t::attr_key_t::MANAGED_THREAD_POOL); - COMPILE_ASSERT(mode != thread_pool_mode_t::DYNAMIC, - "Dynamic threadpool offline mode is not yet supported."); - bool managed_thread_pool = mode == thread_pool_mode_t::MANAGED; - for (auto &f : mod->get_contents()) { - if (f->name_ == "__sc_init__") { - f->attr()["temp.replace_func_name"] = "sc_init_" + module_name; - f->attr()["comments"] = std::vector{ - "Initialize the " + module_name, - "@param __stream the stream pointer, usually " - "get_default_stream()", - "@param __module_data the module global data"}; - break; - } - } - auto &header_src = *optional_out->header_source_; - auto &data_src = *optional_out->data_source_; - auto &offline_src = *optional_out->offline_source_; - - auto &mod_data = mod->attr_.get>( - ir_module_t::attr_key_t::MODULE_DATA_BUFFERS); - - header_src << R"(#include -#include -using generic_val = dnnl::impl::graph::gc::generic_val; - -extern uint8_t )" - << module_name << "_data[" << mod_data->data_.size_ << "];\n\n"; - const char *skip_func_name = "__sc_init___0wrapper"; - for (auto &f : mod->get_contents()) { - if (f->name_ == skip_func_name) { - continue; - } - bool is_static_func = is_func_static(f, true); - if (!is_static_func) { - write_cpp_prototype(&header_src, f, true); - } - - // change the name of the generated func to insert managed thread pool - // code - if (managed_thread_pool && is_main_func_wrapper(f)) { - f->attr()["temp.replace_func_name"] = f->name_ + "_impl"; - } - - do_generate_c(f, offline_src, mod->get_module_vars(), gen_wrapper, - is_static_func, true); - } - if (managed_thread_pool) { - offline_src - << "extern \"C\" void " << module_name - << R"(_0wrapper(void* __stream, int8_t* __restrict__ __module_data, generic_val* __restrict__ args) noexcept{ - gc::runtime::thread_manager::cur_mgr.run_main_function((gc::runtime::thread_manager::main_func_t))" - << module_name - << R"(_0wrapper_impl, (gc::runtime::stream_t *)__stream, __module_data, args); -})"; - } - - // generate module data - - memset((char *)mod_data->data_.data_ + mod_data->initialized_size_, 0, - mod_data->data_.size_ - mod_data->initialized_size_); - data_src << R"(#include - -alignas(64) uint8_t )" - << module_name << "_data[" << mod_data->data_.size_ << "] = {"; - uint8_t *buffer = (uint8_t *)mod_data->data_.data_; - for (size_t i = 0; i < mod_data->initialized_size_; i++) { - data_src << (uint32_t)buffer[i] << ','; - } - data_src << "};\n"; -} - -const_ir_module_ptr c_generator_pass_t::operator()(const_ir_module_ptr mod) { - // TODO(xxx): cfake_jit is created with default ctx, which can't pass - // below assertion. assert(mod->ctx_ == context_); - mod = - preprocess_module_and_make_decl(mod, pre_passes_, source_, optional_out_); - auto timer = SC_SCOPED_TIMER_INFO("pass.time.c_generator_pass.codegen", ""); - for (auto &f : mod->get_contents()) { - do_generate_c(f, source_, mod->get_module_vars(), gen_wrapper_, - is_func_static(f, false), false); - } - if (optional_out_) { - generate_dumped_source(mod, optional_out_, gen_wrapper_); - } - return mod; -} - -c_generator_pass_t -create_c_generator(std::ostream &os, const context_ptr &ctx, bool gen_wrapper, - c_generator_optional_out_t *optional_out) { - return c_generator_pass_t(os, ctx, gen_wrapper, optional_out); -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/codegen/codegen_c.hpp b/legacy/core/src/compiler/codegen/codegen_c.hpp deleted file mode 100644 index 80ceb7372..000000000 --- a/legacy/core/src/compiler/codegen/codegen_c.hpp +++ /dev/null @@ -1,78 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_CODEGEN_CODEGEN_C_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_CODEGEN_CODEGEN_C_HPP - -#include - -#include "../ir/ir_module.hpp" -#include "../ir/util_module_passes.hpp" -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -struct c_generator_optional_out_t { - std::ostream *offline_source_; - std::ostream *header_source_; - std::ostream *data_source_; -}; - -class SC_INTERNAL_API c_generator_pass_t : public module_pass_t { -private: - std::ostream &source_; - context_ptr context_; - bool gen_wrapper_; - sequential_module_pass_t pre_passes_; - c_generator_optional_out_t *optional_out_; - -public: - const_ir_module_ptr operator()(const_ir_module_ptr f) override; - /** - * Generates a single function to the stream - * */ - void operator()(func_t f); - - c_generator_pass_t(std::ostream &source, const context_ptr &ctx, - bool gen_wrapper, - c_generator_optional_out_t *optional_out = nullptr); -}; - -/** - * Creates a pass pipeline. It finds all dependent functions of the function. - * For each function, it runs legalization and optimization. Then runs - * codegen-to-c on it. Finally outputs the C++ source code to `os`. In itself, - * the pipeline remembers all functions that has already been generated to - * prevent multi-defining the functions - * - * @param os the stream to output the generated source code - * @param ctx the context - * @param gen_wrapper if true, generates a function "NAME_wrapper" which has - * type erased prototype - * @param optional_out the optional output for AOT offline mode codegen - * */ -SC_INTERNAL_API c_generator_pass_t create_c_generator( - std::ostream &os, const context_ptr &ctx, bool gen_wrapper = false, - c_generator_optional_out_t *optional_out = nullptr); -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/codegen/codegen_c_internal.hpp b/legacy/core/src/compiler/codegen/codegen_c_internal.hpp deleted file mode 100644 index 8d2fd960c..000000000 --- a/legacy/core/src/compiler/codegen/codegen_c_internal.hpp +++ /dev/null @@ -1,109 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_CODEGEN_CODEGEN_C_INTERNAL_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_CODEGEN_CODEGEN_C_INTERNAL_HPP - -#include "../ir/util_module_passes.hpp" -#include "../ir/viewer.hpp" -#include - -/** - * This header exposes some utility functions for C++-like language codegen. - * To use standard C++ codegen, please include codegen_c.hpp instead - * */ - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -class codegen_c_vis : public ir_viewer_t { -protected: - ostream *os; - int indents = 0; - bool prototype_only; - bool is_static; - ostream &print_param(const expr &v); - ostream &print_func_params(const func_c &v, bool with_type = true); - void trinary_func_codegen_c(const std::vector &args, - const char *funcname); - void binary_func_codegen_c(const std::vector &args, - const char *funcname); - void unary_func_codegen_c(const expr &arg, const char *funcname); - -public: - bool is_offline_ = false; - virtual ostream &print_cpp_var_def(const var &v); - virtual ostream &print_tensor_def(const tensor &v); - using ir_viewer_t::dispatch; - using ir_viewer_t::view; - codegen_c_vis(ostream *os, bool prototype_only, bool is_static = false); - func_c dispatch(func_c v) override; - stmt_c dispatch(stmt_c v) override; - void view(constant_c v) override; - void view(var_c v) override; - void view(cast_c v) override; - - void print_binary(const binary_c &v, const char *op); - void print_binary(const logic_c &v, const char *op); - void print_binary(const cmp_c &v, const char *op); - - virtual void print_type(sc_data_type_t dtype); - - void view(add_c v) override; - void view(sub_c v) override; - void view(mul_c v) override; - void view(div_c v) override; - void view(mod_c v) override; - void view(cmp_eq_c v) override; - void view(cmp_lt_c v) override; - void view(cmp_le_c v) override; - void view(cmp_gt_c v) override; - void view(cmp_ge_c v) override; - void view(cmp_ne_c v) override; - void view(logic_and_c v) override; - void view(logic_or_c v) override; - void view(logic_not_c v) override; - void view(select_c v) override; - void view(indexing_c v) override; - void view(tensorptr_c v) override; - void view(intrin_call_c v) override; - void view(func_addr_c v) override; - void view(call_c v) override; - void view(tensor_c v) override; - void view(assign_c v) override; - void view(stmts_c v) override; - void view(if_else_c v) override; - void view(evaluate_c v) override; - void view(returns_c v) override; - void view(define_c v) override; - void view(for_loop_c v) override; -}; - -struct c_generator_optional_out_t; -extern const_ir_module_ptr preprocess_module_and_make_decl( - const const_ir_module_ptr &mod, module_pass_t &pre_passes, - std::ostream &source, c_generator_optional_out_t *optionalout = nullptr); -extern ostream &print_cpp_type(ostream &os, sc_data_type_t dtype); -extern void write_cpp_prototype(std::ostream *source_, const func_c &f, - bool is_offline = false); -extern void write_cpp_generic_wrapper(std::ostream *source_, const func_c &f, - bool is_parallel); -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/codegen/codegen_llvm.cpp b/legacy/core/src/compiler/codegen/codegen_llvm.cpp deleted file mode 100644 index f1cc9ff17..000000000 --- a/legacy/core/src/compiler/codegen/codegen_llvm.cpp +++ /dev/null @@ -1,79 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#include -#include -#include -#include -#include -#include -#include - -#include -// the visitor for lowering TIR to LLVM IR -#include - -using namespace llvm; -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -#if SC_LLVM_BACKEND > 16 -// starting from LLVM17, they use STL's optional container -template using Optional = std::optional; -#endif - -static std::string dump_module_to_string(Module *m) { - std::string ret; - raw_string_ostream os(ret); - os << *m; - return ret; -} - -const_ir_module_ptr llvm_generator_pass::operator()(const_ir_module_ptr f) { - auto passes = get_default_precodegen_passes(f->ctx_, gen_wrapper_); - auto mod = run_precodegen_passes(passes, f); - std::string unique_name; - const auto &tmpdir = utils::compiler_configs_t::get_temp_dir_path(); - if (f->ctx_->flags_.debug_info_) { - std::string file_name; - file_name = "llvm_jit-" + utils::get_unique_name_for_file() + ".gcir"; - std::string unique_name = tmpdir + "/" + file_name; - std::ofstream ofs; - utils::open_file_for_write(ofs, unique_name); - out_source_path_ = unique_name; - print_ir_and_annotate_source_pos(*mod, ofs); - } else { - out_source_path_ = ""; - } - - codegen_llvm_vis_t vis{f->ctx_, llvm_ctx_, tmpdir, out_source_path_}; - auto timer = SC_SCOPED_TIMER_INFO("pass.time.llvm_generator_pass", ""); - for (auto &funct : mod->get_contents()) { - vis.dispatch(funct); - } - if (f->ctx_->flags_.debug_info_) { - vis.dbuilder_->finalize(); - } - out_module_ = std::move(vis.module_); - SC_MODULE_INFO << dump_module_to_string(out_module_.get()); - return mod; -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/codegen/codegen_llvm.hpp b/legacy/core/src/compiler/codegen/codegen_llvm.hpp deleted file mode 100644 index aad9fbc74..000000000 --- a/legacy/core/src/compiler/codegen/codegen_llvm.hpp +++ /dev/null @@ -1,49 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_CODEGEN_CODEGEN_LLVM_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_CODEGEN_CODEGEN_LLVM_HPP - -#include -#include - -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -class SC_INTERNAL_API llvm_generator_pass : public module_pass_t { -public: - llvm::LLVMContext &llvm_ctx_; - std::unique_ptr &out_module_; - bool gen_wrapper_; - std::string out_source_path_; - const_ir_module_ptr operator()(const_ir_module_ptr f) override; - llvm_generator_pass(llvm::LLVMContext &llvm_ctx, - std::unique_ptr &out_module, - bool gen_wrapper) - : llvm_ctx_(llvm_ctx), out_module_(out_module), - gen_wrapper_(gen_wrapper) {} -}; -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/codegen/llvm/basic_expr.cpp b/legacy/core/src/compiler/codegen/llvm/basic_expr.cpp deleted file mode 100644 index e65b98a39..000000000 --- a/legacy/core/src/compiler/codegen/llvm/basic_expr.cpp +++ /dev/null @@ -1,469 +0,0 @@ -/******************************************************************************* - * Copyright 2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#include -#include -#include -#include -#include -#include -#include -#include - -#include "compiler/ir/attr_keys.hpp" -#include "shared_include.hpp" -#include "util/fp16.hpp" - -using namespace llvm; -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -void codegen_llvm_vis_t::generate_bin_op(const expr_c &v, const expr_c &l, - const expr_c &r) { - auto cate = get_etype_category_nothrow(l->dtype_.type_code_); - auto lhs = generate_expr(l); - auto rhs = generate_expr(r); - auto original_fmf = builder_.getFastMathFlags(); - if (!any_map_t::fetch_or_else(v->attr_.get(), attr_keys::fast_math, true)) { - FastMathFlags fmflag; - fmflag.setFast(false); - fmflag.setAllowContract(false); - builder_.setFastMathFlags(fmflag); - } -#define HANDLE_BIN_OP2(scname, intname, fpname, case_other) \ - case sc_expr_type::scname: \ - switch (cate) { \ - case CATE_INT: \ - case CATE_UINT: \ - current_val_ = builder_.Create##intname(lhs, rhs); \ - break; \ - case CATE_FLOAT: \ - current_val_ = builder_.Create##fpname(lhs, rhs); \ - break; \ - case CATE_OTHER: { \ - case_other \ - } \ - default: \ - COMPILE_ASSERT(false, "Cannot generate binary op for this type: " << v); \ - } \ - break -#define HANDLE_POINTER_CMP(llvm_name) \ - if (l->dtype_.type_code_ == sc_data_etype::POINTER && \ - r->dtype_.type_code_ == sc_data_etype::POINTER) { \ - current_val_ = builder_.Create##llvm_name(lhs, rhs); \ - break; \ - } -#define HANDLE_POINTER_EQ HANDLE_POINTER_CMP(ICmpEQ) -#define HANDLE_POINTER_NE HANDLE_POINTER_CMP(ICmpNE) -#define HANDLE_BIN_OP(scname, intname) \ - HANDLE_BIN_OP2(scname, intname, F##intname, ) -#define HANDLE_BIN_SIGNED_OP2(scname, intname1, intname2, fpname) \ - case sc_expr_type::scname: \ - switch (cate) { \ - case CATE_INT: \ - current_val_ = builder_.Create##intname1##S##intname2(lhs, rhs); \ - break; \ - case CATE_UINT: \ - current_val_ = builder_.Create##intname1##U##intname2(lhs, rhs); \ - break; \ - case CATE_FLOAT: \ - current_val_ = builder_.Create##fpname(lhs, rhs); \ - break; \ - default: \ - COMPILE_ASSERT(false, "Cannot generate binary op for this type: " << v); \ - } \ - break -#define HANDLE_BIN_SIGNED_OP(scname, intname) \ - HANDLE_BIN_SIGNED_OP2(scname, , intname, F##intname) -#define HANDLE_CMP_SIGNED_OP(scname, llvmname) \ - HANDLE_BIN_SIGNED_OP2(scname, ICmp, llvmname, FCmpO##llvmname) - switch (v->node_type_) { - HANDLE_BIN_OP(add, Add); - HANDLE_BIN_OP(sub, Sub); - HANDLE_BIN_OP(mul, Mul); - HANDLE_BIN_SIGNED_OP(div, Div); - HANDLE_BIN_SIGNED_OP(mod, Rem); - HANDLE_BIN_OP2(cmp_eq, ICmpEQ, FCmpOEQ, HANDLE_POINTER_EQ); - HANDLE_BIN_OP2(cmp_ne, ICmpNE, FCmpONE, HANDLE_POINTER_NE); - HANDLE_CMP_SIGNED_OP(cmp_lt, LT); - HANDLE_CMP_SIGNED_OP(cmp_le, LE); - HANDLE_CMP_SIGNED_OP(cmp_gt, GT); - HANDLE_CMP_SIGNED_OP(cmp_ge, GE); - case sc_expr_type::logic_and: - current_val_ = builder_.CreateAnd(lhs, rhs); - break; - case sc_expr_type::logic_or: - current_val_ = builder_.CreateOr(lhs, rhs); - break; - default: - assert(0); - } - builder_.setFastMathFlags(original_fmf); -} - -void codegen_llvm_vis_t::view(binary_c v) { generate_bin_op(v, v->l_, v->r_); } -void codegen_llvm_vis_t::view(cmp_c v) { generate_bin_op(v, v->l_, v->r_); } - -void codegen_llvm_vis_t::view(logic_c v) { generate_bin_op(v, v->l_, v->r_); } - -void codegen_llvm_vis_t::view(constant_c v) { - std::vector vals; - vals.reserve(v->value_.size()); - auto cate = get_etype_category_nothrow(v->dtype_.type_code_); - bool is_bf16 = v->dtype_.type_code_ == sc_data_etype::BF16; - bool is_f16 = v->dtype_.type_code_ == sc_data_etype::F16; - sc_data_type_t base_type = v->dtype_; - base_type.lanes_ = 1; - auto llvm_base_type = get_type(base_type); - switch (cate) { - case CATE_FLOAT: { - for (auto &val : v->value_) { - if (is_bf16) { - uint64_t val_u64 = bf16_t(val.f32).storage_; - vals.push_back(ConstantInt::get(llvm_base_type, val_u64, false)); - } else if (is_f16) { - uint16_t val_u16 = fp16_t(val.f32).storage_; - vals.push_back(ConstantFP::get( - llvm_base_type, APFloat(APFloat::IEEEhalf(), APInt(16, val_u16)))); - } else { - vals.push_back(ConstantFP::get(llvm_base_type, APFloat(val.f32))); - } - } - } break; - case CATE_UINT: - case CATE_INT: { - bool is_signed = cate == CATE_INT; - for (auto &val : v->value_) { - vals.push_back(ConstantInt::get(llvm_base_type, val.u64, is_signed)); - } - } break; - default: - COMPILE_ASSERT(v->dtype_.is_pointer(), - "Unexpected type for LLVM. Expecting pointer."); - for (auto &val : v->value_) { - vals.push_back(ConstantExpr::getIntToPtr(builder_.getInt64(val.u64), - llvm_base_type)); - } - break; - } - if (vals.size() != v->dtype_.lanes_) { - COMPILE_ASSERT(vals.size() == 1, "Bad constant node. Expecting 1 value"); - // broadcast value - current_val_ = builder_.CreateVectorSplat(v->dtype_.lanes_, vals.front()); - } else { - if (vals.size() != 1) { - current_val_ = ConstantVector::get(vals); - } else { - current_val_ = vals.front(); - } - } -} - -void codegen_llvm_vis_t::view(var_c v) { - auto ret = get_defined_var_ptr(v); - bool is_global_base = utils::string_startswith(v->name_, "__module_data"); - bool is_special_params = (is_global_base || v->name_ == "__stream"); - if (is_lvalue_mode_) { - assert(!is_special_params); - is_lvalue_mode_ = false; - current_val_ = ret; - } else { - if (is_special_params) { - current_val_ = ret; - } else { - current_val_ = - builder_.CreateLoad(get_type(v->dtype_), ret, v->name_ + "_v"); - } - } -} - -llvm::Type *codegen_llvm_vis_t::get_llvm_bf16_native_type(unsigned lanes) { -#if SC_LLVM_BACKEND > 15 - // llvm 16's bf16 casting intrinsic uses bf16 instead of i16 - return VectorType::get(builder_.getBFloatTy(), lanes, false); -#else - return get_type(sc_data_type_t::bf16(lanes)); -#endif -} - -void codegen_llvm_vis_t::view(cast_c v) { - auto cate_out = get_etype_category_nothrow(v->dtype_.type_code_); - auto cate_in = get_etype_category_nothrow(v->in_->dtype_.type_code_); - auto in_v = generate_expr(v->in_); - auto outtype = get_type(v->dtype_); - auto check_cate = [&v]() { - COMPILE_ASSERT(v->dtype_ == datatypes::generic, "Unexpected outtype " << v); - }; - - if (v->in_->dtype_.is_etype(sc_data_etype::F32) && - v->dtype_.is_etype(sc_data_etype::BF16)) { -#if SC_LLVM_BACKEND > 10 - switch (v->in_->dtype_.lanes_) { - case 1: { - Value *vec = builder_.CreateVectorSplat(4, in_v); - vec = builder_.CreateIntrinsic( - Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128, {}, - {vec, UndefValue::get(get_llvm_bf16_native_type(8)), - /*mask*/ - builder_.CreateVectorSplat(4, builder_.getInt1(true))}); - current_val_ = builder_.CreateExtractElement(vec, UINT64_C(0)); - } break; - case 4: - current_val_ = builder_.CreateIntrinsic( - Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128, {}, - {in_v, UndefValue::get(get_llvm_bf16_native_type(8)), - /*mask*/ - builder_.CreateVectorSplat(4, builder_.getInt1(true))}); -#if SC_LLVM_BACKEND == 11 - current_val_ = builder_.CreateShuffleVector(current_val_, current_val_, - ArrayRef({0, 1, 2, 3})); -#else - current_val_ = builder_.CreateShuffleVector(current_val_, {0, 1, 2, 3}); -#endif - break; - case 8: - current_val_ = builder_.CreateIntrinsic( - Intrinsic::x86_avx512bf16_cvtneps2bf16_256, {}, {in_v}); - break; - case 16: - current_val_ = builder_.CreateIntrinsic( - Intrinsic::x86_avx512bf16_cvtneps2bf16_512, {}, {in_v}); - break; - default: - std::stringstream ss; - ss << "Unsupport cast lanes " << v->in_->dtype_.lanes_; - throw std::runtime_error(ss.str()); - } -#if SC_LLVM_BACKEND > 15 - // llvm 16's bf16 casting intrinsic returns bf16 instead of i16 - current_val_ = builder_.CreateBitCast(current_val_, outtype); -#endif - return; -#else - throw std::runtime_error("Current version of LLVM cannot handle bf16"); -#endif - } - switch (cate_in) { - case CATE_FLOAT: { - switch (cate_out) { - case CATE_FLOAT: - current_val_ = builder_.CreateFPCast(in_v, outtype); - break; - case CATE_INT: - current_val_ = builder_.CreateFPToSI(in_v, outtype); - break; - case CATE_UINT: - current_val_ = builder_.CreateFPToUI(in_v, outtype); - break; - case CATE_OTHER: { - check_cate(); - auto bits = utils::get_sizeof_etype(v->in_->dtype_.type_code_) * 8; - auto ret = builder_.CreateBitCast(in_v, IntegerType::get(context_, bits)); - current_val_ = builder_.CreateZExtOrBitCast(ret, builder_.getInt64Ty()); - } break; - } - } break; - case CATE_INT: { - switch (cate_out) { - case CATE_FLOAT: - current_val_ = builder_.CreateSIToFP(in_v, outtype); - break; - case CATE_INT: - case CATE_UINT: - current_val_ = builder_.CreateSExtOrTrunc(in_v, outtype); - break; - case CATE_OTHER: { - check_cate(); - current_val_ = builder_.CreateZExtOrBitCast(in_v, builder_.getInt64Ty()); - } break; - } - } break; - case CATE_UINT: { - switch (cate_out) { - case CATE_FLOAT: - current_val_ = builder_.CreateUIToFP(in_v, outtype); - break; - case CATE_INT: - case CATE_UINT: - current_val_ = builder_.CreateZExtOrTrunc(in_v, outtype); - break; - case CATE_OTHER: { - if (v->dtype_.is_pointer()) { - current_val_ = builder_.CreateIntToPtr(in_v, outtype); - } else { - check_cate(); - current_val_ = - builder_.CreateZExtOrBitCast(in_v, builder_.getInt64Ty()); - } - } break; - } - } break; - case CATE_OTHER: - if (v->in_->dtype_ == datatypes::generic) { - auto bits = module_->getDataLayout().getTypeAllocSizeInBits(outtype); - auto ret = - builder_.CreateTruncOrBitCast(in_v, IntegerType::get(context_, bits)); - switch (cate_out) { - case CATE_OTHER: - COMPILE_ASSERT(v->dtype_.is_pointer(), "Unexpected out type " << v); - current_val_ = builder_.CreateIntToPtr(ret, outtype); - break; - case CATE_FLOAT: - current_val_ = builder_.CreateBitCast(ret, outtype); - break; - case CATE_INT: - case CATE_UINT: - current_val_ = ret; - break; - } - } else { - COMPILE_ASSERT(v->in_->dtype_.is_pointer(), "Unexpected in type " << v); - if (v->dtype_.is_pointer()) { - // pointer to pointer - current_val_ = builder_.CreatePointerCast(in_v, outtype); - } else { - // pointer to generic val or uint64 - if (v->dtype_ != datatypes::index) { - check_cate(); - } - current_val_ = builder_.CreatePtrToInt(in_v, builder_.getInt64Ty()); - } - } - break; - } -} - -void codegen_llvm_vis_t::view(logic_not_c v) { - current_val_ = builder_.CreateNot(generate_expr(v->in_)); -} -void codegen_llvm_vis_t::view(select_c v) { - auto l = generate_expr(v->l_); - auto r = generate_expr(v->r_); - auto cond = convert_mask(v->cond_, v->l_->dtype_.lanes_ == 4); - current_val_ = builder_.CreateSelect(cond, l, r); -} - -void codegen_llvm_vis_t::view(indexing_c v) { - bool is_lvalue_mode = is_lvalue_mode_; - is_lvalue_mode_ = false; - COMPILE_ASSERT(v->idx_.size() == 1, "Expecting 1D array: " << v); - auto base = generate_expr(v->ptr_); - assert(v->ptr_->dtype_.is_pointer()); - auto element_type = get_type(v->ptr_->dtype_.get_pointer_element()); - auto ptr = - builder_.CreateGEP(element_type, base, generate_expr(v->idx_.front())); - auto target_type = get_type(v->dtype_); - if (target_type != element_type) { - // allow pointer to pointer - assert(v->dtype_ == datatypes::pointer || - llvm::cast(*target_type).getElementType() == - element_type); - ptr = builder_.CreatePointerCast(ptr, target_type->getPointerTo()); - } - if (is_lvalue_mode) { - current_val_ = ptr; - } else { - bool is_volatile = - (v->ptr_->attr_ && v->ptr_->attr_->get_or_else("volatile", false)); - if (v->mask_.defined()) { - Value *mask; - auto bit_len = utils::get_sizeof_type(v->mask_->dtype_) * 8; - if (v->dtype_.lanes_ != bit_len) { - COMPILE_ASSERT(v->dtype_.lanes_ == 4, - "Currently only 8bit -> 4bit is supported, but get " - << v->dtype_ << " lanes = " << v->dtype_.lanes_); - mask = convert_mask(v->mask_, true); - } else { - mask = convert_mask(v->mask_); - } - auto znode = make_expr(0UL, v->dtype_); - auto zero = generate_expr(znode); -#if SC_LLVM_BACKEND > 12 - current_val_ = - set_alias(builder_.CreateMaskedLoad(get_type(v->dtype_), ptr, - SC_LLVM_ALIGN(1), mask, zero), - v->ptr_); -#elif SC_LLVM_BACKEND >= 11 - current_val_ = set_alias( - builder_.CreateMaskedLoad(ptr, SC_LLVM_ALIGN(1), mask, zero), - v->ptr_); -#else - current_val_ = - set_alias(builder_.CreateMaskedLoad(ptr, 1, mask, zero), v->ptr_); -#endif - } else { - if (v->dtype_.lanes_ > 1) { - current_val_ = - set_alias(builder_.CreateAlignedLoad(get_type(v->dtype_), ptr, - SC_LLVM_ALIGN(1), is_volatile), - v->ptr_); - } else { - current_val_ = set_alias( - builder_.CreateLoad(get_type(v->dtype_), ptr, is_volatile), - v->ptr_); - } - } - } -} -void codegen_llvm_vis_t::view(tensorptr_c v) { - is_lvalue_mode_ = true; - current_val_ = generate_expr(v->base_); -} - -void codegen_llvm_vis_t::view(func_addr_c v) { - current_val_ = builder_.CreatePointerCast(get_or_create_func(v->func_), - builder_.getInt8PtrTy()); -} -void codegen_llvm_vis_t::view(call_c v) { - std::vector args; - auto the_func = std::dynamic_pointer_cast(v->func_); - Value *ll_func; - FunctionType *ft = nullptr; - if (the_func) { - auto F = get_or_create_func(the_func); - ll_func = F; - ft = F->getFunctionType(); - } else { - auto the_expr = std::dynamic_pointer_cast(v->func_); - assert(the_expr); - auto proto_func = the_expr->attr().get_or_else("prototype", func_t()); - COMPILE_ASSERT(proto_func, "Call node expects an expr with prototype"); - ft = create_func_type(proto_func); - ll_func = generate_expr(expr_c(the_expr)); - ll_func = builder_.CreatePointerCast(ll_func, ft->getPointerTo()); - } - for (size_t i = 0; i < v->args_.size(); i++) { - auto &val = v->args_[i]; - auto ll_value = generate_expr(val); - auto target_type = *(ft->param_begin() + i); - if (ll_value->getType() != target_type) { - COMPILE_ASSERT(target_type == builder_.getInt8PtrTy(), - "LLVM can only handle autocast to pointer"); - ll_value = builder_.CreatePointerCast(ll_value, target_type); - } - args.push_back(ll_value); - } - current_val_ = builder_.CreateCall(ft, ll_func, args); -} -void codegen_llvm_vis_t::view(tensor_c v) { - current_val_ = get_defined_var_ptr(v); -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/codegen/llvm/intrinsic.cpp b/legacy/core/src/compiler/codegen/llvm/intrinsic.cpp deleted file mode 100644 index 65a91a3e0..000000000 --- a/legacy/core/src/compiler/codegen/llvm/intrinsic.cpp +++ /dev/null @@ -1,881 +0,0 @@ -/******************************************************************************* - * Copyright 2023-2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#include -#include -#include -#include -#include -#include -#include -#include - -#include "shared_include.hpp" -#include "util/utils.hpp" - -using namespace llvm; -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -/** - * Implementing LLVM-x86 intrinsics - * 1. first find the GCC/Clang built-in intrinsic name in - * https://github.com/llvm/llvm-project/blob/main/clang/lib/Headers - * e.g. Goto definition of _mm512_cvtusepi32_epi8, you will get - * __builtin_ia32_pmovusdb512_mask - * 2. Find the built-in function name in - * https://github.com/llvm/llvm-project/blob/main/llvm/include/llvm/IR/IntrinsicsX86.td - * 3. Now we have the intrinsic name in LLVM : - * x86_avx512_mask_pmovus_db_512 - * */ -void codegen_llvm_vis_t::view(intrin_call_c v) { - auto cate = get_etype_category_nothrow(v->dtype_.type_code_); - - switch (v->type_) { - case intrin_type::reinterpret: { - assert(v->args_.size() == 1); - auto inval = generate_expr(v->args_[0]); - auto outty = get_type(v->dtype_); - if (outty->isPointerTy()) { - auto src_cate = get_type_category_nothrow(v->args_[0]->dtype_); - bool is_src_int = src_cate == CATE_INT || src_cate == CATE_UINT; - if (is_src_int) { - current_val_ = builder_.CreateIntToPtr(inval, outty); - } else { - current_val_ = builder_.CreatePointerCast(inval, outty); - } - } else if (inval->getType()->isPointerTy()) { - auto dst_cate = get_type_category_nothrow(v->dtype_); - bool is_dest_int = dst_cate == CATE_INT || dst_cate == CATE_UINT; - COMPILE_ASSERT(is_dest_int, "Expecting pointer to int for reinterpret"); - current_val_ = builder_.CreatePtrToInt(inval, outty); - } else { - current_val_ = builder_.CreateBitCast(inval, outty); - } - } break; - case intrin_type::abs: { - assert(v->args_.size() == 1); - auto inval = generate_expr(v->args_[0]); - - std::string intrin_name; - llvm::raw_string_ostream os(intrin_name); - switch (cate) { - case CATE_FLOAT: - current_val_ = builder_.CreateUnaryIntrinsic(Intrinsic::fabs, inval); - break; - case CATE_INT: { - auto znode = make_expr(0UL, v->args_[0]->dtype_); - auto zero = generate_expr(znode); - auto sign = builder_.CreateICmpSGT(inval, zero); - auto neg = builder_.CreateSub(zero, inval); - current_val_ = builder_.CreateSelect(sign, inval, neg); - } break; - default: - assert(0); - break; - } - } break; - case intrin_type::sqrt: { - current_val_ = call_unary_llvm_intrin(v, cate, Intrinsic::sqrt, true); - } break; - case intrin_type::rsqrt: { - // todo: The precision of AVX2 intrinsic is ~0.000366, not meet - // f32 precision judgements. Use fast-math pass to enable - // avx2/sse intrinsic with low precision(bf16/int8) - if (ctx_->machine_.cpu_flags_.fAVX512F && v->dtype_.lanes_ > 1) { - auto inval = generate_expr(v->args_[0]); - switch (v->dtype_.lanes_) { - case 4: - current_val_ = - builder_.CreateIntrinsic(Intrinsic::x86_avx512_rsqrt14_ps_128, {}, - {inval, inval, builder_.getInt8(0xff)}); - - break; - case 8: - current_val_ = - builder_.CreateIntrinsic(Intrinsic::x86_avx512_rsqrt14_ps_256, {}, - {inval, inval, builder_.getInt8(0xff)}); - - break; - case 16: - COMPILE_ASSERT(ctx_->machine_.cpu_flags_.fAVX512F, - "rsqrt of 16 floats needs AVX512"); - current_val_ = - builder_.CreateIntrinsic(Intrinsic::x86_avx512_rsqrt14_ps_512, {}, - {inval, inval, builder_.getInt16(0xffff)}); - break; - default: - COMPILE_ASSERT(false, "LLVM backend has not yet support " - "rsqrt with lanes = " - << v->dtype_.lanes_); - break; - } - } else { - current_val_ = call_unary_llvm_intrin(v, cate, Intrinsic::sqrt, true); - Value *ones = ConstantFP::get(builder_.getFloatTy(), APFloat(1.0f)); - if (v->dtype_.lanes_ > 1) { - ones = builder_.CreateVectorSplat(v->dtype_.lanes_, ones); - } - current_val_ = builder_.CreateFDiv(ones, current_val_); - } - } break; - case intrin_type::int_and: { - current_val_ = call_binary_llvm_normal(v, &llvm::IRBuilder<>::CreateAnd); - } break; - case intrin_type::int_or: { - current_val_ = call_binary_llvm_normal(v, &llvm::IRBuilder<>::CreateOr); - } break; - case intrin_type::int_xor: { - current_val_ = call_binary_llvm_normal(v, &llvm::IRBuilder<>::CreateXor); - } break; - case intrin_type::round: { - current_val_ = call_unary_llvm_intrin(v, cate, Intrinsic::nearbyint, true); - } break; - case intrin_type::ceil: { - current_val_ = call_unary_llvm_intrin(v, cate, Intrinsic::ceil, true); - } break; - case intrin_type::floor: { - current_val_ = call_unary_llvm_intrin(v, cate, Intrinsic::floor, true); - } break; - case intrin_type::log: { - current_val_ = call_unary_llvm_intrin(v, cate, Intrinsic::log, true); - } break; - case intrin_type::max: { - if (cate == CATE_FLOAT) { - current_val_ = call_binary_llvm_intrin(v, cate, Intrinsic::maxnum, true); - } else { - current_val_ = make_int_min_max(v, false, cate); - } - } break; - case intrin_type::min: { - if (cate == CATE_FLOAT) { - current_val_ = call_binary_llvm_intrin(v, cate, Intrinsic::minnum, true); - } else { - current_val_ = make_int_min_max(v, true, cate); - } - } break; - case intrin_type::shl: { - assert(v->args_.size() == 2); - auto inval1 = generate_expr(v->args_[0]); - auto inval2 = generate_expr(v->args_[1]); - COMPILE_ASSERT(cate == CATE_INT || cate == CATE_UINT, - "Bad type. Expecting int: " << v); - current_val_ = builder_.CreateShl(inval1, inval2); - } break; - case intrin_type::shr: { - assert(v->args_.size() == 2); - auto inval1 = generate_expr(v->args_[0]); - auto inval2 = generate_expr(v->args_[1]); - if (cate == CATE_INT) { - current_val_ = builder_.CreateAShr(inval1, inval2); - } else { - COMPILE_ASSERT(cate == CATE_UINT, "Bad type. Expecting int: " << v); - current_val_ = builder_.CreateLShr(inval1, inval2); - } - } break; - case intrin_type::broadcast: { - assert(v->args_.size() == 1); - auto inval1 = generate_expr(v->args_[0]); - auto lanes = v->dtype_.lanes_; - auto in_lanes = v->args_[0]->dtype_.lanes_; - if (lanes != 1) { - if (in_lanes != 1) { - while (in_lanes < lanes) { - std::vector array(in_lanes << 1); - for (uint16_t i = 0; i < (in_lanes << 1); i++) { - array[i] = i; - } - inval1 = builder_.CreateShuffleVector(inval1, inval1, array); - in_lanes = in_lanes << 1; - } - current_val_ = inval1; - } else { - current_val_ = builder_.CreateVectorSplat(lanes, inval1); - } - } else { - current_val_ = inval1; - } - } break; - case intrin_type::fmadd: - case intrin_type::fnmadd: { - assert(v->args_.size() == 3); - auto inval1 = generate_expr(v->args_[0]); - auto inval2 = generate_expr(v->args_[1]); - auto inval3 = generate_expr(v->args_[2]); - if (v->type_ == intrin_type::fnmadd) { - inval1 = builder_.CreateFNeg(inval1); - } - auto ret = builder_.CreateIntrinsic(Intrinsic::fma, {get_type(v->dtype_)}, - {inval1, inval2, inval3}); - ret->setFastMathFlags(builder_.getFastMathFlags()); - current_val_ = ret; - } break; - case intrin_type::reduce_add: - case intrin_type::reduce_mul: - case intrin_type::reduce_max: - case intrin_type::reduce_min: { - Intrinsic::ID cur_intrinsic = Intrinsic::not_intrinsic; -#if SC_LLVM_BACKEND > 11 -#define LLVM_INTRINSIC_EXP_V2(name) Intrinsic::vector_reduce_##name -#define LLVM_INTRINSIC_EXP LLVM_INTRINSIC_EXP_V2 -#elif SC_LLVM_BACKEND > 8 -#define LLVM_INTRINSIC_EXP_V2(name) \ - Intrinsic::experimental_vector_reduce_v2_##name -#define LLVM_INTRINSIC_EXP(name) Intrinsic::experimental_vector_reduce_##name -#else -#define LLVM_INTRINSIC_EXP_V2(name) Intrinsic::experimental_vector_reduce_##name -#define LLVM_INTRINSIC_EXP LLVM_INTRINSIC_EXP_V2 -#endif - if (v->type_ == intrin_type::reduce_add) { - if (cate == CATE_FLOAT) { - cur_intrinsic = LLVM_INTRINSIC_EXP_V2(fadd); - } else { - cur_intrinsic = LLVM_INTRINSIC_EXP(add); - } - } else if (v->type_ == intrin_type::reduce_mul) { - if (cate == CATE_FLOAT) { - cur_intrinsic = LLVM_INTRINSIC_EXP_V2(fmul); - } else { - cur_intrinsic = LLVM_INTRINSIC_EXP(mul); - } - } else if (v->type_ == intrin_type::reduce_max) { - if (cate == CATE_FLOAT) { - cur_intrinsic = LLVM_INTRINSIC_EXP(fmax); - } else if (cate == CATE_INT) { - cur_intrinsic = LLVM_INTRINSIC_EXP(smax); - } else if (cate == CATE_UINT) { - cur_intrinsic = LLVM_INTRINSIC_EXP(umax); - } - } else if (v->type_ == intrin_type::reduce_min) { - if (cate == CATE_FLOAT) { - cur_intrinsic = LLVM_INTRINSIC_EXP(fmin); - } else if (cate == CATE_INT) { - cur_intrinsic = LLVM_INTRINSIC_EXP(smin); - } else if (cate == CATE_UINT) { - cur_intrinsic = LLVM_INTRINSIC_EXP(umin); - } - } - assert(v->args_.size() == 1); - auto inval = generate_expr(v->args_[0]); - if ((v->type_ == intrin_type::reduce_add || - v->type_ == intrin_type::reduce_mul) && - cate == CATE_FLOAT) { - bool is_fp16 = v->args_[0]->dtype_.is_etype(sc_data_etype::F16); - auto start_val = - v->type_ == intrin_type::reduce_mul ? APFloat(1.0f) : APFloat(0.0f); - auto fp16_start_val = v->type_ == intrin_type::reduce_mul - ? static_cast(0x3C00) - : static_cast(0); - current_val_ = builder_.CreateIntrinsic( -#if SC_LLVM_BACKEND > 11 - cur_intrinsic, {inval->getType()}, -#elif SC_LLVM_BACKEND > 8 - cur_intrinsic, {get_type(v->dtype_), inval->getType()}, -#else - cur_intrinsic, - {get_type(v->dtype_), get_type(v->dtype_), inval->getType()}, -#endif - {ConstantFP::get( - get_type(v->dtype_), - is_fp16 ? APFloat(APFloat::IEEEhalf(), APInt(16, fp16_start_val)) - : start_val), - inval}); - llvm::cast(*current_val_) - .setFastMathFlags(builder_.getFastMathFlags()); - } else { - current_val_ = builder_.CreateIntrinsic( -#if SC_LLVM_BACKEND >= 10 - cur_intrinsic, {inval->getType()}, -#else - cur_intrinsic, {get_type(v->dtype_), inval->getType()}, -#endif - {inval}); - } - } break; - case intrin_type::saturated_cast: { - current_val_ = do_lower_saturated_cast(v); - } break; - case intrin_type::round_and_cast: { - assert(v->args_.size() == 1); - auto inval1 = generate_expr(v->args_[0]); - COMPILE_ASSERT((v->dtype_.type_code_ == sc_data_etype::S32 || - (v->dtype_.type_code_ == sc_data_etype::U32 && - v->dtype_.lanes_ > 1)) && - v->args_[0]->dtype_.type_code_ == sc_data_etype::F32, - "LLVM backend has not yet support round_and_cast " - "like " - "this: " - << v); - bool is_u32 = v->dtype_.type_code_ == sc_data_etype::U32; - if (is_u32) { - COMPILE_ASSERT(ctx_->machine_.cpu_flags_.fAVX512F, - "round_and_cast of u32 needs AVX512"); - } - switch (v->dtype_.lanes_) { - case 1: { - current_val_ = builder_.CreateFPToSI( - builder_.CreateUnaryIntrinsic(Intrinsic::roundeven, inval1), - builder_.getInt32Ty()); - } break; - case 4: { - if (is_u32) { - current_val_ = builder_.CreateIntrinsic( - Intrinsic::x86_avx512_mask_cvtps2udq_128, {}, - {inval1, UndefValue::get(get_type(v->dtype_)), - /*mask*/ builder_.getInt8(0xff), builder_.getInt32(0x04)}); - } else { - current_val_ = - builder_.CreateIntrinsic(Intrinsic::x86_sse2_cvtps2dq, {}, inval1); - } - } break; - case 8: { - if (is_u32) { - current_val_ = builder_.CreateIntrinsic( - Intrinsic::x86_avx512_mask_cvtps2udq_256, {}, - {inval1, UndefValue::get(get_type(v->dtype_)), - /*mask*/ builder_.getInt8(0xff), builder_.getInt32(0x04)}); - } else { - current_val_ = builder_.CreateIntrinsic( - Intrinsic::x86_avx_cvt_ps2dq_256, {}, inval1); - } - } break; - case 16: { - COMPILE_ASSERT(ctx_->machine_.cpu_flags_.fAVX512F, - "round_and_cast of 16 floats needs AVX512"); - current_val_ = builder_.CreateIntrinsic( - is_u32 ? Intrinsic::x86_avx512_mask_cvtps2udq_512 - : Intrinsic::x86_avx512_mask_cvtps2dq_512, - {}, - {inval1, UndefValue::get(get_type(v->dtype_)), - /*mask*/ builder_.getInt16(0xffff), - /*rounding mode = - _MM_FROUND_CUR_DIRECTION 0x04*/ - builder_.getInt32(0x04)}); - } break; - default: - COMPILE_ASSERT(false, "LLVM backend has not yet support " - "round_and_cast with lanes = " - << v->dtype_.lanes_); - break; - } - } break; - case intrin_type::permutex2var: { - assert(v->args_.size() == 3); - auto inval1 = generate_expr(v->args_[0]); - auto inval2 = generate_expr(v->args_[1]); - auto inval3 = generate_expr(v->args_[2]); - switch (v->args_[0]->dtype_.type_code_) { - case sc_data_etype::F32: - switch (v->args_[0]->dtype_.lanes_) { - case 4: - current_val_ = - builder_.CreateIntrinsic(Intrinsic::x86_avx512_vpermi2var_ps_128, - {}, {inval1, inval2, inval3}); - break; - case 8: - current_val_ = - builder_.CreateIntrinsic(Intrinsic::x86_avx512_vpermi2var_ps_256, - {}, {inval1, inval2, inval3}); - break; - case 16: - current_val_ = - builder_.CreateIntrinsic(Intrinsic::x86_avx512_vpermi2var_ps_512, - {}, {inval1, inval2, inval3}); - break; - default: - COMPILE_ASSERT(false, "Unimplement lanes for permute2var: " - << v->args_[0]->dtype_.lanes_); - } - break; - case sc_data_etype::U8: - current_val_ = - builder_.CreateIntrinsic(Intrinsic::x86_avx512_vpermi2var_qi_128, {}, - {inval1, inval2, inval3}); - break; - default: - COMPILE_ASSERT(false, "Unimplement datatype for permute2var"); - } - } break; - case intrin_type::permutexvar: { - COMPILE_ASSERT(v->args_.size() == 2, - "need two args in permutexvar instructions."); - auto inval1 = generate_expr(v->args_[0]); - auto inval2 = generate_expr(v->args_[1]); - int lanes = v->intrin_attrs_->get("lanes"); - auto elem_bits = - utils::get_sizeof_etype(v->args_[1]->dtype_.type_code_) * 8 * lanes; - switch (elem_bits) { - case 8: { - current_val_ = builder_.CreateIntrinsic( - Intrinsic::x86_avx512_permvar_qi_512, {}, {inval2, inval1}); - } break; - case 16: { - current_val_ = builder_.CreateIntrinsic( - Intrinsic::x86_avx512_permvar_hi_512, {}, {inval2, inval1}); - } break; - case 64: { - // Because there are two forms of invocation of the - // vpermq instruction. One is such as - // _mm256_permutexvar_epi64 (__m256i idx, __m256i a) and - // _mm256_permute4x64_epi64 (__m256i a, const int imm8). The - // corresponding code construction shufflevector of these - // two forms in llvm is different, and we are consistent - // with the source code of llvm. - if (v->args_[0].isa()) { - unsigned imm = get_expr_as_int(v->args_[0]); - assert(v->args_[0].isa() || elem_bits > 0); - bool is_avx512 = utils::get_sizeof_type(v->args_[1]->dtype_) * 8 == 512; - auto target_type = is_avx512 ? get_type(sc_data_type_t::index(8)) - : get_type(sc_data_type_t::index(4)); - unsigned numelts = is_avx512 ? 8 : 4; - auto outtype = get_type(v->args_[1]->dtype_); - auto tmp1 = builder_.CreateBitCast(inval2, target_type); - // These intrinsics operate on 256-bit lanes of four - // 64-bit elements. - std::vector indices(numelts); - for (unsigned l = 0; l != numelts; l += 4) - for (unsigned i = 0; i != 4; ++i) - indices[l + i] = l + ((imm >> (2 * i)) & 0x3); - - auto ret = builder_.CreateShuffleVector(tmp1, indices, "perm"); - current_val_ = builder_.CreateBitCast(ret, outtype); - } else { - COMPILE_ASSERT(false, "Currently we do not need " - "_mm512_permutexvar_epi64 (__m512i idx, " - "__m512i a)"); - } - } break; - default: { - COMPILE_ASSERT(false, "Currently, we don't support " - "\"" << v->args_[1]->dtype_ - << "\""); - } break; - } - } break; - case intrin_type::unpack_high: - case intrin_type::unpack_low: { - COMPILE_ASSERT(v->args_.size() == 2, - "Expecting size of args = 2, but get " << v->args_.size()); - auto inval1 = generate_expr(v->args_[0]); - auto inval2 = generate_expr(v->args_[1]); - Value *tmp1; - Value *tmp2; - auto elem_bits = v->intrin_attrs_->get("elem_bits"); - std::vector hi_array, lo_array; - const int type_bits = utils::get_sizeof_type(v->dtype_) * 8; - // We should unpack according to the number of the avx data type - // bits. - switch (type_bits) { - case 128: { - switch (elem_bits) { - case 8: { - auto target_type = get_type(sc_data_type_t::u8(16)); - tmp1 = builder_.CreateBitCast(inval1, target_type); - tmp2 = builder_.CreateBitCast(inval2, target_type); - hi_array = std::vector{ - 8, 16 + 8, 9, 16 + 9, 10, 16 + 10, 11, 16 + 11, - 12, 16 + 12, 13, 16 + 13, 14, 16 + 14, 15, 16 + 15}; - lo_array.resize(16); - std::transform(hi_array.begin(), hi_array.end(), lo_array.begin(), - [](shuffle_idx_t x) { return x - 8; }); - } break; - case 16: { - auto target_type = get_type(sc_data_type_t::u16(8)); - tmp1 = builder_.CreateBitCast(inval1, target_type); - tmp2 = builder_.CreateBitCast(inval2, target_type); - hi_array = - std::vector{4, 8 + 4, 5, 8 + 5, 6, 8 + 6, 7, 8 + 7}; - lo_array.resize(8); - std::transform(hi_array.begin(), hi_array.end(), lo_array.begin(), - [](shuffle_idx_t x) { return x - 4; }); - } break; - case 32: { - auto target_type = get_type(sc_data_type_t::u32(4)); - tmp1 = builder_.CreateBitCast(inval1, target_type); - tmp2 = builder_.CreateBitCast(inval2, target_type); - hi_array = std::vector{2, 4 + 2, 3, 4 + 3}; - lo_array.resize(4); - std::transform(hi_array.begin(), hi_array.end(), lo_array.begin(), - [](shuffle_idx_t x) { return x - 2; }); - } break; - case 64: { - auto target_type = get_type(sc_data_type_t::index(2)); - tmp1 = builder_.CreateBitCast(inval1, target_type); - tmp2 = builder_.CreateBitCast(inval2, target_type); - hi_array = std::vector{1, 2 + 1}; - lo_array.resize(2); - std::transform(hi_array.begin(), hi_array.end(), lo_array.begin(), - [](shuffle_idx_t x) { return x - 1; }); - } break; - default: { - COMPILE_ASSERT(false, "Only support 8, 16, 32 and 64 bits"); - }; - }; - } break; - case 256: { - switch (elem_bits) { - case 8: { - auto target_type = get_type(sc_data_type_t::u8(32)); - tmp1 = builder_.CreateBitCast(inval1, target_type); - tmp2 = builder_.CreateBitCast(inval2, target_type); - hi_array = std::vector{ - 8, 32 + 8, 9, 32 + 9, 10, 32 + 10, 11, 32 + 11, - 12, 32 + 12, 13, 32 + 13, 14, 32 + 14, 15, 32 + 15, - 24, 32 + 24, 25, 32 + 25, 26, 32 + 26, 27, 32 + 27, - 28, 32 + 28, 29, 32 + 29, 30, 32 + 30, 31, 32 + 31}; - lo_array.resize(32); - std::transform(hi_array.begin(), hi_array.end(), lo_array.begin(), - [](shuffle_idx_t x) { return x - 8; }); - } break; - case 16: { - auto target_type = get_type(sc_data_type_t::u16(16)); - tmp1 = builder_.CreateBitCast(inval1, target_type); - tmp2 = builder_.CreateBitCast(inval2, target_type); - hi_array = std::vector{ - 4, 16 + 4, 5, 16 + 5, 6, 16 + 6, 7, 16 + 7, - 12, 16 + 12, 13, 16 + 13, 14, 16 + 14, 15, 16 + 15}; - lo_array.resize(16); - std::transform(hi_array.begin(), hi_array.end(), lo_array.begin(), - [](shuffle_idx_t x) { return x - 4; }); - } break; - case 32: { - auto target_type = get_type(sc_data_type_t::u32(8)); - tmp1 = builder_.CreateBitCast(inval1, target_type); - tmp2 = builder_.CreateBitCast(inval2, target_type); - hi_array = - std::vector{2, 8 + 2, 3, 8 + 3, 6, 8 + 6, 7, 8 + 7}; - lo_array.resize(8); - std::transform(hi_array.begin(), hi_array.end(), lo_array.begin(), - [](shuffle_idx_t x) { return x - 2; }); - } break; - case 64: { - auto target_type = get_type(sc_data_type_t::index(4)); - tmp1 = builder_.CreateBitCast(inval1, target_type); - tmp2 = builder_.CreateBitCast(inval2, target_type); - hi_array = std::vector{1, 4 + 1, 3, 4 + 3}; - lo_array.resize(4); - std::transform(hi_array.begin(), hi_array.end(), lo_array.begin(), - [](shuffle_idx_t x) { return x - 1; }); - } break; - default: { - COMPILE_ASSERT(false, "Only support 8, 16, 32 and 64 bits"); - } break; - } - } break; - case 512: { - switch (elem_bits) { - case 8: { - auto target_type = get_type(sc_data_type_t::u8(64)); - tmp1 = builder_.CreateBitCast(inval1, target_type); - tmp2 = builder_.CreateBitCast(inval2, target_type); - hi_array = std::vector{ - 8, 64 + 8, 9, 64 + 9, 10, 64 + 10, 11, 64 + 11, 12, 64 + 12, - 13, 64 + 13, 14, 64 + 14, 15, 64 + 15, 24, 64 + 24, 25, 64 + 25, - 26, 64 + 26, 27, 64 + 27, 28, 64 + 28, 29, 64 + 29, 30, 64 + 30, - 31, 64 + 31, 40, 64 + 40, 41, 64 + 41, 42, 64 + 42, 43, 64 + 43, - 44, 64 + 44, 45, 64 + 45, 46, 64 + 46, 47, 64 + 47, 56, 64 + 56, - 57, 64 + 57, 58, 64 + 58, 59, 64 + 59, 60, 64 + 60, 61, 64 + 61, - 62, 64 + 62, 63, 64 + 63}; - lo_array.resize(64); - std::transform(hi_array.begin(), hi_array.end(), lo_array.begin(), - [](shuffle_idx_t x) { return x - 8; }); - } break; - case 16: { - auto target_type = get_type(sc_data_type_t::u16(32)); - tmp1 = builder_.CreateBitCast(inval1, target_type); - tmp2 = builder_.CreateBitCast(inval2, target_type); - hi_array = std::vector{ - 4, 32 + 4, 5, 32 + 5, 6, 32 + 6, 7, 32 + 7, - 12, 32 + 12, 13, 32 + 13, 14, 32 + 14, 15, 32 + 15, - 20, 32 + 20, 21, 32 + 21, 22, 32 + 22, 23, 32 + 23, - 28, 32 + 28, 29, 32 + 29, 30, 32 + 30, 31, 32 + 31}; - lo_array.resize(32); - std::transform(hi_array.begin(), hi_array.end(), lo_array.begin(), - [](shuffle_idx_t x) { return x - 4; }); - } break; - case 32: { - auto target_type = get_type(sc_data_type_t::u32(16)); - tmp1 = builder_.CreateBitCast(inval1, target_type); - tmp2 = builder_.CreateBitCast(inval2, target_type); - hi_array = std::vector{ - 2, 18, 3, 19, 2 + 4, 18 + 4, 3 + 4, 19 + 4, - 2 + 8, 18 + 8, 3 + 8, 19 + 8, 2 + 12, 18 + 12, 3 + 12, 19 + 12}; - lo_array.resize(16); - std::transform(hi_array.begin(), hi_array.end(), lo_array.begin(), - [](shuffle_idx_t x) { return x - 2; }); - } break; - case 64: { - auto target_type = get_type(sc_data_type_t::index(8)); - tmp1 = builder_.CreateBitCast(inval1, target_type); - tmp2 = builder_.CreateBitCast(inval2, target_type); - hi_array = std::vector{1, 9, 1 + 2, 9 + 2, - 1 + 4, 9 + 4, 1 + 6, 9 + 6}; - lo_array.resize(8); - std::transform(hi_array.begin(), hi_array.end(), lo_array.begin(), - [](shuffle_idx_t x) { return x - 1; }); - } break; - default: { - COMPILE_ASSERT(false, "Only support 8, 16, 32 and 64 bits"); - } break; - } - } break; - default: { - COMPILE_ASSERT(false, "Invalid simd bits: " << type_bits); - } break; - } - ArrayRef arr = - v->type_ == intrin_type::unpack_high ? hi_array : lo_array; - auto res = builder_.CreateShuffleVector(tmp1, tmp2, arr); - current_val_ = builder_.CreateBitCast(res, get_type(v->dtype_)); - } break; - case intrin_type::shuffle: { - COMPILE_ASSERT(v->args_.size() == 2, - "args size must be 2, but get " << v->args_.size()); - auto inval1 = generate_expr(v->args_[0]); - auto inval2 = generate_expr(v->args_[1]); - auto imm8 = v->intrin_attrs_->get("shuffle_imm"); - auto type_bits = v->intrin_attrs_->get("type_bits"); - switch (type_bits) { - case 32: { - unsigned num_elts = v->args_[0]->dtype_.lanes_; - unsigned num_lanes = - utils::get_sizeof_type(v->args_[0]->dtype_) * 8 / 128; - unsigned num_lanes_elts = num_elts / num_lanes; - - // Splat the 8-bits of immediate 4 times to help the - // loop wrap around. - imm8 = (imm8 & 0xff) * 0x01010101; - - std::vector indices(num_elts); - for (unsigned l = 0; l != num_elts; l += num_lanes_elts) { - for (unsigned i = 0; i != num_lanes_elts; ++i) { - unsigned index = imm8 % num_lanes_elts; - imm8 /= num_lanes_elts; - if (i >= (num_lanes_elts / 2)) - index += num_elts; - indices[l + i] = l + index; - } - } - - current_val_ = - builder_.CreateShuffleVector(inval1, inval2, indices, "shuf32"); - } break; - case 128: { - unsigned num_elts = v->args_[0]->dtype_.lanes_; - unsigned num_lanes = - utils::get_sizeof_type(v->args_[0]->dtype_) * 8 == 512 ? 4 : 2; - unsigned num_lanes_elts = num_elts / num_lanes; - - std::vector indices(num_elts); - for (unsigned l = 0; l != num_elts; l += num_lanes_elts) { - unsigned index = (imm8 % num_lanes) * num_lanes_elts; - imm8 /= num_lanes; // Discard the bits we just used. - if (l >= (num_elts / 2)) - index += num_elts; // Switch to other source. - for (unsigned i = 0; i != num_lanes_elts; ++i) { - indices[l + i] = index + i; - } - } - - current_val_ = - builder_.CreateShuffleVector(inval1, inval2, indices, "shuf128"); - } break; - default: { - COMPILE_ASSERT(false, "Curerntly only support type_bits 32 and 128, " - "but get " - << type_bits); - } break; - }; - } break; - case intrin_type::permute: { - assert(v->args_.size() == 2); - - auto imm8 = v->intrin_attrs_->get("permute_imm"); - auto val0 = generate_expr(v->args_[0]); - auto val1 = generate_expr(v->args_[1]); - - unsigned numelts = v->args_[0]->dtype_.lanes_; - - // This takes a very simple approach since there are two - // lanes and a shuffle can have 2 inputs. So we reserve the - // first input for the first lane and the second input for - // the second lane. This may result in duplicate sources, - // but this can be dealt with in the backend. - - Value *outops[2]; - int indices[16]; - for (unsigned l = 0; l != 2; ++l) { - // Determine the source for this lane. - if (imm8 & (1 << ((l * 4) + 3))) - outops[l] = llvm::ConstantAggregateZero::get(val0->getType()); - else if (imm8 & (1 << ((l * 4) + 1))) - outops[l] = val1; - else - outops[l] = val0; - - for (unsigned i = 0; i != numelts / 2; ++i) { - // Start with ith element of the source for this - // lane. - unsigned idx = (l * numelts) + i; - // If bit 0 of the immediate half is set, switch to - // the high half of the source. - if (imm8 & (1 << (l * 4))) - idx += numelts / 2; - indices[(l * (numelts / 2)) + i] = idx; - } - } - - current_val_ = builder_.CreateShuffleVector( - outops[0], outops[1], ArrayRef(indices, numelts), - "vperm"); - } break; - case intrin_type::prefetch: { - assert(v->args_.size() == 1); - auto locality = v->intrin_attrs_->get("locality"); - assert(locality <= 3 && locality >= 0 && "bad locality for prefetch"); - auto inval1 = generate_expr(v->args_[0]); - current_val_ = builder_.CreateIntrinsic( - Intrinsic::prefetch, -#if SC_LLVM_BACKEND > 8 - {builder_.getInt8PtrTy()}, -#else - {}, -#endif - {builder_.CreatePointerCast(inval1, builder_.getInt8PtrTy()), - /*rw*/ builder_.getInt32(0), - /*locality*/ - builder_.getInt32(3 - locality), - /*type:i/d*/ builder_.getInt32(1)}); - } break; - case intrin_type::gather: { - assert(v->args_.size() == 2); - COMPILE_ASSERT(v->dtype_.is_etype(sc_data_etype::F32), - "Expecting f32 for gather: " << v); - auto inval1 = builder_.CreatePointerCast(generate_expr(v->args_[0]), - builder_.getInt8PtrTy()); - auto inval2 = generate_expr(v->args_[1]); - auto full_mask = generate_expr(make_expr( - std::vector(v->dtype_.lanes_, UINT64_C(0xffffffff)), - v->dtype_)); - auto dummy_vec = generate_expr(make_expr( - std::vector(v->dtype_.lanes_, UINT64_C(0)), v->dtype_)); - switch (v->dtype_.lanes_) { - case 1: { - auto indexing = make_expr( - v->args_[0], std::vector{v->args_[1]}, expr()); - current_val_ = generate_expr(indexing); - break; - } - case 4: - current_val_ = builder_.CreateIntrinsic( - Intrinsic::x86_avx2_gather_d_ps, {}, - {dummy_vec, inval1, inval2, full_mask, builder_.getInt8(4)}); - break; - case 8: - current_val_ = builder_.CreateIntrinsic( - Intrinsic::x86_avx2_gather_d_ps_256, {}, - {dummy_vec, inval1, inval2, full_mask, builder_.getInt8(4)}); - break; - case 16: { - COMPILE_ASSERT(ctx_->machine_.cpu_flags_.fAVX512F, - "gather of 16 floats needs AVX512"); - current_val_ = builder_.CreateIntrinsic( - Intrinsic::x86_avx512_gather_dps_512, {}, - {dummy_vec, inval1, inval2, builder_.getInt16(0xffff), - builder_.getInt32(4)}); - break; - } - default: - COMPILE_ASSERT(false, "LLVM backend has not yet support " - "gather with lanes = " - << v->dtype_.lanes_); - break; - } - } break; - case intrin_type::insert: { - auto val0 = generate_expr(v->args_[0]); - auto val1 = generate_expr(v->args_[1]); - unsigned index = v->intrin_attrs_->get("insert_imm"); - const unsigned elem_bits = utils::get_sizeof_type(v->args_[1]->dtype_) * 8; - auto lanes = v->args_[0]->dtype_.lanes_; - auto type_code = v->args_[0]->dtype_.type_code_; - if (ctx_->machine_.cpu_flags_.fAVX512F && elem_bits >= 128) { - // avx512 128bit insert can use this part. - unsigned dst_num_elts = lanes, src_num_elts = lanes / 2; - unsigned subvectors = dst_num_elts / src_num_elts; - index &= subvectors - 1; - index *= src_num_elts; - std::vector indices(dst_num_elts); - for (unsigned i = 0; i != dst_num_elts; ++i) { - indices[i] = - (i >= src_num_elts) ? src_num_elts + (i % src_num_elts) : i; - } - Value *op1 = builder_.CreateShuffleVector(val1, indices, "widen"); - for (unsigned i = 0; i != dst_num_elts; ++i) { - if (i >= index && i < (index + src_num_elts)) { - indices[i] = (i - index) + dst_num_elts; - } else { - indices[i] = i; - } - } - current_val_ = builder_.CreateShuffleVector(val0, op1, indices, "insert"); - } else { - assert(elem_bits <= 128); - uint64_t idx = index; - idx &= lanes - 1; - current_val_ = builder_.CreateInsertElement(val0, val1, idx); - } - } break; - case intrin_type::extract: { - auto val0 = generate_expr(v->args_[0]); - unsigned index = v->intrin_attrs_->get("extract_imm"); - const unsigned elem_bits = utils::get_sizeof_type(v->dtype_) * 8; - auto lanes = v->args_[0]->dtype_.lanes_; - auto type_code = v->args_[0]->dtype_.type_code_; - if (ctx_->machine_.cpu_flags_.fAVX512F && elem_bits >= 128) { - // avx512 128bit extract can use this part. - unsigned dst_num_elts = - index / utils::get_sizeof_etype(v->dtype_.as_etype()), - src_num_elts = lanes; - unsigned subvectors = src_num_elts / dst_num_elts; - index &= subvectors - 1; - index *= src_num_elts; - - std::vector indices(dst_num_elts); - for (unsigned i = 0; i != dst_num_elts; ++i) - indices[i] = i + index; - current_val_ = builder_.CreateShuffleVector(val0, indices, "extract"); - } else { - assert(elem_bits <= 128); - uint64_t idx = (uint64_t)index; - idx &= lanes - 1; - current_val_ = builder_.CreateExtractElement(val0, idx); - } - } break; - default: { - std::stringstream ss; - ss << "Intrinsics not implemented "; - v->to_string(ss); - throw std::runtime_error(ss.str()); - } - } -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/codegen/llvm/intrinsic_impl.cpp b/legacy/core/src/compiler/codegen/llvm/intrinsic_impl.cpp deleted file mode 100644 index 5e4d0e3b9..000000000 --- a/legacy/core/src/compiler/codegen/llvm/intrinsic_impl.cpp +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright (C) 2025 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions - * and limitations under the License. - * SPDX-License-Identifier: Apache-2.0 - */ -#include -#include -#include -#include -#include -#include -#include -#include - -#include "shared_include.hpp" - -using namespace llvm; -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -Value *codegen_llvm_vis_t::call_unary_llvm_intrin(const intrin_call_c &v, - type_category cate, - Intrinsic::ID id, - bool must_fp) { - assert(v->args_.size() == 1); - auto inval = generate_expr(v->args_[0]); - if (must_fp) { - COMPILE_ASSERT(cate == CATE_FLOAT, "Bad type. Expecting float: " << v); - } - return builder_.CreateUnaryIntrinsic(id, inval); -} - -Value *codegen_llvm_vis_t::call_binary_llvm_intrin(const intrin_call_c &v, - type_category cate, - Intrinsic::ID id, - bool must_fp) { - assert(v->args_.size() == 2); - auto inval1 = generate_expr(v->args_[0]); - auto inval2 = generate_expr(v->args_[1]); - if (must_fp) { - COMPILE_ASSERT(cate == CATE_FLOAT, "Bad type. Expecting float: " << v); - } - return builder_.CreateBinaryIntrinsic(id, inval1, inval2); -} - -Value *codegen_llvm_vis_t::call_binary_llvm_normal(const intrin_call_c &v, - llvm_binary_func op) { - assert(v->args_.size() == 2); - auto inval1 = generate_expr(v->args_[0]); - auto inval2 = generate_expr(v->args_[1]); - return (builder_.*op)(inval1, inval2, ""); -} - -Value *codegen_llvm_vis_t::make_int_min_max(const intrin_call_c &v, bool ismin, - type_category cate) { - assert(v->args_.size() == 2); - auto v1 = generate_expr(v->args_[0]); - auto v2 = generate_expr(v->args_[1]); - return make_int_min_max(v1, v2, ismin, cate); -} - -Value *codegen_llvm_vis_t::make_int_min_max(Value *v1, Value *v2, bool ismin, - type_category cate) { - // fix-me: use smax/smin for newer LLVM - llvm::Value *(llvm::IRBuilder<>::*ptr)(llvm::Value *LHS, llvm::Value *RHS, - const llvm::Twine &Name); - if (ismin) { - if (cate == CATE_INT) { - ptr = &IRBuilder<>::CreateICmpSLE; - } else { - ptr = &IRBuilder<>::CreateICmpULE; - } - } else { - if (cate == CATE_INT) { - ptr = &IRBuilder<>::CreateICmpSGE; - } else { - ptr = &IRBuilder<>::CreateICmpUGE; - } - } - return builder_.CreateSelect((builder_.*ptr)(v1, v2, ""), v1, v2); -} - -Value *codegen_llvm_vis_t::do_lower_saturated_cast(const intrin_call_c &v) { - COMPILE_ASSERT(ctx_->machine_.cpu_flags_.fAVX512F, - "lowered saturated_cast needs AVX512F"); - assert(v->args_.size() == 1); - auto inval1 = generate_expr(v->args_[0]); - auto intype = v->args_[0]->dtype_; - auto out_llvm_ty = get_type(v->dtype_); - auto ths = this; - // the fast path for AVX512 - auto pmovus_db_512 = [ths, out_llvm_ty](Value *v, bool issigned) { - Intrinsic::ID id = issigned ? Intrinsic::x86_avx512_mask_pmovs_db_512 - : Intrinsic::x86_avx512_mask_pmovus_db_512; - return ths->builder_.CreateIntrinsic( - id, {}, - {v, UndefValue::get(out_llvm_ty), ths->builder_.getInt16(0xffff)}); - }; - if (v->dtype_ == sc_data_type_t::s8(16)) { - if (intype == sc_data_type_t::s32(16)) { - return pmovus_db_512(inval1, true); - } - } else if (v->dtype_ == sc_data_type_t::u8(16)) { - if (intype == sc_data_type_t::s32(16) || - intype == sc_data_type_t::u32(16)) { - return pmovus_db_512(inval1, false); - } - } - COMPILE_ASSERT(false, "lowered saturated_cast cannot handle: " - << v << '(' << intype << "->" << v->dtype_ << ')'); -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/codegen/llvm/llvm_visitor.hpp b/legacy/core/src/compiler/codegen/llvm/llvm_visitor.hpp deleted file mode 100644 index f9344de92..000000000 --- a/legacy/core/src/compiler/codegen/llvm/llvm_visitor.hpp +++ /dev/null @@ -1,200 +0,0 @@ -/******************************************************************************* - * Copyright 2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_CODEGEN_LLVM_LLVM_VISITOR_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_CODEGEN_LLVM_LLVM_VISITOR_HPP - -/* -Developer notes: -We break the implementation of codegen_llvm_vis_t into multiple files, because -we need to workaround a hidden g++-7 bug linking with LLVM16. We found that when -the size of a single cpp source/binary for LLVM-IR generation increases to a -certain size, the generated in-memory LLVM-IR will be broken. It results in -segfault in LLVM's internal passes. g++-9 does not break LLVM16. -*/ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -class codegen_llvm_vis_t : public ir_viewer_t { -public: - context_ptr ctx_; - llvm::LLVMContext &context_; - llvm::IRBuilder<> builder_; - std::unique_ptr module_; - std::unique_ptr dbuilder_; - llvm::DICompileUnit *dbg_cu_ = nullptr; - std::vector dbg_scopes_; - llvm::Function *current_func_ = nullptr; - llvm::Value *current_val_ = nullptr; - // the **pointer** of local var in a function - std::unordered_map var_ptr_in_func_; - using vec_metadata = llvm::SmallVector; - // tensor to - std::unordered_map *> - tsr_to_alias_scope_; - std::unordered_map> - alias_set_to_alias_scope_; - std::unordered_map name_to_func_; - bool is_lvalue_mode_ = false; - - codegen_llvm_vis_t(const context_ptr &ctx, llvm::LLVMContext &context, - const std::string &source_dir, - const std::string &source_file_name); - - using ir_viewer_t::dispatch; - using ir_viewer_t::view; - - expr_c dispatch(expr_c v) override; - - stmt_c dispatch(stmt_c v) override; - - const std::string &get_node_name(const expr_c &c); - - llvm::FunctionType *create_func_type(const func_c &v); - - llvm::DISubroutineType *create_func_dtype(const func_c &v); - - llvm::Function *get_or_create_func(const func_c &v); - - llvm::Value *generate_expr(const expr_c &e); - - std::unordered_map> - type_cache_; - std::pair do_get_type(sc_data_type_t dtype); - - std::pair get_type_both(sc_data_type_t dtype); - - llvm::Type *get_type(sc_data_type_t dtype); - - llvm::Value *get_defined_var_ptr(const expr_c &e); - - llvm::Value *define_var(const expr_c &e, llvm::Value *initv); - - void set_dbg_info_for_func_arg(llvm::Value *v, llvm::DISubprogram *SP, - llvm::DIFile *dunit, sc_data_type_t type, - const std::string &name, int argidx, - int lineno, bool need_ref); - void emit_location(const node_base *p); - void prepare_alias_metadata( - const std::vector &tensors, - llvm::MDNode *alias_domain, llvm::MDBuilder &MDB); - - func_c dispatch(func_c v) override; - void view(constant_c v) override; - void view(var_c v) override; - - llvm::Type *get_llvm_bf16_native_type(unsigned lanes); - - void view(cast_c v) override; - void generate_bin_op(const expr_c &v, const expr_c &l, const expr_c &r); - void view(binary_c v) override; - void view(cmp_c v) override; - - void view(logic_c v) override; - void view(logic_not_c v) override; - void view(select_c v) override; - - llvm::Instruction *set_alias(llvm::Instruction *inst, const expr_c &tsr); - - llvm::Value *set_alias(llvm::Value *inst, const expr_c &tsr); - - llvm::Value *convert_mask(const expr &in, const bool is_int4 = false); - - void view(indexing_c v) override; - void view(tensorptr_c v) override; - - llvm::Value *gen_vec_const(uint64_t elements, float f); - - llvm::Value *call_unary_llvm_intrin(const intrin_call_c &v, - type_category cate, - llvm::Intrinsic::ID id, bool must_fp); - - llvm::Value *call_binary_llvm_intrin(const intrin_call_c &v, - type_category cate, - llvm::Intrinsic::ID id, bool must_fp); - - typedef llvm::Value *(llvm::IRBuilder<>::*llvm_binary_func)( - llvm::Value *LHS, llvm::Value *RHS, const llvm::Twine &Name); - llvm::Value *call_binary_llvm_normal(const intrin_call_c &v, - llvm_binary_func op); - - llvm::Value *make_int_min_max(const intrin_call_c &v, bool ismin, - type_category cate); - - llvm::Value *make_int_min_max(llvm::Value *v1, llvm::Value *v2, bool ismin, - type_category cate); - - llvm::Value *do_lower_saturated_cast(const intrin_call_c &v); - - void view(intrin_call_c v) override; - void view(func_addr_c v) override; - void view(call_c v) override; - void view(tensor_c v) override; - // void view(stmts_c v) override; - // void view(evaluate_c v) override; - - void generate_codeblock(const stmt_c &v, llvm::BasicBlock *current, - llvm::BasicBlock *cont); - - void view(assign_c v) override; - - void view(if_else_c v) override; - - void view(returns_c v) override; - - void set_dbg_info_for_local_var(const source_pos *pos, sc_data_type_t type, - const std::string &name, - llvm::Value *llvm_value, bool need_ref); - - void set_dbg_info_for_local_var(const define_node_t *v, - const std::string &name, - llvm::Value *llvm_value, bool need_ref); - - void view(define_c v) override; - - void view(for_loop_c v) override; - ~codegen_llvm_vis_t(); -}; -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/codegen/llvm/shared_include.hpp b/legacy/core/src/compiler/codegen/llvm/shared_include.hpp deleted file mode 100644 index 7fa8e53ed..000000000 --- a/legacy/core/src/compiler/codegen/llvm/shared_include.hpp +++ /dev/null @@ -1,115 +0,0 @@ -/******************************************************************************* - * Copyright 2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_CODEGEN_LLVM_SHARED_INCLUDE_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_CODEGEN_LLVM_SHARED_INCLUDE_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#if SC_LLVM_BACKEND > 16 -#include -#else -#include -#endif -#if SC_LLVM_BACKEND > 13 -#include -#else -#include -#endif -#if SC_LLVM_BACKEND >= 17 -#include -#else -#include -#endif -#include -#include -#include -#include -#include -#include -#include -#include - -#if SC_LLVM_BACKEND > 15 -#if SC_LLVM_BACKEND < 17 -#include -#include -#endif -#include -#endif - -#if SC_LLVM_BACKEND > 8 -#include -#endif - -#include "llvm_visitor.hpp" - -SC_MODULE(codegen.llvm); - -#if SC_LLVM_BACKEND > 8 -#define SC_LLVM_ALIGN(a) Align(a) -#else -#define SC_LLVM_ALIGN(a) (a) -#endif - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -#if SC_LLVM_BACKEND > 16 -// starting from LLVM17, they use STL's optional container -template using Optional = std::optional; -#endif - -#if SC_LLVM_BACKEND > 10 -using shuffle_idx_t = int; -#else -using shuffle_idx_t = uint32_t; -#endif - -using namespace llvm; -#if SC_LLVM_BACKEND >= 18 -using LLVM_CodeGenOptLevel = llvm::CodeGenOptLevel; -#else -using LLVM_CodeGenOptLevel = llvm::CodeGenOpt::Level; -#endif - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/codegen/llvm/stmt.cpp b/legacy/core/src/compiler/codegen/llvm/stmt.cpp deleted file mode 100644 index 11613a990..000000000 --- a/legacy/core/src/compiler/codegen/llvm/stmt.cpp +++ /dev/null @@ -1,242 +0,0 @@ -/******************************************************************************* - * Copyright 2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#include -#include -#include -#include -#include -#include -#include -#include - -#include "shared_include.hpp" - -using namespace llvm; -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -void codegen_llvm_vis_t::generate_codeblock(const stmt_c &v, - BasicBlock *current, - BasicBlock *cont) { - builder_.SetInsertPoint(current); - dispatch(v); - current = builder_.GetInsertBlock(); - if (current->empty() || !llvm::isa(current->back())) { - builder_.CreateBr(cont); - } -} - -void codegen_llvm_vis_t::view(assign_c v) { - auto val = generate_expr(v->value_); - is_lvalue_mode_ = true; - auto ptr = generate_expr(v->var_); - if (v->var_->dtype_.lanes_ > 1) { - ptr = builder_.CreatePointerCast(ptr, - get_type(v->var_->dtype_)->getPointerTo()); - } - if (v->var_->dtype_.is_pointer() && - v->value_->dtype_.type_code_ != v->var_->dtype_.type_code_) { - val = builder_.CreatePointerCast(val, get_type(v->var_->dtype_)); - } - if (v->value_->dtype_.lanes_ > 1 && v->var_.isa()) { - // assigning to tensor - if (v->var_.static_as()->mask_.defined()) { - Value *mask; - auto v_index = v->var_.static_as(); - auto bit_len = utils::get_sizeof_type(v_index->mask_->dtype_) * 8; - if (v_index->dtype_.lanes_ != bit_len) { - COMPILE_ASSERT(v_index->dtype_.lanes_ == 4, - "Currently only 8bit -> 4bit is supported."); - mask = convert_mask(v_index->mask_, true); - } else { - mask = convert_mask(v_index->mask_); - } -#if SC_LLVM_BACKEND > 10 - set_alias(builder_.CreateMaskedStore(val, ptr, SC_LLVM_ALIGN(1), mask), - v->var_); -#else - set_alias(builder_.CreateMaskedStore(val, ptr, 1, mask), v->var_); -#endif - } else { - set_alias(builder_.CreateAlignedStore(val, ptr, SC_LLVM_ALIGN(1)), - v->var_); - } - } else { - set_alias(builder_.CreateStore(val, ptr), v->var_); - } -} - -void codegen_llvm_vis_t::view(if_else_c v) { - auto cond = generate_expr(v->condition_); - BasicBlock *tb = BasicBlock::Create(context_, "if_t", current_func_); - BasicBlock *cb = BasicBlock::Create(context_, "if_cont", current_func_); - BasicBlock *fb = v->else_case_.defined() - ? BasicBlock::Create(context_, "if_f", current_func_) - : cb; - fb->moveBefore(cb); - builder_.CreateCondBr(cond, tb, fb); - generate_codeblock(v->then_case_, tb, cb); - if (fb != cb) { - generate_codeblock(v->else_case_, fb, cb); - } - builder_.SetInsertPoint(cb); -} - -void codegen_llvm_vis_t::view(returns_c v) { - if (v->value_.defined()) { - builder_.CreateRet(generate_expr(v->value_)); - } else { - builder_.CreateRetVoid(); - } -} - -void codegen_llvm_vis_t::view(define_c v) { - COMPILE_ASSERT(v->linkage_ != linkage::static_local && - v->linkage_ != linkage::private_global, - "LLVM backend cannot handle non-local variable " - "definitions"); - if (v->var_.isa()) { - auto thevar = v->var_.static_as(); - if (thevar->attr_ && - thevar->attr_->has_key(attr_keys::module_global_offset)) { - // if it is a global variable that is lowered to local - auto &offset = thevar->attr_->get_any(attr_keys::module_global_offset); - Value *ptr; - if (auto absptr = offset.get_or_null()) { - ptr = builder_.CreateIntToPtr( - builder_.getInt64(reinterpret_cast(*absptr)), - get_type(thevar->dtype_)->getPointerTo(), thevar->name_); - } else { - auto module_ptr = current_func_->arg_begin() + 1; - assert(module_ptr->getName() == "__module_data_arg"); - - ptr = builder_.CreateGEP(builder_.getInt8Ty(), module_ptr, - builder_.getInt64(offset.get())); - ptr = builder_.CreatePointerCast( - ptr, get_type(thevar->dtype_)->getPointerTo(), thevar->name_); - } - var_ptr_in_func_.insert(std::make_pair(thevar, ptr)); - set_dbg_info_for_local_var(v.get(), thevar->name_, ptr, false); - } else { - Value *init_v = nullptr; - if (v->init_.defined()) { - init_v = generate_expr(v->init_); - } - auto retv = define_var(thevar, init_v); - set_dbg_info_for_local_var(v.get(), thevar->name_, retv, false); - } - } else if (v->var_.isa()) { - tensor t = v->var_.static_as(); - if (auto alias_info = alias_info::get_alias_info(*t)) { - auto alias_itr = alias_set_to_alias_scope_.find(alias_info); - if (alias_itr != alias_set_to_alias_scope_.end()) { - tsr_to_alias_scope_[t] = &alias_itr->second; - } - } - // if it is a view of the rescheduled buffer/ local tensor on - // heap - if (v->init_.defined()) { - Value *ptr = generate_expr(v->init_); - ptr = builder_.CreatePointerCast( - ptr, get_type(t->elem_dtype_)->getPointerTo(), t->name_); - var_ptr_in_func_.insert(std::make_pair(t, ptr)); - set_dbg_info_for_local_var(v.get(), t->name_, ptr, true); - return; - } - - // explicitly align tensor with cache line size, except that - // tensor is a scalar or bytes size < 64. - bool need_align = false; - // check condition. - if (t->dims_.size() == 1 && - get_const_as_int(t->dims_[0].checked_as()) == 1) { - // it is a scalar - } else { - size_t shape = 1; - for (auto &d : t->dims_) { - shape *= get_const_as_int(d.checked_as()); - } - size_t dtsize = utils::get_sizeof_etype(t->elem_dtype_.type_code_); - // check bytes size - if (shape * dtsize > 64) - need_align = true; - } - auto ptr = builder_.CreateAlloca(get_type(t->elem_dtype_), - generate_expr(t->dims_.front()), t->name_); - // cache line alignment - - if (need_align) { - ptr->setAlignment(SC_LLVM_ALIGN(64)); - } - - var_ptr_in_func_.insert(std::make_pair(t, ptr)); - set_dbg_info_for_local_var(v.get(), t->name_, ptr, true); - } else { - assert(0 && "Bad var type"); - } -} - -void codegen_llvm_vis_t::view(for_loop_c v) { - COMPILE_ASSERT(v->kind_ == for_type::NORMAL, - "LLVM backend can only handle normal for-loops"); - auto itr_v = define_var(v->var_, generate_expr(v->iter_begin_)); - - if (ctx_->flags_.debug_info_) { - auto pos = v->attr_->get_or_null("source_pos"); - if (pos) { - set_dbg_info_for_local_var( - pos, v->var_->dtype_, v->var_.checked_as()->name_, itr_v, false); - } - } - - BasicBlock *chk = BasicBlock::Create(context_, "for_check", current_func_); - BasicBlock *body = BasicBlock::Create(context_, "for_body", current_func_); - BasicBlock *cont = BasicBlock::Create(context_, "for_cont", current_func_); - builder_.CreateBr(chk); - { - builder_.SetInsertPoint(chk); - auto cate = get_type_category(v->var_->dtype_); - auto end_v = generate_expr(v->iter_end_); - auto itr_value = builder_.CreateLoad(get_type(v->var_->dtype_), itr_v); - Value *cond; - if (cate == CATE_INT) { - cond = builder_.CreateICmpSLT(itr_value, end_v); - } else { - assert(cate == CATE_UINT); - cond = builder_.CreateICmpULT(itr_value, end_v); - } - builder_.CreateCondBr(cond, body, cont); - } - { - builder_.SetInsertPoint(body); - dispatch(v->body_); - if (body->empty() || !llvm::isa(body->back())) { - auto step_v = generate_expr(v->step_); - Value *itr_value = builder_.CreateLoad(get_type(v->var_->dtype_), itr_v); - itr_value = builder_.CreateAdd(itr_value, step_v); - builder_.CreateStore(itr_value, itr_v); - builder_.CreateBr(chk); - } - } - cont->moveAfter(builder_.GetInsertBlock()); - builder_.SetInsertPoint(cont); -} -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/codegen/llvm/support.cpp b/legacy/core/src/compiler/codegen/llvm/support.cpp deleted file mode 100644 index 1363d6f6a..000000000 --- a/legacy/core/src/compiler/codegen/llvm/support.cpp +++ /dev/null @@ -1,698 +0,0 @@ -/******************************************************************************* - * Copyright 2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#include -#include -#include -#include -#include -#include -#include -#include - -#include "shared_include.hpp" -#include - -// g++-7 + LLVM16 has an unknown bug, which will generate broken TargetMachine -// we use C-API instead -#if SC_GNUC_VERSION_LT(8) && SC_LLVM_BACKEND >= 16 -#include -#define WORKAROUND_LLVM_TM -#endif - -using namespace llvm; -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -static void print_helper(Value *v) { v->print(llvm::errs()); } - -#define WHEN_ARCH_NAME(...) __VA_ARGS__ -#define ARCH_GraniteRapids \ - case 0xae: \ - case 0xad: -#define ARCH_EmeraldRapids case 0xcf: -#define ARCH_SapphireRapids case 0x8f: -#define END_ARCH() break; - -#define WHEN_LLVM_VER_GE(V, VALUE) \ - if (SC_LLVM_BACKEND >= (V)) { \ - return VALUE; \ - } -#define WHEN_LLVM_VER_RANGE(V1, V2, VALUE) \ - if (SC_LLVM_BACKEND >= (V1) && SC_LLVM_BACKEND <= (V2)) { \ - return VALUE; \ - } - -// Handle the case that the CPU is known to us, but not to current version of -// LLVM. We help LLVM to better fallback to a closer generation of CPU known to -// it. -static const char *handle_cpu_name(runtime::cpu_flags_t &flags) { - if (flags.family != 6) { - return nullptr; - } - constexpr const char *sapphirerapids = "sapphirerapids"; - constexpr const char *icelake_server = "icelake-server"; - constexpr const char *let_llvm_handle = nullptr; - switch (flags.model) { - // clang-format off - WHEN_ARCH_NAME(ARCH_GraniteRapids ARCH_EmeraldRapids) - WHEN_LLVM_VER_GE(16, let_llvm_handle) - WHEN_LLVM_VER_RANGE(12, 15, sapphirerapids) - WHEN_LLVM_VER_RANGE(9, 11, icelake_server) - END_ARCH() - WHEN_ARCH_NAME(ARCH_SapphireRapids) - WHEN_LLVM_VER_GE(12, let_llvm_handle) - WHEN_LLVM_VER_RANGE(9, 11, icelake_server) - END_ARCH() - // clang-format on - } - // default, let llvm decide - return let_llvm_handle; -} - -std::unique_ptr -get_llvm_target_machine(LLVM_CodeGenOptLevel optlevel) { - auto target_triple = sys::getProcessTriple(); - - std::string err; - auto target = TargetRegistry::lookupTarget(target_triple, err); - if (!target) { - throw std::runtime_error(err); - } - TargetOptions opt; -#if SC_LLVM_BACKEND > 15 - auto the_none = std::nullopt; -#else - auto the_none = llvm::None; -#endif - - auto reloc_model = Optional(Reloc::Static); - auto inferred_cpu_name = - handle_cpu_name(runtime::get_runtime_target_machine().cpu_flags_); - auto host_cpu = inferred_cpu_name ? inferred_cpu_name : sys::getHostCPUName(); - if (inferred_cpu_name) { - SC_MODULE_WARN << "Your CPU is not recognized by LLVM. Falling back to " - << inferred_cpu_name; - } - llvm::StringMap feature_map; - sys::getHostCPUFeatures(feature_map); - llvm::SubtargetFeatures f; - for (auto &feature : feature_map) { - f.AddFeature(feature.first(), feature.second); - } - std::string features = f.getString(); -#ifdef WORKAROUND_LLVM_TM - SC_UNUSED(the_none); - SC_UNUSED(reloc_model); - auto tm = (TargetMachine *)LLVMCreateTargetMachine( - (LLVMTargetRef)target, target_triple.c_str(), host_cpu.data(), - features.c_str(), LLVMCodeGenLevelDefault, LLVMRelocStatic, - LLVMCodeModelJITDefault); -#else - auto tm = target->createTargetMachine(target_triple, host_cpu, features, opt, - reloc_model, the_none, optlevel, true); -#endif - return std::unique_ptr(tm); -} - -std::unique_ptr get_llvm_target_machine( - LLVM_CodeGenOptLevel optlevel = LLVM_CodeGenOptLevel::Default); - -codegen_llvm_vis_t::codegen_llvm_vis_t(const context_ptr &ctx, - LLVMContext &context, - const std::string &source_dir, - const std::string &source_file_name) - : ctx_(ctx), context_(context), builder_(context_), - module_(utils::make_unique("name", context_)), - dbuilder_(utils::make_unique(*module_)) { - static bool initialized = []() { - // make sure LLVM native targets are initialized once and avoid - // race condition - InitializeNativeTarget(); - InitializeNativeTargetAsmParser(); - InitializeNativeTargetAsmPrinter(); - return true; - }(); - SC_UNUSED(initialized); - auto tm = get_llvm_target_machine(); - module_->setTargetTriple(tm->getTargetTriple().str()); - module_->setDataLayout(tm->createDataLayout()); - FastMathFlags fmflag; - // some optimization in FastMath may cause accuracy loss, which needs - // further investigation in the future - fmflag.setFast(false); - // keep FMA optimization on - fmflag.setAllowContract(true); - // turn on following options for performance - fmflag.setAllowReassoc(true); - fmflag.setNoNaNs(true); - builder_.setFastMathFlags(fmflag); - if (ctx->flags_.debug_info_) { - dbg_cu_ = dbuilder_->createCompileUnit( - dwarf::DW_LANG_C, dbuilder_->createFile(source_file_name, source_dir), - "oneDNN Graph Compiler", false, "", 0); - - if (!tm->getTargetTriple().isOSWindows()) { - // Add the current debug info version into the module. - module_->addModuleFlag(Module::Warning, "Debug Info Version", - DEBUG_METADATA_VERSION); - - // Darwin only supports dwarf2. - if (tm->getTargetTriple().isOSDarwin()) - module_->addModuleFlag(llvm::Module::Warning, "Dwarf Version", 2); - } else { - module_->addModuleFlag(llvm::Module::Warning, "CodeView", 1); - } - } -} - -expr_c codegen_llvm_vis_t::dispatch(expr_c v) { - emit_location(v.get()); - return ir_viewer_t::dispatch(v); -} - -stmt_c codegen_llvm_vis_t::dispatch(stmt_c v) { - emit_location(v.get()); - return ir_viewer_t::dispatch(v); -} - -const std::string &codegen_llvm_vis_t::get_node_name(const expr_c &c) { - if (c.isa()) { - return c.static_as()->name_; - } - return c.checked_as()->name_; -} - -FunctionType *codegen_llvm_vis_t::create_func_type(const func_c &v) { - std::vector tys; - for (auto ¶m : v->params_) { - tys.push_back(get_type(param->dtype_)); - } - FunctionType *FT = FunctionType::get(get_type(v->ret_type_), tys, false); - return FT; -} - -DISubroutineType *codegen_llvm_vis_t::create_func_dtype(const func_c &v) { - std::vector tys{get_type_both(v->ret_type_).second}; - for (auto ¶m : v->params_) { - tys.push_back(get_type_both(param->dtype_).second); - } - return dbuilder_->createSubroutineType(dbuilder_->getOrCreateTypeArray(tys)); -} - -Function *codegen_llvm_vis_t::get_or_create_func(const func_c &v) { - auto itr = name_to_func_.find(v->name_); - if (itr != name_to_func_.end()) { - return itr->second; - } - auto FT = create_func_type(v); - bool is_private = - v->attr_ && v->attr_->get_or_else(function_attrs::private_, false); - Function *F = Function::Create( - FT, is_private ? Function::InternalLinkage : Function::ExternalLinkage, - v->name_, module_.get()); - assert(FT == F->getFunctionType()); - for (size_t i = 0; i < v->params_.size(); i++) { - (F->arg_begin() + i)->setName(get_node_name(v->params_[i]) + "_arg"); - } - name_to_func_.insert(std::make_pair(v->name_, F)); - if (v->attr_ && v->attr_->get_or_else(function_attrs::pure, false)) { -#if SC_LLVM_BACKEND < 16 - F->addFnAttr(llvm::Attribute::AttrKind::ReadNone); -#else - F->addFnAttr(llvm::Attribute::getWithMemoryEffects( - context_, llvm::MemoryEffects::none())); -#endif -#if SC_LLVM_BACKEND > 10 - F->addFnAttr(llvm::Attribute::AttrKind::Speculatable); -#endif - } - if (v->attr_ && v->attr_->get_or_else(function_attrs::no_alias, false)) { - F->setReturnDoesNotAlias(); - } - F->addFnAttr(llvm::Attribute::AttrKind::NoUnwind); - return F; -} - -codegen_llvm_vis_t::~codegen_llvm_vis_t() = default; - -Value *codegen_llvm_vis_t::generate_expr(const expr_c &e) { - dispatch(e); - return current_val_; -} -std::pair -codegen_llvm_vis_t::do_get_type(sc_data_type_t dtype) { - Type *ty = nullptr; - DIType *dty = nullptr; - if (dtype.is_etype_pointer() && dtype.type_code_ != sc_data_etype::POINTER) { - auto ret = do_get_type(dtype.get_pointer_element()); - return {ret.first->getPointerTo(), - dbuilder_->createPointerType(ret.second, 64)}; - } - switch (dtype.type_code_) { - case sc_data_etype::UNDEF: - throw std::runtime_error("Unsupported dtype"); - case sc_data_etype::BF16: - ty = builder_.getInt16Ty(); - dty = dbuilder_->createBasicType("bf16", 16, dwarf::DW_ATE_unsigned); - break; - case sc_data_etype::F16: - ty = builder_.getHalfTy(); - dty = dbuilder_->createBasicType("f16", 16, dwarf::DW_ATE_unsigned); - break; - case sc_data_etype::U16: - ty = builder_.getInt16Ty(); - dty = dbuilder_->createBasicType("u16", 16, dwarf::DW_ATE_unsigned); - break; - case sc_data_etype::F32: - ty = builder_.getFloatTy(); - dty = dbuilder_->createBasicType("f32", 32, dwarf::DW_ATE_float); - break; - case sc_data_etype::S32: - ty = builder_.getInt32Ty(); - dty = dbuilder_->createBasicType("s32", 32, dwarf::DW_ATE_signed); - break; - case sc_data_etype::U32: - ty = builder_.getInt32Ty(); - dty = dbuilder_->createBasicType("u32", 32, dwarf::DW_ATE_unsigned); - break; - case sc_data_etype::S8: - ty = builder_.getInt8Ty(); - dty = dbuilder_->createBasicType("s8", 8, dwarf::DW_ATE_signed); - break; - case sc_data_etype::U8: - ty = builder_.getInt8Ty(); - dty = dbuilder_->createBasicType("u8", 8, dwarf::DW_ATE_unsigned); - break; - case sc_data_etype::INDEX: - case sc_data_etype::GENERIC: - ty = builder_.getInt64Ty(); - dty = dbuilder_->createBasicType("u64", 64, dwarf::DW_ATE_unsigned); - break; - case sc_data_etype::BOOLEAN: - ty = builder_.getInt1Ty(); - dty = dbuilder_->createBasicType("bool", 1, dwarf::DW_ATE_unsigned); - break; - case sc_data_etype::VOID_T: - ty = builder_.getVoidTy(); - dty = dbuilder_->createBasicType("void", 0, dwarf::DW_ATE_address); - break; - case sc_data_etype::POINTER: - ty = builder_.getInt8PtrTy(); - dty = dbuilder_->createBasicType("pointer", 64, dwarf::DW_ATE_address); - break; - - default: - assert("Unreachable" && 0); - break; - } - if (dtype.lanes_ > 1) { -#if SC_LLVM_BACKEND > 10 - ty = VectorType::get(ty, dtype.lanes_, false); -#else - ty = VectorType::get(ty, dtype.lanes_); -#endif - - auto subscript = dbuilder_->getOrCreateSubrange(0, dtype.lanes_); - llvm::DINodeArray subscriptarray = dbuilder_->getOrCreateArray(subscript); - dty = dbuilder_->createVectorType(utils::get_sizeof_type(dtype) * 8, 8, dty, - subscriptarray); - } - return {ty, dty}; -} - -std::pair -codegen_llvm_vis_t::get_type_both(sc_data_type_t dtype) { - auto itr = type_cache_.find(dtype); - if (itr != type_cache_.end()) { - return itr->second; - } - auto ret = do_get_type(dtype); - type_cache_.insert(std::make_pair(dtype, ret)); - return ret; -} - -Type *codegen_llvm_vis_t::get_type(sc_data_type_t dtype) { - return get_type_both(dtype).first; -} - -Value *codegen_llvm_vis_t::get_defined_var_ptr(const expr_c &e) { - auto itr = var_ptr_in_func_.find(e); - assert(itr != var_ptr_in_func_.end()); - return itr->second; -} - -Value *codegen_llvm_vis_t::define_var(const expr_c &e, Value *initv) { - auto ptr = builder_.CreateAlloca(get_type(e->dtype_), nullptr, - e.checked_as()->name_); - if (initv) - builder_.CreateStore(initv, ptr); - var_ptr_in_func_.insert(std::make_pair(e, ptr)); - return ptr; -} - -void codegen_llvm_vis_t::set_dbg_info_for_func_arg( - llvm::Value *v, DISubprogram *SP, DIFile *dunit, sc_data_type_t type, - const std::string &name, int argidx, int lineno, bool need_ref) { - if (!ctx_->flags_.debug_info_) { - return; - } - auto types = get_type_both(type); - auto dbgtype = types.second; - if (need_ref) { - auto tmp = builder_.CreateAlloca(v->getType()); - builder_.CreateStore(v, tmp); - v = tmp; - } - // Create a debug descriptor for the variable. - DILocalVariable *D = dbuilder_->createParameterVariable( - SP, name, argidx, dunit, lineno, dbgtype, true); - - dbuilder_->insertDeclare(v, D, dbuilder_->createExpression(), - DILocation::get(SP->getContext(), lineno, 0, SP), - builder_.GetInsertBlock()); -} - -void codegen_llvm_vis_t::emit_location(const node_base *p) { - if (!ctx_->flags_.debug_info_) { - return; - } - if (!p) { - return builder_.SetCurrentDebugLocation(DebugLoc()); - } - if (p->attr_) { - if (auto loc = p->attr_->get_or_null("source_pos")) { - DIScope *Scope; - if (dbg_scopes_.empty()) - Scope = dbg_cu_; - else - Scope = dbg_scopes_.back(); - builder_.SetCurrentDebugLocation( - DILocation::get(Scope->getContext(), loc->line_, loc->pos_, Scope)); - } - } -} - -void codegen_llvm_vis_t::prepare_alias_metadata( - const std::vector &tensors, - MDNode *alias_domain, MDBuilder &MDB) { - std::unordered_set> cliques; - int64_t new_tensor_id = -1; - for (auto &aid : tensors) { - if (aid->alias_cliques_.empty()) { - auto new_clique = std::make_shared(); - aid->add_to_clique(new_clique); - new_clique->id_ = new_tensor_id; - new_tensor_id--; - } - cliques.insert(aid->alias_cliques_.begin(), aid->alias_cliques_.end()); - } - std::vector> cliques_sorted{ - cliques.begin(), cliques.end()}; - std::sort(cliques_sorted.begin(), cliques_sorted.end(), - [](const std::shared_ptr &v1, - const std::shared_ptr &v2) { - return v1->id_ < v2->id_; - }); - std::unordered_map clique_to_MD; - for (auto &clique : cliques_sorted) { - MDNode *scope = MDB.createAnonymousAliasScope(alias_domain, - std::to_string(clique->id_)); - clique_to_MD[clique.get()] = scope; - } - for (auto &aid : tensors) { - auto shared_aid = aid->shared_from_this(); - vec_metadata alias_scope; - vec_metadata noalias_scope; - for (auto &clique : cliques_sorted) { - if (clique->set_.has(shared_aid)) { - alias_scope.emplace_back(clique_to_MD[clique.get()]); - } else { - noalias_scope.emplace_back(clique_to_MD[clique.get()]); - } - } - alias_set_to_alias_scope_[aid] = {MDNode::get(context_, alias_scope), - MDNode::get(context_, noalias_scope)}; - } -} - -func_c codegen_llvm_vis_t::dispatch(func_c v) { - var_ptr_in_func_.clear(); - tsr_to_alias_scope_.clear(); - alias_set_to_alias_scope_.clear(); - if (utils::string_startswith(v->name_, "_should_inline_")) { - return v; - } - if (!v->body_.defined()) { - return v; - } - auto F = get_or_create_func(v); - BasicBlock *BB = BasicBlock::Create(context_, "entry", F); - builder_.SetInsertPoint(BB); - - unsigned LineNo = 1; - unsigned ScopeLine = 1; - DIFile *dunit = nullptr; - DISubprogram *SP = nullptr; - if (ctx_->flags_.debug_info_) { - auto pos = v->attr_->get("source_pos"); - - LineNo = pos.line_; - ScopeLine = LineNo; - dunit = - dbuilder_->createFile(dbg_cu_->getFilename(), dbg_cu_->getDirectory()); - - SP = dbuilder_->createFunction( - dunit, v->name_, StringRef(), dunit, LineNo, create_func_dtype(v), - ScopeLine, DINode::FlagPrototyped, DISubprogram::SPFlagDefinition); - F->setSubprogram(SP); - - // Push the current scope. - dbg_scopes_.push_back(SP); - - // Unset the location for the prologue emission (leading - // instructions with no location in a function are considered - // part of the prologue and the debugger will run past them when - // breaking on a function) - emit_location(nullptr); - } - - current_func_ = F; - F->addFnAttr("no-frame-pointer-elim", "true"); - F->addFnAttr("frame-pointer", "all"); - bool has_alias = false; - - for (size_t i = 0; i < v->params_.size(); i++) { - if (v->params_[i].isa()) { - auto ainfo = alias_info::get_alias_info(*v->params_[i]); - if (ainfo && !ainfo->has_no_alias()) { - has_alias = true; - } else { - F->addParamAttr(i, llvm::Attribute::AttrKind::NoAlias); - } - - F->addParamAttr(i, llvm::Attribute::AttrKind::NoCapture); - F->addParamAttr(i, llvm::Attribute::AttrKind::NonNull); - } - } - using alias_id_vec = - std::vector>; - std::vector alias_ids; - if (v->attr_) { - if (auto local_tsr_alias_set = - v->attr_->get_or_null("alias_sets")) { - has_alias = true; - alias_ids.reserve(local_tsr_alias_set->size()); - for (auto &v : *local_tsr_alias_set) { - alias_ids.push_back(v.get()); - } - } - } - - // if has custom alias info, need to construct alias scope/noalias - // for LLVM each alias scope are exclusive to each other - if (has_alias) { - MDBuilder MDB(context_); - MDNode *alias_domain = MDB.createAnonymousAliasScopeDomain(v->name_); - - for (size_t i = 0; i < v->params_.size(); i++) { - if (v->params_[i].isa()) { - auto ainfo = alias_info::get_or_create_alias_info(*v->params_[i]); - alias_ids.push_back(ainfo.get()); - } - } - prepare_alias_metadata(alias_ids, alias_domain, MDB); - for (size_t i = 0; i < v->params_.size(); i++) { - if (v->params_[i].isa()) { - auto ainfo = alias_info::get_or_create_alias_info(*v->params_[i]); - tsr_to_alias_scope_[v->params_[i]] = - &(alias_set_to_alias_scope_[ainfo.get()]); - } - } - } - bool is_low_level = - v->attr_ && v->attr_->get_or_else(function_attrs::low_level, false); - // LLVM func args are SSA values and cannot be modified. We use - // alloca to alloc modifiable slots for each params - for (size_t i = 0; i < v->params_.size(); i++) { - Value *arg = F->args().begin() + i; - auto &p = v->params_[i]; - if (p.isa()) { - auto varnode = p.static_as(); - if (is_low_level) { - auto varalloca = define_var(v->params_[i], arg); - set_dbg_info_for_func_arg(varalloca, SP, dunit, p->dtype_, - varnode->name_, i + 1, LineNo, false); - } else { - switch (i) { - case 0: - assert(arg->getName() == "__stream_arg"); - var_ptr_in_func_.insert(std::make_pair(p, arg)); - set_dbg_info_for_func_arg(arg, SP, dunit, p->dtype_, varnode->name_, - i + 1, LineNo, true); - break; - case 1: - assert(arg->getName() == "__module_data_arg"); - var_ptr_in_func_.insert(std::make_pair(p, arg)); - set_dbg_info_for_func_arg(arg, SP, dunit, p->dtype_, varnode->name_, - i + 1, LineNo, true); - break; - default: { - auto varalloca = define_var(v->params_[i], arg); - set_dbg_info_for_func_arg(varalloca, SP, dunit, p->dtype_, - varnode->name_, i + 1, LineNo, false); - break; - } - } - } - } else { - assert(p.isa()); - auto tnode = p.static_as(); - var_ptr_in_func_.insert(std::make_pair(p, arg)); - set_dbg_info_for_func_arg(arg, SP, dunit, tnode->dtype_, tnode->name_, - i + 1, LineNo, true); - } - } - dispatch(v->body_); - if (builder_.GetInsertBlock()->empty() || - !builder_.GetInsertBlock()->back().isTerminator()) { - assert(v->ret_type_ == datatypes::void_t); - builder_.CreateRetVoid(); - } - if (ctx_->flags_.debug_info_) { - dbg_scopes_.pop_back(); - } - return v; -} - -Instruction *codegen_llvm_vis_t::set_alias(Instruction *inst, - const expr_c &tsr) { - if (tsr.isa()) { - return set_alias(inst, tsr.static_as()->ptr_); - } - auto itr = tsr_to_alias_scope_.find(tsr); - if (itr != tsr_to_alias_scope_.end()) { - // alias.scope metadata. - inst->setMetadata( - LLVMContext::MD_alias_scope, - MDNode::concatenate(inst->getMetadata(LLVMContext::MD_alias_scope), - itr->second->first)); - - // noalias metadata. - inst->setMetadata( - LLVMContext::MD_noalias, - MDNode::concatenate(inst->getMetadata(LLVMContext::MD_noalias), - itr->second->second)); - } - return inst; -} - -Value *codegen_llvm_vis_t::set_alias(Value *inst, const expr_c &tsr) { - set_alias(static_cast(inst), tsr); - return inst; -} - -Value *codegen_llvm_vis_t::convert_mask(const expr &in, const bool is_int4) { - // true means mask must have 4 bits. Otherwise, it is always false. - auto mask = generate_expr(in); - auto &dtype = in->dtype_; - if (dtype.lanes_ == 1 && !dtype.is_etype(sc_data_etype::BOOLEAN)) { - auto ty_int1 = builder_.getInt1Ty(); - auto bit_len = is_int4 ? 4 : utils::get_sizeof_type(dtype) * 8; - auto mask_ty = -#if SC_LLVM_BACKEND > 10 - VectorType::get(ty_int1, bit_len, false); -#else - VectorType::get(ty_int1, bit_len); -#endif - if (is_int4) { - mask = builder_.CreateTrunc(mask, builder_.getIntNTy(4)); - } - mask = builder_.CreateBitCast(mask, mask_ty); - } - return mask; -} - -void codegen_llvm_vis_t::set_dbg_info_for_local_var(const source_pos *pos, - sc_data_type_t type, - const std::string &name, - Value *llvm_value, - bool need_ref) { - if (!ctx_->flags_.debug_info_) { - return; - } - auto types = get_type_both(type); - auto dbgtype = types.second; - if (need_ref) { - auto tmp = builder_.CreateAlloca(llvm_value->getType()); - builder_.CreateStore(llvm_value, tmp); - llvm_value = tmp; - } - // Create a debug descriptor for the variable. - DILocalVariable *D = dbuilder_->createAutoVariable( - dbg_scopes_.back(), name, dbg_cu_->getFile(), pos->line_, dbgtype, true); - - dbuilder_->insertDeclare(llvm_value, D, dbuilder_->createExpression(), - DILocation::get(dbg_scopes_.back()->getContext(), - pos->line_, pos->pos_, - dbg_scopes_.back()), - builder_.GetInsertBlock()); -} - -void codegen_llvm_vis_t::set_dbg_info_for_local_var(const define_node_t *v, - const std::string &name, - Value *llvm_value, - bool need_ref) { - if (!ctx_->flags_.debug_info_) { - return; - } - auto pos = v->attr_->get_or_null("source_pos"); - if (pos) { - set_dbg_info_for_local_var(pos, v->var_->dtype_, name, llvm_value, - need_ref); - } -} - -Value *codegen_llvm_vis_t::gen_vec_const(uint64_t elements, float f) { - return builder_.CreateVectorSplat( - elements, ConstantFP::get(builder_.getFloatTy(), APFloat(f))); -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/codegen/precodegen_passes.cpp b/legacy/core/src/compiler/codegen/precodegen_passes.cpp deleted file mode 100644 index da5105ce4..000000000 --- a/legacy/core/src/compiler/codegen/precodegen_passes.cpp +++ /dev/null @@ -1,190 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#include -#include -#include - -#include "precodegen_passes.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -sequential_module_pass_t get_ssa_passes(const context_ptr &ctx) { - std::vector ret; - ret.emplace_back(module_function_pass_t::make()); - ret.emplace_back(module_function_pass_t::make(false)); - ret.emplace_back( - module_function_pass_t::make()); - ret.emplace_back(module_function_pass_t::make()); - return sequential_module_pass_t(std::move(ret)); -} - -sequential_module_pass_t get_default_precodegen_passes(const context_ptr &ctx, - bool gen_wrapper) { - std::vector ret; - ret.reserve(64); - ret.emplace_back(utils::make_unique()); - if (gen_wrapper) { - ret.emplace_back(utils::make_unique()); - } - ret.emplace_back(utils::make_unique()); - if (ctx->flags_.concat_optimization_) { - ret.emplace_back(utils::make_unique()); - } - ret.emplace_back(utils::make_unique()); - ret.emplace_back(utils::make_unique()); - ret.emplace_back(module_function_pass_t::make(ctx)); - ret.emplace_back(utils::make_unique()); - if (ctx->flags_.trace_) { - ret.emplace_back(utils::make_unique()); - } - - ret.emplace_back(utils::make_unique()); - ret.emplace_back(utils::make_unique()); - ret.emplace_back(module_function_pass_t::make(true)); - - if (ctx->flags_.buffer_schedule_ > 0 && ctx->flags_.tensor_inplace_) { - ret.emplace_back(utils::make_unique(ctx)); - } - ret.emplace_back(module_function_pass_t::make(ctx)); - ret.emplace_back(module_function_pass_t::make()); - - ret.emplace_back( - module_function_pass_t::make()); - ret.emplace_back( - module_function_pass_t::make()); - ret.emplace_back(utils::make_unique(false)); - ret.emplace_back(module_function_pass_t::make()); - ret.emplace_back(module_function_pass_t::make(true)); - if (ctx->flags_.index2var_) { - ret.emplace_back(module_function_pass_t::make()); - } - if (ctx->flags_.dead_write_elimination_) { - ret.emplace_back(module_function_pass_t::make()); - } - ret.emplace_back(module_function_pass_t::make(true)); - - if (runtime_config_t::get().managed_thread_pool_ == - thread_pool_mode_t::DYNAMIC) { - ret.emplace_back(utils::make_unique(true)); - } else { - ret.emplace_back( - module_function_pass_t::make( - ctx, true, ctx->flags_.tensor_inplace_)); - ret.emplace_back( - module_function_pass_t::make()); - } - ret.emplace_back(utils::make_unique(false)); - ret.emplace_back(module_function_pass_t::make(true)); - ret.emplace_back(module_function_pass_t::make()); - - ret.emplace_back(utils::make_unique()); - ret.emplace_back(utils::make_unique()); - ret.emplace_back(module_function_pass_t::make(ctx)); - ret.emplace_back(utils::make_unique(ctx)); - ret.emplace_back(utils::make_unique()); - ret.emplace_back(utils::make_unique()); - ret.emplace_back(module_function_pass_t::make(false)); - - if (ctx->flags_.tensor2var_) { - ret.emplace_back(module_function_pass_t::make()); - } - if (ctx->flags_.buffer_schedule_ > 0) { - ret.emplace_back(module_function_pass_t::make( - ctx, true, ctx->flags_.tensor_inplace_)); - } - ret.emplace_back( - utils::make_unique(ctx->flags_.kernel_optim_)); - ret.emplace_back(utils::make_unique( - runtime_config_t::get().get_num_threads() == 1)); - ret.emplace_back(module_function_pass_t::make(false)); - - ret.emplace_back(utils::make_unique()); - ret.emplace_back( - module_function_pass_t::make(128)); - if (ctx->flags_.ssa_passes_) { - ret.emplace_back(module_function_pass_t::make()); - ret.emplace_back(module_function_pass_t::make()); - ret.emplace_back( - module_function_pass_t::make()); - ret.emplace_back(module_function_pass_t::make()); - } - validate_pass_order(ctx, ret, gen_wrapper); - return sequential_module_pass_t(std::move(ret)); -} - -const_ir_module_ptr run_precodegen_passes(module_pass_t &pass, - const_ir_module_ptr mod) { // NOLINT - func_t init_func = mod->make_init_func(); - auto mod_with_init = std::make_shared(*mod); - if (init_func) { - mod_with_init->add_func({init_func}); - } - // todo: use attr in function to skip some of the passes - auto mod_cpy = pass(mod_with_init); - if (mod->ctx_->flags_.print_ir_) { - std::cerr << mod_cpy; - } - return mod_cpy; -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/codegen/precodegen_passes.hpp b/legacy/core/src/compiler/codegen/precodegen_passes.hpp deleted file mode 100644 index 8e7f207c3..000000000 --- a/legacy/core/src/compiler/codegen/precodegen_passes.hpp +++ /dev/null @@ -1,38 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_CODEGEN_PRECODEGEN_PASSES_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_CODEGEN_PRECODEGEN_PASSES_HPP - -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -sequential_module_pass_t get_default_precodegen_passes(const context_ptr &ctx, - bool gen_wrapper); - -const_ir_module_ptr run_precodegen_passes(module_pass_t &pass, - const_ir_module_ptr f); -sequential_module_pass_t get_ssa_passes(const context_ptr &ctx); -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/config/context.cpp b/legacy/core/src/compiler/config/context.cpp deleted file mode 100644 index 97c65750e..000000000 --- a/legacy/core/src/compiler/config/context.cpp +++ /dev/null @@ -1,243 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#include "context.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifndef _WIN32 -extern char **environ; -#endif - -SC_MODULE(target) -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -using namespace env_key; - -static void check_within(int &val, int lo, int hi, int defaultv, - const char *prompt) { - if (val < lo || val > hi) { - SC_MODULE_WARN << prompt << val << ", set to default = " << defaultv; - val = defaultv; - } -} - -// The function reset some cpu flags by DNNL_MAX_CPU_ISA/ONEDNN_MAX_CPU_ISA, cpu -// flags will take the intersection of compiler/target machine/onednn env -// variable/os. -static void reset_cpu_flags_by_dnnl_envs(runtime::target_machine_t &tm) { - std::string dnnl_isa = utils::getenv_string("DNNL_MAX_CPU_ISA"); - std::string onednn_isa = utils::getenv_string("ONEDNN_MAX_CPU_ISA"); - // DNNL_MAX_CPU_ISA and ONEDNN_MAX_CPU_ISA can not be set at same time. - assert(dnnl_isa.empty() || onednn_isa.empty()); - std::string &max_isa = onednn_isa.empty() ? dnnl_isa : onednn_isa; - if (!max_isa.empty()) { - // copy here for recover if env is invalid - auto old_cpu_flags = tm.cpu_flags_; - do { - // amx fp16 - if (max_isa == "AVX512_CORE_AMX_FP16") { - break; - } - tm.cpu_flags_.fAVX512AMXFP16 = false; - // amx - if (max_isa == "AVX512_CORE_AMX") { - break; - } - tm.cpu_flags_.fAVX512AMXTILE = false; - tm.cpu_flags_.fAVX512AMXINT8 = false; - tm.cpu_flags_.fAVX512AMXBF16 = false; - // avx512 fp16 - if (max_isa == "AVX512_CORE_FP16") { - break; - } - tm.cpu_flags_.fAVX512FP16 = false; - // avx512 bf16 - if (max_isa == "AVX512_CORE_BF16") { - break; - } - tm.cpu_flags_.fAVX512BF16 = false; - // avx512 vnni - if (max_isa == "AVX512_CORE_VNNI") { - break; - } - tm.cpu_flags_.fAVX512VNNI = false; - // avx512 core - if (max_isa == "AVX512_CORE") { - break; - } - tm.cpu_flags_.fAVX512F = false; - tm.cpu_flags_.fAVX512BW = false; - tm.cpu_flags_.fAVX512VL = false; - tm.cpu_flags_.fAVX512DQ = false; - // avx vnni 2 - if (max_isa == "AVX2_VNNI_2") { - break; - } - // avx vnni - if (max_isa == "AVX2_VNNI") { - break; - } - // avx2 - if (max_isa == "AVX2") { - break; - } - tm.cpu_flags_.fAVX2 = false; - // avx - if (max_isa == "AVX") { - break; - } - tm.cpu_flags_.fAVX = false; - // sse41 - if (max_isa == "SSE41") { - break; - } - - // unsupport - SC_MODULE_WARN << "Unsupported ISA type: " << max_isa; - - // use old cpu flags - tm.cpu_flags_ = old_cpu_flags; - } while (false); - runtime::target_machine_t::set_simd_length_and_max_cpu_threads( - tm.cpu_flags_); - } - // double check amx by syscall - if (tm.cpu_flags_.fAVX512AMXTILE && - !dnnl::impl::cpu::x64::amx::is_available()) { - tm.cpu_flags_.fAVX512AMXTILE = false; - tm.cpu_flags_.fAVX512AMXINT8 = false; - tm.cpu_flags_.fAVX512AMXBF16 = false; - } -} - -template static void parse_value(const char *name, T &v) { - auto strv = utils::getenv_string(name); - if (!strv.empty()) { - v = T(std::stoi(strv)); - }; -} - -context_ptr make_context_from_env() { - runtime::target_machine_t tm = runtime::get_native_target_machine(); - scflags_t flags; - -#if defined(SC_LLVM_BACKEND) - jit_kind jit = jit_kind::llvm; -#elif SC_BUILTIN_JIT_ENABLED - jit_kind jit = jit_kind::xbyak; -#elif SC_CFAKE_JIT_ENABLED - jit_kind jit = jit_kind::cfake; -#endif - { - const char *jit_env_var_name = env_names[SC_CPU_JIT]; - const char *cfakejit_switch_name = "c"; - auto buf = utils::getenv_string(jit_env_var_name); - if (!buf.empty()) { -#if SC_CFAKE_JIT_ENABLED - if (buf == cfakejit_switch_name) { - jit = jit_kind::cfake; - } -#else - if (false) { - // make compiler happy - } -#endif -#if SC_BUILTIN_JIT_ENABLED - else if (buf == "builtin") { - jit = jit_kind::xbyak; - } -#endif -#if defined(SC_LLVM_BACKEND) - else if (buf == "llvm") { - jit = jit_kind::llvm; - } -#endif - else { - SC_MODULE_WARN << "Bad value for SC_CPU_JIT=" << buf - << ", setting to default value=" -#if defined(SC_LLVM_BACKEND) - "llvm"; -#elif SC_BUILTIN_JIT_ENABLED - "builtin"; -#elif SC_CFAKE_JIT_ENABLED - "cfake"; -#endif - } - } - } - - flags.jit_kind_ = jit; - jit_engine_t::set_target_machine(jit, flags, tm); - reset_cpu_flags_by_dnnl_envs(tm); - tm.brgemm_use_amx_ = - tm.cpu_flags_.fAVX512AMXTILE && - (tm.cpu_flags_.fAVX512AMXBF16 || tm.cpu_flags_.fAVX512AMXINT8); - set_runtime_target_machine(tm); - std::string tracep = utils::getenv_string(env_names[SC_TRACE]); - if (!tracep.empty() && tracep != "0") { - flags.trace_ = true; - SC_MODULE_WARN << "Trace is ON"; - } - - int opt_level = utils::getenv_int(env_names[SC_OPT_LEVEL], 3); - check_within(opt_level, 0, 3, 3, "Bad optimization level in SC_OPT_LEVEL: "); - flags.opt_level_ = sc_opt_level(opt_level); - - if (opt_level == 0) { - // disable opt passes - flags.buffer_schedule_ = 0; - flags.dead_write_elimination_ = false; - flags.kernel_optim_ = 0; - flags.index2var_ = false; - flags.tensor_inplace_ = false; - } - - return std::make_shared(flags, std::move(tm)); -} - -context_ptr get_default_context() { - static auto v = make_context_from_env(); - return v; -} - -uint16_t context_t::get_max_vector_lanes(sc_data_etype etype) const { - return machine_.get_device_flags().get_max_vector_lanes(etype); -} - -bool context_t::use_amx() const { return machine_.use_amx(); } - -context_t::context_t(const scflags_t &flags, - runtime::target_machine_t &&machine, - runtime::engine_t *engine) - : engine_(engine ? engine : runtime::get_default_stream()->engine_), - flags_(flags), machine_(std::move(machine)) {} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/config/context.hpp b/legacy/core/src/compiler/config/context.hpp deleted file mode 100644 index c9072efa1..000000000 --- a/legacy/core/src/compiler/config/context.hpp +++ /dev/null @@ -1,89 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_CONFIG_CONTEXT_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_CONFIG_CONTEXT_HPP -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace runtime { -struct engine_t; -} - -enum class jit_kind { - cfake = 0, -#if defined(SC_LLVM_BACKEND) - llvm, -#endif - xbyak, -}; - -enum class sc_opt_level : int { lv0 = 0, lv1, lv2, lv3 }; -enum class fusion_opt_level : int { lv0 = 0, lv1, lv2, lv3 }; - -struct scflags_t { - enum class brgemm_backend_t : int { dnnl = 0, max_num }; - - jit_kind jit_kind_ = jit_kind::cfake; - sc_opt_level opt_level_ = sc_opt_level::lv3; - int backend_opt_level_ = 3; - bool tensor_inplace_ = true; - bool bf16_fast_trunc_ = false; - bool const_share_ = true; - bool trace_ = false; - bool dead_write_elimination_ = true; - int buffer_schedule_ = 3; // 0 off, 1 whole reuse, 2 size first, 3 hot first - brgemm_backend_t brgemm_backend_ = brgemm_backend_t::dnnl; - int kernel_optim_ = 1; // 0 off, 1 external-runtime-oriented opt, - bool index2var_ = true; - bool tensor2var_ = true; - bool print_ir_ = false; - bool ssa_passes_ = false; - bool prefetch_ = true; - fusion_opt_level fusion_level_ = fusion_opt_level::lv3; - bool use_cost_model_ = true; - bool debug_info_ = false; - // whether jit supports directly generating amx intrinsics instead of using - // dnnl - bool jit_support_amx_intrinsics_ = false; - bool concat_optimization_ = true; - bool graph_default_private_ = true; -}; - -struct context_t { - runtime::engine_t *engine_; - scflags_t flags_; - runtime::target_machine_t machine_; - context_t(const scflags_t &flags, runtime::target_machine_t &&machine, - runtime::engine_t *engine = nullptr); - context_t(const context_t &) = default; - uint16_t get_max_vector_lanes(sc_data_etype etype) const; - bool use_amx() const; -}; -using context_ptr = std::shared_ptr; - -SC_API context_ptr make_context_from_env(); -SC_API context_ptr get_default_context(); -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl -#endif diff --git a/legacy/core/src/compiler/dimensions.hpp b/legacy/core/src/compiler/dimensions.hpp deleted file mode 100644 index 45d876a2c..000000000 --- a/legacy/core/src/compiler/dimensions.hpp +++ /dev/null @@ -1,44 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_DIMENSIONS_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_DIMENSIONS_HPP -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -using sc_dim = int64_t; -using sc_dims = std::vector; -namespace dimensions { -constexpr sc_dim dynamic_any = -1; -} - -inline uint64_t dim2unsigned(sc_dim v) { - assert(v >= 0); - return v; -} - -inline bool is_dynamic_dim(sc_dim v) { return v < 0; } - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/attr_keys.hpp b/legacy/core/src/compiler/ir/attr_keys.hpp deleted file mode 100644 index 39109c3b0..000000000 --- a/legacy/core/src/compiler/ir/attr_keys.hpp +++ /dev/null @@ -1,67 +0,0 @@ -/******************************************************************************* - * Copyright 2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_ATTR_KEYS_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_ATTR_KEYS_HPP - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -namespace attr_keys { -// bool. Default false. Applied on local tensors. If true, the tensor must be -// allocated using runtime memory allocator instead of the native stack -constexpr const char *runtime_stack_alloc = "runtime_stack_alloc"; -// string. The name of the next function to call. Applied on evaluate-call -// nodes. If true, the next op to be called has no dependency on the current op. -// It can be merged with the previous op to remove the barrier. -constexpr const char *no_post_barrier = "no_post_barrier"; -// bool. Default false. It should be applied on tensors, indicating that the -// tensor should be 1xs32 and its element's value is auto-assigned by the -// compiler. The content of the tensor is bind to whether the const cache is -// valid and already initialized. -constexpr const char *is_init_for_const_cache = "is_init_for_const_cache"; -// std::shared_ptr. It should be marked on shared -// constant tensor nodes -constexpr const char *shared_const = "shared_const"; -// bool. Default false. Marked on stmt nodes that is related to shared constant -// lazy initialization flag __is_init. module_global_resolver uses this flag to -// find the __is_init related stmt. -constexpr const char *is_shared_const_init_stmt = "is_shared_const_init_stmt"; -// size_t. Marked by module_global_resolver on tensor nodes -// ("__shared_const_base_*") that are base tensors of buffers in shared const -// tensor cache. It is the index of the handle to these base tensors. It will be -// used in local_tensor_lower -constexpr const char *shared_const_base_idx = "shared_const_base_idx"; -// bool. Default false. Marked "true" on top-level parallel for_loop nodes that -// the user would like to use dynamic parallel dispatching -constexpr const char *dynamic_parallel = "dynamic_parallel"; -// bool. Default false. Applied on base tensor. -// If true, the tensor is read only in the funtion. -constexpr const char *read_only_tensor = "read_only_tensor"; -// bool. whether codegen enable fast math. -// If false, codegen will not use fast math in the calculation. (Currently only -// works in binary expr calculation of llvm codegen.) -constexpr const char *fast_math = "fast_math"; -} // namespace attr_keys - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/builder.cpp b/legacy/core/src/compiler/ir/builder.cpp deleted file mode 100644 index e535160e1..000000000 --- a/legacy/core/src/compiler/ir/builder.cpp +++ /dev/null @@ -1,936 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "builder.hpp" - -#include - -#include "passlet/passlet.hpp" -#include "passlet/structural_analysis.hpp" -#include "sc_expr.hpp" -#include "sc_function.hpp" -#include "sc_stmt.hpp" -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -static void merge_attrs(std::unique_ptr &mergeto, - const std::unique_ptr &mergefrom) { - if (mergeto) { - auto &ths_attr_map = mergefrom->as_map(); - mergeto->as_map().insert(ths_attr_map.begin(), ths_attr_map.end()); - } else { - mergeto = utils::make_unique(*mergefrom); - } -} - -static void copy_temp_data(const node_base *src, node_base *dst) { - if (src->temp_data_) { - auto &data = *src->temp_data_; - if (!data.vtable()) { - return; - } - if (!data.vtable()->copy_assigner_ && !data.vtable()->copy_ctor_) { - return; - } - dst->temp_data() = data; - } -} - -expr copy_attr(const expr_base &ths, expr &&newexpr) { - if (ths.attr_) { - merge_attrs(newexpr->attr_, ths.attr_); - } - copy_temp_data(&ths, newexpr.get()); - return std::move(newexpr); -} - -stmt copy_attr(const stmt_base_t &ths, stmt &&newstmt) { - if (ths.attr_) { - merge_attrs(newstmt->attr_, ths.attr_); - } - copy_temp_data(&ths, newstmt.get()); - return std::move(newstmt); -} - -func_t copy_attr(const func_base &ths, func_t &&newfunc) { - if (ths.attr_) { - merge_attrs(newfunc->attr_, ths.attr_); - } - copy_temp_data(&ths, newfunc.get()); - return std::move(newfunc); -} - -static std::vector vector_remove_const(const std::vector &v) { - std::vector ret; - ret.reserve(v.size()); - for (auto &i : v) { - ret.emplace_back(i.remove_const()); - } - return ret; -} - -// add ret to parent node of s. -void add_parent_node(const stmt &s, const stmt &ret) { - s->attr()["builder.parent_node"] = passlet::structural_result_t{ret, s}; -} - -passlet::structural_result_t *get_parent_struct(const stmt &v) { - if (!v->attr_) - return nullptr; - return v->attr_->get_or_null( - "builder.parent_node"); -} - -// This function can find the parent node in IR, if the node has no parent -// node, return stmt(). -stmt get_parent_node(const stmt &node) { - auto stru_res = get_parent_struct(node); - if (!stru_res) - return stmt(); - return stru_res->get_parent_node(); -} - -stmt get_common_parent_node(const stmt &node1, const stmt &node2) { - auto stru_res1 = get_parent_struct(node1), - stru_res2 = get_parent_struct(node2); - if (!stru_res1 || !stru_res2) - return stmt(); - return stmt{ - const_cast( - stru_res1->find_shared_parent( - *stru_res2, - [](passlet::passlet_t *ths, - const node_base *v) -> passlet::structural_result_t * { - if (!v->attr_) - return nullptr; - return v->attr_->get_or_null( - "builder.parent_node"); - }, - true, true)) - ->shared_from_this()}; -} - -expr &get_base_tensor(expr &buffer) { - expr *tsr = &buffer; - while (!tsr->isa()) { - COMPILE_ASSERT(tsr->isa(), - "Only tensor or tensorptr is accepted") - auto base = tsr->static_as()->base_; - COMPILE_ASSERT(base.isa(), - "tensor_ptr base should be indexing, but got: " << base); - tsr = &(base.static_as()->ptr_); - } - return *tsr; -} - -tensor get_real_tensor(const expr &buffer) { - auto buf = buffer; - return get_base_tensor(buf).checked_as(); -} - -void set_base_tensor(expr &tptr, const expr &tsr) { - get_base_tensor(tptr) = tsr; -} - -namespace builder { - -static thread_local builder_impl_t *cur_builder = nullptr; - -builder_impl_t *get_current_builder() { return cur_builder; } - -void set_current_builder(builder_impl_t *b) { cur_builder = b; } - -expr make_constant(float val) { return make_expr(val); }; - -expr make_constant(int32_t val) { - return make_expr(static_cast(val)); -}; - -expr make_constant(uint64_t val) { return make_expr(val); }; - -expr make_constant(const std::vector &val, sc_data_type_t dtype) { - return make_expr(val, dtype); -} - -expr make_var(sc_data_type_t type, const std::string &name) { - return make_expr(type, name); -}; - -expr make_cast(sc_data_type_t type, const expr_c &in) { - return make_expr(type, in.remove_const()); -} - -func_t make_func(const std::string &name, const std::vector ¶ms, - stmt body, sc_data_type_t ret_type) { - return func_t(new func_base(name, params, std::move(body), ret_type)); -} - -func_t make_func(const std::string &name, const std::vector ¶ms, - const stmt_c &body, sc_data_type_t ret_type) { - return make_func(name, vector_remove_const(params), body.remove_const(), - ret_type); -} - -expr builder_impl_t::make_str(const std::string &str) { - static std::atomic counter = {0}; - expr ret = - make_tensor("_str_0_" + std::to_string(++counter), - std::vector{expr(str.size() + 1)}, datatypes::u8); - push_var_tensor_def(ret, linkage::local); - for (size_t i = 0; i < str.size(); i++) { - push_assign(make_indexing(ret, expr(i)), - make_expr(uint64_t(str.at(i)), datatypes::u8)); - } - push_assign(make_indexing(ret, expr(str.size())), - make_expr(uint64_t(0), datatypes::u8)); - return ret; -} - -expr tensor_ptr(const expr &tens, const std::vector &idx, - const std::vector &shape, bool is_slice) { - COMPILE_ASSERT( - tens.isa() || tens.isa(), - "tensor_ptr only accepts a tensor or tensorptr, got: " << tens); - const std::vector *real_shape; - if (is_slice && shape.empty()) { - if (tens.isa()) { - real_shape = &tens.static_as()->dims_; - } else { - real_shape = &tens.static_as()->shape_; - } - } else { - real_shape = &shape; - } - return make_expr(make_expr(tens, idx, expr()), - *real_shape, is_slice); -} - -expr tensor_ptr(const expr_c &tens, const std::vector &idx, - const std::vector &shape, bool is_slice) { - return tensor_ptr(tens.remove_const(), vector_remove_const(idx), - vector_remove_const(shape), is_slice); -} - -expr tensor_ptr(const expr &tensor, std::initializer_list idx, - std::initializer_list shape, bool is_slice) { - return tensor_ptr(tensor, std::vector(idx), std::vector(shape), - is_slice); -} - -expr remake_binary(const expr_c &l, const expr_c &r, const expr_c &original) { - if (original.isa()) { - auto orig_type = original.static_as()->type_; - switch (orig_type) { - case intrin_type::min: - case intrin_type::max: - case intrin_type::int_and: - case intrin_type::int_or: - case intrin_type::int_xor: - case intrin_type::shl: - case intrin_type::shr: - return copy_attr( - *original, - make_expr( - orig_type, std::vector{l.remove_const(), r.remove_const()}, - *original.static_as()->intrin_attrs_)); - default: - assert(0 && "Bad op for remake_binary"); - } - } - switch (original->node_type_) { - case sc_expr_type::add: - return copy_attr(*original, make_add(l, r)); - case sc_expr_type::sub: - return copy_attr(*original, make_sub(l, r)); - case sc_expr_type::mul: - return copy_attr(*original, make_mul(l, r)); - case sc_expr_type::div: - return copy_attr(*original, make_div(l, r)); - case sc_expr_type::mod: - return copy_attr(*original, make_mod(l, r)); - case sc_expr_type::cmp_eq: - return copy_attr(*original, make_cmp_eq(l, r)); - case sc_expr_type::cmp_ne: - return copy_attr(*original, make_cmp_ne(l, r)); - case sc_expr_type::cmp_lt: - return copy_attr(*original, make_cmp_lt(l, r)); - case sc_expr_type::cmp_le: - return copy_attr(*original, make_cmp_le(l, r)); - case sc_expr_type::cmp_gt: - return copy_attr(*original, make_cmp_gt(l, r)); - case sc_expr_type::cmp_ge: - return copy_attr(*original, make_cmp_ge(l, r)); - case sc_expr_type::logic_and: - return copy_attr(*original, make_logic_and(l, r)); - case sc_expr_type::logic_or: - return copy_attr(*original, make_logic_or(l, r)); - default: - assert(0 && "Bad op for remake_binary"); - return expr(); - } -} - -#define GEN_BINARY(name) \ - expr make_##name(const expr_c &left, const expr_c &right) { \ - return make_expr(left.remove_const(), right.remove_const()); \ - }; - -expr make_func_addr(func_t v) { - return make_expr(std::move(v)); -} - -expr make_phi(const std::vector &values, bool is_loop_phi) { - return make_expr(values, is_loop_phi); -} - -intrin_call remake_intrin_call(const intrin_call_c &v, - const std::vector &newargs) { - return make_expr(v->type_, newargs, *v->intrin_attrs_); -} - -low_level_intrin remake_low_level_intrin(const low_level_intrin_c &v, - const std::vector &newargs) { - low_level_intrin new_intrin = v->remake().static_as(); - new_intrin->args_ = newargs; - return new_intrin; -} - -expr make_x86_intrin(x86_intrin_type::x86_intrin_type_t type, - const std::vector &args, const any_map_t &attrs) { - auto intrin = make_expr( - low_level_intrin_kind::x86_general, // x86 low level kind - static_cast(type), // x86 intrin type - args, attrs); - get_x86_intrinsic_handler(intrin->type_).on_initialize(*intrin); - return intrin; -} - -expr make_reinterpret(const expr_c &v, sc_data_type_t dtype) { - any_map_t attr; - attr[intrin_attr::out_dtype] = dtype; - auto ret = make_expr( - intrin_type::reinterpret, std::vector{v.remove_const()}, attr); - return ret; -} - -expr make_isnan(const expr_c &v) { - auto ret = make_expr( - intrin_type::isnan, std::vector{v.remove_const()}, any_map_t()); - ret->dtype_ = sc_data_type_t::boolean(v->dtype_.lanes_); - return ret; -} - -expr make_saturated_cast(const expr_c &v, sc_data_type_t dtype) { - any_map_t attr; - attr[intrin_attr::out_dtype] = dtype; - auto ret = make_expr( - intrin_type::saturated_cast, std::vector{v.remove_const()}, attr); - return ret; -} - -expr make_round_and_cast(const expr_c &v, sc_data_type_t dtype) { - return make_expr( - intrin_type::round_and_cast, std::vector{v.remove_const()}, - any_map_t{{intrin_attr::out_dtype, dtype}}); -} - -expr make_min(const expr_c &left, const expr_c &right) { - return make_expr( - intrin_type::min, - std::vector{left.remove_const(), right.remove_const()}, - any_map_t()); -} - -expr make_max(const expr_c &left, const expr_c &right) { - return make_expr( - intrin_type::max, - std::vector{left.remove_const(), right.remove_const()}, - any_map_t()); -} - -expr make_int_and(const expr_c &left, const expr_c &right) { - return make_expr( - intrin_type::int_and, - std::vector{left.remove_const(), right.remove_const()}, - any_map_t()); -} - -expr make_int_or(const expr_c &left, const expr_c &right) { - return make_expr( - intrin_type::int_or, - std::vector{left.remove_const(), right.remove_const()}, - any_map_t()); -} - -expr make_int_xor(const expr_c &left, const expr_c &right) { - return make_expr( - intrin_type::int_xor, - std::vector{left.remove_const(), right.remove_const()}, - any_map_t()); -} - -expr make_shl(const expr_c &left, const expr_c &right) { - return make_expr( - intrin_type::shl, - std::vector{left.remove_const(), right.remove_const()}, - any_map_t()); -} - -expr make_shr(const expr_c &left, const expr_c &right) { - return make_expr( - intrin_type::shr, - std::vector{left.remove_const(), right.remove_const()}, - any_map_t()); -} - -expr make_abs(const expr_c &v) { - return make_expr( - intrin_type::abs, std::vector{v.remove_const()}, any_map_t()); -} - -expr make_round(const expr_c &v) { - return make_expr( - intrin_type::round, std::vector{v.remove_const()}, any_map_t()); -} - -expr make_floor(const expr_c &v) { - return make_expr( - intrin_type::floor, std::vector{v.remove_const()}, any_map_t()); -} - -expr make_ceil(const expr_c &v) { - return make_expr( - intrin_type::ceil, std::vector{v.remove_const()}, any_map_t()); -} - -expr make_exp(const expr_c &v) { - return make_expr( - intrin_type::exp, std::vector{v.remove_const()}, any_map_t()); -} - -expr make_log(const expr_c &v) { - return make_expr( - intrin_type::log, std::vector{v.remove_const()}, any_map_t()); -} - -expr make_erf(const expr_c &v) { - return make_expr( - intrin_type::erf, std::vector{v.remove_const()}, any_map_t()); -} - -expr make_sqrt(const expr_c &v) { - return make_expr( - intrin_type::sqrt, std::vector{v.remove_const()}, any_map_t()); -} - -expr make_rsqrt(const expr_c &v) { - return make_expr( - intrin_type::rsqrt, std::vector{v.remove_const()}, any_map_t()); -} - -expr make_reduce_add(const expr_c &v) { - return make_expr(intrin_type::reduce_add, - std::vector{v.remove_const()}, - any_map_t()); -} - -expr make_reduce_mul(const expr_c &v) { - return make_expr(intrin_type::reduce_mul, - std::vector{v.remove_const()}, - any_map_t()); -} - -expr make_reduce_max(const expr_c &v) { - return make_expr(intrin_type::reduce_max, - std::vector{v.remove_const()}, - any_map_t()); -} - -expr make_reduce_min(const expr_c &v) { - return make_expr(intrin_type::reduce_min, - std::vector{v.remove_const()}, - any_map_t()); -} - -expr make_broadcast(const expr_c &v, int lanes) { - return make_expr(intrin_type::broadcast, - std::vector{v.remove_const()}, - any_map_t{{"lanes", lanes}}); -} - -expr make_fmadd(const expr_c &v_a, const expr_c &v_b, const expr_c &v_c) { - return make_expr(intrin_type::fmadd, - std::vector{v_a.remove_const(), - v_b.remove_const(), - v_c.remove_const()}, - any_map_t()); -} - -expr make_fnmadd(const expr_c &v_a, const expr_c &v_b, const expr_c &v_c) { - return make_expr(intrin_type::fnmadd, - std::vector{v_a.remove_const(), - v_b.remove_const(), - v_c.remove_const()}, - any_map_t()); -} - -expr make_unpack_low(const expr_c &v_a, const expr_c &v_b, int elem_bits) { - return make_expr( - intrin_type::unpack_low, - std::vector{v_a.remove_const(), v_b.remove_const()}, - any_map_t{{"elem_bits", elem_bits}}); -} - -expr make_unpack_high(const expr_c &v_a, const expr_c &v_b, int elem_bits) { - return make_expr( - intrin_type::unpack_high, - std::vector{v_a.remove_const(), v_b.remove_const()}, - any_map_t{{"elem_bits", elem_bits}}); -} - -expr make_shuffle(const expr_c &v_a, const expr_c &v_b, const int &v_c, - const int &type_bits) { - return make_expr( - intrin_type::shuffle, - std::vector{v_a.remove_const(), v_b.remove_const()}, - any_map_t{{"shuffle_imm", v_c}, {"type_bits", type_bits}}); -} - -expr make_permute(const expr_c &v_a, const expr_c &v_b, const int &v_c, - const int &type_bits) { - return make_expr( - intrin_type::permute, - std::vector{v_a.remove_const(), v_b.remove_const()}, - any_map_t{{"permute_imm", v_c}, {"type_bits", type_bits}}); -} - -expr make_gather(const expr_c &addr, const expr_c &indices) { - return make_expr( - intrin_type::gather, - std::vector{addr.remove_const(), indices.remove_const()}, - any_map_t()); -} - -expr make_permutex2var(const expr_c &v_a, const expr_c &v_b, - const expr_c &v_c) { - return make_expr(intrin_type::permutex2var, - std::vector{v_a.remove_const(), - v_b.remove_const(), - v_c.remove_const()}, - any_map_t()); -} - -expr make_permutexvar(const expr_c &idx, const expr_c &v, const int lanes) { - return make_expr( - intrin_type::permutexvar, - std::vector{idx.remove_const(), v.remove_const()}, - any_map_t{{"lanes", lanes}}); -} - -expr make_insert(const expr_c &v_a, const expr_c &v_b, const int imm) { - return make_expr( - intrin_type::insert, - std::vector{v_a.remove_const(), v_b.remove_const()}, - any_map_t{{"insert_imm", imm}}); -} - -expr make_extract(const expr_c &v_a, const int imm, const int lanes) { - return make_expr( - intrin_type::extract, std::vector{v_a.remove_const()}, - any_map_t{{"extract_imm", imm}, {"lanes", lanes}}); -} - -static expr flatten_2d_index(const expr_c &v_a, const expr_c &row, - const expr_c &col) { - uint64_t parent_rows = v_a->dtype_.rows_; - if (parent_rows == 0) { - parent_rows = 1; - } - uint64_t parent_cols = v_a->dtype_.lanes_ / parent_rows; - auto p_cols_expr = make_constant({parent_cols}, datatypes::u16); - return (make_cast(datatypes::u16, row) * p_cols_expr + - make_cast(datatypes::u16, col)) * - make_constant( - {(uint64_t)utils::get_sizeof_etype(v_a->dtype_.type_code_)}, - datatypes::u16); -} - -expr make_insert(const expr_c &v_a, const expr_c &v_b, const expr_c &row, - const expr_c &col) { - return make_expr( - intrin_type::insert, - std::vector{v_a.remove_const(), v_b.remove_const(), - flatten_2d_index(v_a, row, col)}, - any_map_t{}); -} - -expr make_extract(const expr_c &v_a, const expr_c &row, const expr_c &col, - uint32_t rows, uint32_t cols) { - return make_expr( - intrin_type::extract, - std::vector{v_a.remove_const(), flatten_2d_index(v_a, row, col)}, - any_map_t{{"rows", rows}, {"cols", cols}}); -} - -expr make_read_struct(const expr_c &in, const std::string &struct_name, - const int &field_name) { - return make_expr( - intrin_type::read_struct, std::vector{in.remove_const()}, - any_map_t{{intrin_attr::struct_name, struct_name}, - {intrin_attr::struct_field, field_name}}); -} - -expr make_write_struct(const expr_c &in, const expr_c &field, - const std::string &struct_name, const int &field_name) { - return make_expr( - intrin_type::write_struct, - std::vector{in.remove_const(), field.remove_const()}, - any_map_t{{intrin_attr::struct_name, struct_name}, - {intrin_attr::struct_field, field_name}}); -} - -expr make_get_group_id(uint64_t par_for_level_id) { - return make_expr(intrin_type::get_group_id, - std::vector{par_for_level_id}, - any_map_t()); -} - -expr make_get_group_thread_id(int par_for_level_id) { - return make_expr(intrin_type::get_group_thread_id, - std::vector{par_for_level_id}, - any_map_t()); -} - -GEN_BINARY(add); -GEN_BINARY(sub); -GEN_BINARY(mul); -GEN_BINARY(div); -GEN_BINARY(mod); - -GEN_BINARY(cmp_eq); -GEN_BINARY(cmp_ne); -GEN_BINARY(cmp_lt); -GEN_BINARY(cmp_le); -GEN_BINARY(cmp_gt); -GEN_BINARY(cmp_ge); - -GEN_BINARY(logic_and); -GEN_BINARY(logic_or); - -expr make_logic_not(const expr_c &in) { - return make_expr(in.remove_const()); -} - -expr make_select(const expr_c &cond, const expr_c &l, const expr_c &r) { - return make_expr(cond.remove_const(), l.remove_const(), - r.remove_const()); -} - -expr make_clip(const expr_c &in, const expr_c &clip_min, - const expr_c &clip_max) { - expr ge_clip_min = builder::make_min(in, clip_max); - return builder::make_max(ge_clip_min, clip_min); -} - -expr make_indexing(const expr &ptr, const std::vector &idx, - uint16_t length, const expr &mask, uint16_t rows) { - return make_expr(ptr, idx, length, rows, mask); -} - -expr make_indexing(const expr_c &ptr, const std::vector &idx, - uint16_t length, const expr_c &mask, uint16_t rows) { - return make_expr(ptr.remove_const(), vector_remove_const(idx), - length, rows, mask.remove_const()); -} - -expr make_indexing(const expr &ptr, std::initializer_list idx, - uint16_t length, const expr &mask, uint16_t rows) { - return make_expr(ptr, std::vector(idx), length, rows, - mask); -} - -expr make_indexing(const expr_c &ptr, const expr_c &idx, uint16_t length, - const expr_c &mask, uint16_t rows) { - return make_expr(ptr.remove_const(), - std::vector{idx.remove_const()}, length, - rows, mask.remove_const()); -} - -expr make_call(const func_t &func, const std::vector &args) { - return make_expr(func, args); -} - -expr make_call(const func_c &func, const std::vector &args) { - return make_call(std::const_pointer_cast(func), - vector_remove_const(args)); -} - -expr remake_call(const func_t &func, const std::vector &args, - const call_c &old) { - return copy_attr( - *old, make_expr( - func, args, - std::vector{old->para_attr_})); -} - -expr remake_call(const func_c &func, const std::vector &args, - const call_c &old) { - return remake_call(std::const_pointer_cast(func), - vector_remove_const(args), old); -} - -expr make_tensor(const std::string &name, const std::vector &dims, - sc_data_type_t dtype, address_space addrspace, - const std::shared_ptr &init_value, - const std::vector &strides) { - return make_expr(dtype, name, dims, addrspace, init_value, - strides); -} - -expr make_tensor(const std::string &name, const std::vector &dims, - sc_data_type_t dtype, address_space addrspace, - const std::shared_ptr &init_value, - const std::vector &strides) { - return make_expr(dtype, name, vector_remove_const(dims), - addrspace, init_value, - vector_remove_const(strides)); -} - -expr make_tensor(const std::string &name, std::initializer_list dims, - sc_data_type_t dtype, address_space addrspace, - const std::shared_ptr &init_value, - std::initializer_list strides) { - return make_tensor(name, std::vector(dims), dtype, addrspace, - init_value, std::vector(strides)); -} - -expr make_stensor(const std::string &name, const std::vector &dims, - const std::vector &strides, sc_data_type_t dtype, - address_space addrspace, - const std::shared_ptr &init_value) { - COMPILE_ASSERT(strides.size() == dims.size(), - "Dims and strides shall have same length."); - return make_tensor(name, dims, dtype, addrspace, init_value, strides); -} - -expr make_stensor(const std::string &name, const std::vector &dims, - const std::vector &strides, sc_data_type_t dtype, - address_space addrspace, - const std::shared_ptr &init_value) { - COMPILE_ASSERT(strides.size() == dims.size(), - "Dims and strides shall have same length."); - return make_tensor(name, dims, dtype, addrspace, init_value, strides); -} - -builder_impl_t::basic_block_t::basic_block_t() { - body = builder::make_stmts_unattached({}).checked_as(); -} - -void builder_impl_t::basic_block_t::emit(const stmt &stmt) { - body->seq_.emplace_back(stmt); - // auto attach parent node - add_parent_node(stmt, body); -} - -stmt builder_impl_t::basic_block_t::get() { return std::move(body); } - -builder_impl_t::basic_block_t &builder_impl_t::get_current_scope() { - assert(!scopes.empty()); - return scopes.back(); -} - -void builder_impl_t::emit(const stmt &s) { - COMPILE_ASSERT(!scopes.empty(), - "Emitting to empty scope stack. You need to call " - "push_scope() first"); - get_current_scope().emit(s); -} - -stmt builder_impl_t::pop_scope() { - auto ret = scopes.back().get(); - add_parent_node(ret, stmt()); - scopes.pop_back(); - return ret; -} - -void builder_impl_t::push_scope() { scopes.emplace_back(); } - -stmts builder_impl_t::push_anchor() { - auto s = make_stmt(std::vector{}); - emit(s); - return s; -} - -stmt make_assign_unattached(const expr_c &var, const expr_c &value) { - return make_stmt(var.remove_const(), value.remove_const()); -} - -stmt make_stmts_unattached(const std::vector &seq) { - std::vector s; - s.reserve(seq.size()); - for (auto &v : seq) { - s.emplace_back(v.remove_const()); - } - return make_stmt(std::move(s)); -} - -stmt make_if_else_unattached(const expr_c &condition, const stmt_c &then_case, - const stmt_c &else_case) { - return make_stmt(condition.remove_const(), - then_case.remove_const(), - else_case.remove_const()); -} - -stmt make_evaluate_unattached(const expr_c &val) { - return make_stmt(val.remove_const()); -} - -stmt make_returns_unattached(const expr_c &val) { - return make_stmt(val.remove_const()); -} - -stmt make_var_tensor_def_unattached(const expr_c &var, linkage l, - const expr_c &init) { - return make_stmt(var.remove_const(), l, init.remove_const()); -} - -stmt make_for_loop_unattached(const expr_c &var, const expr_c &iter_begin, - const expr_c &iter_end, const expr_c &step, - const stmt_c &body, bool incremental, - for_type kind, int num_threads) { - return make_stmt( - var.remove_const(), iter_begin.remove_const(), iter_end.remove_const(), - step.remove_const(), body.remove_const(), incremental, kind, num_threads); -} - -stmt builder_impl_t::push_assign(const expr &var, const expr &value) { - auto ret = make_stmt(var, value); - emit(ret); - return ret; -} - -stmt builder_impl_t::push_if_else(const expr &condition_, - const stmt &then_case_, - const stmt &else_case_) { - auto ret = make_stmt(condition_, then_case_, else_case_); - add_parent_node(then_case_, ret); - if (else_case_.defined()) - add_parent_node(else_case_, ret); - emit(ret); - return ret; -} - -stmt builder_impl_t::push_evaluate(const expr &val) { - auto ret = make_stmt(val); - emit(ret); - return ret; -} - -stmt builder_impl_t::push_returns(const expr &val) { - auto ret = make_stmt(val); - emit(ret); - return ret; -} - -stmt builder_impl_t::push_var_tensor_def(const expr &var, linkage l, - const expr &init) { - auto ret = make_stmt(var, l, init); - emit(ret); - return ret; -} - -stmt builder_impl_t::push_for_loop(const expr &var_, const expr &iter_begin_, - const expr &iter_end_, const expr &step_, - const stmt &body_, bool incremental_, - for_type kind, int num_threads) { - auto ret = make_stmt(var_, iter_begin_, iter_end_, step_, - body_, incremental_, kind, num_threads); - add_parent_node(body_, ret); - emit(ret); - return ret; -} - -stmt builder_impl_t::brgemm( - const expr_c &x, const expr_c &w, const expr_c &y, const expr_c &blocks, - const expr_c &M, const expr_c &N, const expr_c &K, const expr_c &ldx, - const expr_c &ldw, const expr_c &ldy, const expr_c &x_block_stride, - const expr_c &w_block_stride, const std::vector &postops_data, - const expr_c &c_buf, const expr_c &bd_mask_idx, const expr_c &top_pad, - const expr_c &bottom_pad, const brgemm_args::extra_args_t &extras) { - auto args = std::vector{x.remove_const(), - w.remove_const(), - y.remove_const(), - blocks.remove_const(), - M.remove_const(), - N.remove_const(), - K.remove_const(), - ldx.remove_const(), - ldw.remove_const(), - ldy.remove_const(), - x_block_stride.remove_const(), - w_block_stride.remove_const()}; - args.insert(args.end(), postops_data.begin(), postops_data.end()); - args.emplace_back(c_buf.remove_const()); - args.emplace_back(bd_mask_idx.remove_const()); - args.emplace_back(top_pad.remove_const()); - args.emplace_back(bottom_pad.remove_const()); - return push_evaluate(make_expr( - intrin_type::brgemm, args, - any_map_t{{intrin_attr::brgemm_extras, extras}})); -} - -stmt builder_impl_t::list_brgemm( - const expr_c &x, const expr_c &w, const expr_c &y, const expr_c &blocks, - const expr_c &M, const expr_c &N, const expr_c &K, const expr_c &ldx, - const expr_c &ldw, const expr_c &ldy, const expr_c &x_block_stride, - const expr_c &w_block_stride, const expr_c &len, - const std::vector &postops_data, const expr_c &c_buf, - const expr_c &bd_mask_idx, const expr_c &top_pad, const expr_c &bottom_pad, - const brgemm_args::extra_args_t &extras) { - auto args = std::vector{x.remove_const(), - w.remove_const(), - y.remove_const(), - blocks.remove_const(), - M.remove_const(), - N.remove_const(), - K.remove_const(), - ldx.remove_const(), - ldw.remove_const(), - ldy.remove_const(), - x_block_stride.remove_const(), - w_block_stride.remove_const(), - len.remove_const()}; - args.insert(args.end(), postops_data.begin(), postops_data.end()); - args.emplace_back(c_buf.remove_const()); - args.emplace_back(bd_mask_idx.remove_const()); - args.emplace_back(top_pad.remove_const()); - args.emplace_back(bottom_pad.remove_const()); - return push_evaluate(make_expr( - intrin_type::list_brgemm, args, - any_map_t{{intrin_attr::brgemm_extras, extras}})); -} - -} // namespace builder -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/builder.hpp b/legacy/core/src/compiler/ir/builder.hpp deleted file mode 100644 index 6dace31d3..000000000 --- a/legacy/core/src/compiler/ir/builder.hpp +++ /dev/null @@ -1,1182 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_BUILDER_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_BUILDER_HPP - -#include -#include - -#include "intrinsics.hpp" -#include "sc_data_type.hpp" -#include "sc_expr.hpp" -#include "sc_function.hpp" -#include "sc_stmt.hpp" -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -/** - * The IR builder. It contains utility functions that helps - * to build the IR. For easier IR building, you can use easy_build, - * which is based on these function. Each thread has a - * thread-local pointer which points to the current builder. This - * feature is useful for easy_build to save the user from providing - * the current builder. - * */ -namespace builder { -/** - * The internal implementation of IR builder. Use ir_builder_t - * in most cases instead - * */ -class SC_INTERNAL_API builder_impl_t { -public: - /** - * A basic block is a list of statements that is temporarily - * stored in the builder. A basic block can be appended with - * a new statement. The builder will finally compose the - * statements in a basic-block into a stmts node. - * */ - struct basic_block_t { - stmts body; - /** - * Appends a stmt into the BB - * @param stmt the statement - * */ - void emit(const stmt &stmt); - /** - * Get the contained body and clear this field - * @return the `body` stmts - * */ - stmt get(); - // return `seq_` field of `body`, which is std::vector type - std::vector &as_seq() { return body->seq_; } - basic_block_t(); - basic_block_t(basic_block_t &&other) : body(std::move(other.body)) {} - }; - - /** - * A builder may have nested basic blockes. It uses a stack to track - * the current basic block. - * */ - std::vector scopes; - - /** - * Gets the current basic block on the top of the scopes - * */ - basic_block_t &get_current_scope(); - builder_impl_t() = default; - - /** - * Pushs a statement into the current basic block - * */ - void emit(const stmt &s); - - /** - * Pops out the current basic block. Generates a stmts node - * from it - * @return the stmts node for the poped basic block - * */ - stmt pop_scope(); - - /** - * Pushes a empty current basic block on the top of the scopes. - * emit() after calling this function will push the statements - * into this new basic block - * */ - void push_scope(); - - /** - * Makes and pushes an assign statement - * @param var the dest LValue - * @param value the src RValue - * @return the pushed node - * */ - stmt push_assign(const expr &var, const expr &value); - - /** - * Makes and pushes an if_else - * @param condition_ the condition, should be boolean typed - * @param then_case_ the `then` block - * @param else_case_ the `else` block, nullable. - * @return the pushed node - * */ - stmt push_if_else(const expr &condition_, const stmt &then_case_, - const stmt &else_case_); - - /** - * Makes and pushes an evaluate statement - * @param val the expression - * @return the pushed node - * */ - stmt push_evaluate(const expr &val); - - /** - * Makes and pushes an return statement - * @param val the expression, nullable if the current function returns - * void_t - * @return the pushed node - * */ - stmt push_returns(const expr &val = expr()); - - /** - * Makes and push a variable def statement - * @param var defined var or tensor - * @param linkage - * @param init init value, nullable - * @return the pushed node - * */ - stmt push_var_tensor_def(const expr &var, linkage linkage = linkage::local, - const expr &init = expr()); - - /** - * Makes and pushes a for_loop - * @param var the iterate variable. The loop-var is expected to - * be used only within the scope of the loop. Should be an integer var - * @param iter_begin the initial value of var_ - * @param iter_end the max bound of the loop-var var_. Can never be reached - * @param step the step of var_ in each iteration. - * @param body the body of the loop - * @param incremental if the loop-var var_ is incremental. Not currently - * used. - * @param kind the kind of the loop. @see for_type - * @param num_threads the number of threads in parallel-for - * @return the pushed node - * */ - stmt push_for_loop(const expr &var, const expr &iter_begin, - const expr &iter_end, const expr &step, const stmt &body, - bool incremental, for_type kind, int num_threads = 0); - - /** - * Makes and pushes blank stmts as an anchor - * */ - stmts push_anchor(); - - /** - * Makes a brgemm-call node and evaluates it - * */ - stmt brgemm(const expr_c &a, const expr_c &b, const expr_c &c, - const expr_c &blocks, const expr_c &M, const expr_c &N, - const expr_c &K, const expr_c &lda, const expr_c &ldb, - const expr_c &ldc, const expr_c &a_stride, const expr_c &b_stride, - const std::vector &postops_data, const expr_c &c_buf, - const expr_c &bd_mask_idx, const expr_c &top_pad, - const expr_c &bottom_pad, - const brgemm_args::extra_args_t &extras); - - /** - * Makes a brgemm-call node and evaluates it - * */ - stmt list_brgemm(const expr_c &a, const expr_c &b, const expr_c &c, - const expr_c &blocks, const expr_c &M, const expr_c &N, - const expr_c &K, const expr_c &lda, const expr_c &ldb, - const expr_c &ldc, const expr_c &a_stride, - const expr_c &b_stride, const expr_c &len, - const std::vector &postops_data, const expr_c &c_buf, - const expr_c &bd_mask_idx, const expr_c &top_pad, - const expr_c &bottom_pad, - const brgemm_args::extra_args_t &extras); - - /** - * Makes a string simulated by tensor node in the current location - * For debug use only (e.g. in fmtprint). It may have performance overhead - * @param name the function name - * @return the created node, should be a tensor - * */ - expr make_str(const std::string &str); -}; - -/** - * Gets the current thread-local builder. If you have never set - * a builder by `set_current()`, this function will return nullptr - * */ -SC_INTERNAL_API builder_impl_t *get_current_builder(); - -/** - * Sets the current thread-local builder. - * @param b the pointer to the builder - * */ -SC_INTERNAL_API void set_current_builder(builder_impl_t *b); - -/** - * Makes a binary/logic/cmp node of the same type of original, with new LHS=l - * and RHS=r. Copies attrs and other info from original - * @return the created node - * */ -expr remake_binary(const expr_c &l, const expr_c &r, const expr_c &original); - -/** - * Makes a constant node of f32 - * @param val the value - * @return the created node - * */ -expr make_constant(float val); - -/** - * Makes a constant node of s32 - * @param val the value - * @return the created node - * */ -expr make_constant(int32_t val); - -/** - * Makes a constant node of index type - * @param val the value - * @return the created node - * */ -expr make_constant(uint64_t val); - -/** - * Makes a constant node of std::vector - * @param val the value - * @return the created node - * */ -expr make_constant(const std::vector &val, sc_data_type_t dtype); - -/** - * Makes a var node - * @param type the type of the variable - * @param name the name of the variable - * @return the created node - * */ -expr make_var(sc_data_type_t type, const std::string &name); - -/** - * Makes a cast node - * @param type the destination type of the casting - * @param in the expression to convert - * @return the created node - * */ -SC_INTERNAL_API expr make_cast(sc_data_type_t type, const expr_c &in); - -/** - * Makes a add (+) node - * @param left left hand side - * @param right right hand side - * @return the created node - * */ -expr make_add(const expr_c &left, const expr_c &right); - -/** - * Makes a sub (-) node - * @param left left hand side - * @param right right hand side - * @return the created node - * */ -expr make_sub(const expr_c &left, const expr_c &right); - -/** - * Makes a mul (*) node - * @param left left hand side - * @param right right hand side - * @return the created node - * */ -SC_INTERNAL_API expr make_mul(const expr_c &left, const expr_c &right); - -/** - * Makes a div (div) node - * @param left left hand side - * @param right right hand side - * @return the created node - * */ -expr make_div(const expr_c &left, const expr_c &right); - -/** - * Makes a mod (%) node - * @param left left hand side - * @param right right hand side - * @return the created node - * */ -expr make_mod(const expr_c &left, const expr_c &right); - -/** - * Makes a min node - * @param left left hand side - * @param right right hand side - * @return the created node - * */ -expr make_min(const expr_c &left, const expr_c &right); - -/** - * Makes a reinterpret_cast node - * @param v the value - * @param dtype the target type, must have the same size of v - * @return the created node - * */ -expr make_reinterpret(const expr_c &v, sc_data_type_t dtype); - -/** - * Makes a isnan(float) node - * @param v the value - * @return the created node - * */ -expr make_isnan(const expr_c &v); - -/** - * Makes a saturated cast node - * @param v the value - * @param dtype the target dtype, must have the same size of v - * @return the created node - * */ -expr make_saturated_cast(const expr_c &v, sc_data_type_t dtype); - -/** - * Makes a round-and-cast node. The input should be a FP32 vector or scalar - * type. The output should be S32 vector or scalar type. - * @param v the value - * @param dtype the target dtype, must have the same size of v - * @return the created node - * */ -expr make_round_and_cast(const expr_c &v, sc_data_type_t dtype); - -/** - * Makes a max node - * @param left left hand side - * @param right right hand side - * @return the created node - * */ -expr make_max(const expr_c &left, const expr_c &right); - -/** - * Makes a abs node (absolute value) - * @param v the input value - * @return the created node - * */ -expr make_abs(const expr_c &v); - -/** - * Makes a round node - * @param v the input value - * @return the created node - * */ -expr make_round(const expr_c &v); - -/** - * Makes a floor node - * @param v the input value - * @return the created node - * */ -expr make_floor(const expr_c &v); - -/** - * Makes a ceil node - * @param v the input value - * @return the created node - * */ -expr make_ceil(const expr_c &v); - -/** - * Makes an exp node - * @param v the input value - * @return the created node - * */ -expr make_exp(const expr_c &v); - -/** - * Makes a log node - * @param v the input value - * @return the created node - * */ -expr make_log(const expr_c &v); - -/** - * Makes a erf node - * @param v the input value - * @return the created node - * */ -expr make_erf(const expr_c &v); - -/** - * Makes an sqrt node - * @param v the input value - * @return the created node - * */ -expr make_sqrt(const expr_c &v); - -/** - * Makes an rsqrt node - * @param v the input value - * @return the created node - * */ -expr make_rsqrt(const expr_c &v); - -/** - * Makes an reduce add node - * @param v the input value - * @return the created node - * */ -expr make_reduce_add(const expr_c &v); - -/** - * Makes an reduce max node - * @param v the input value - * @return the created node - * */ -expr make_reduce_max(const expr_c &v); - -/** - * Makes an reduce min node - * @param v the input value - * @return the created node - * */ -expr make_reduce_min(const expr_c &v); - -/** - * Makes an reduce mul node - * @param v the input value - * @return the created node - * */ -expr make_reduce_mul(const expr_c &v); - -/** - * Makes a broadcast node - * @param v the input value - * @param lanes the lanes of output value - * @return the created node - * */ -expr make_broadcast(const expr_c &v, int lanes); - -/** - * Makes an fmadd node - * @param v_a the first input value - * @param v_b the second input value - * @param v_c the third input value - * @return the created node - * */ -expr make_fmadd(const expr_c &v_a, const expr_c &v_b, const expr_c &v_c); - -/** - * Makes an fnmadd node - * @param v_a the first input value - * @param v_b the second input value - * @param v_c the third input value - * @return the created node - * */ -expr make_fnmadd(const expr_c &v_a, const expr_c &v_b, const expr_c &v_c); - -/** - * Makes an unpack_low node - * Unpack and interleave elements from the low half of each 128-bit lane - * in v_a and v_b - * - * _mm256_unpacklo_ps - * DEFINE INTERLEAVE_DWORDS(src1[127:0], src2[127:0]) { - * dst[31:0] := src1[31:0] - * dst[63:32] := src2[31:0] - * dst[95:64] := src1[63:32] - * dst[127:96] := src2[63:32] - * RETURN dst[127:0] - * } - * dst[127:0] := INTERLEAVE_DWORDS(a[127:0], b[127:0]) - * dst[255:128] := INTERLEAVE_DWORDS(a[255:128], b[255:128]) - * dst[MAX:256] := 0 - * - * @param v_a the first input value - * @param v_b the second input value - * @param elem_bits the bits of unpack element, e.g. f32=>32, s64=>64 - * @return the created node - * */ -expr make_unpack_low(const expr_c &v_a, const expr_c &v_b, int elem_bits = 32); - -/** - * Makes an unpack_high node - * Unpack and interleave elements from the high half of each 128-bit lane - * in v_a and v_b - * - * _mm256_unpackhi_ps - * DEFINE INTERLEAVE_HIGH_DWORDS(src1[127:0], src2[127:0]) { - * dst[31:0] := src1[95:64] - * dst[63:32] := src2[95:64] - * dst[95:64] := src1[127:96] - * dst[127:96] := src2[127:96] - * RETURN dst[127:0] - * } - * dst[127:0] := INTERLEAVE_HIGH_DWORDS(a[127:0], b[127:0]) - * dst[255:128] := INTERLEAVE_HIGH_DWORDS(a[255:128], b[255:128]) - * dst[MAX:256] := 0 - * - * @param v_a the first input value - * @param v_b the second input value - * @param elem_bits the lanes of unpack element, e.g. f32=>32, s64=>64 - * @return the created node - * */ -expr make_unpack_high(const expr_c &v_a, const expr_c &v_b, int elem_bits = 32); - -/** - * Makes a shuffle node - * Shuffle elements in v_a and v_b within 128-bit lanes using the control in v_c - * - * _mm256_shuffle_ps - * DEFINE SELECT4(src, control) { - * CASE(control[1:0]) OF - * 0: tmp[31:0] := src[31:0] - * 1: tmp[31:0] := src[63:32] - * 2: tmp[31:0] := src[95:64] - * 3: tmp[31:0] := src[127:96] - * ESAC - * RETURN tmp[31:0] - * } - * dst[31:0] := SELECT4(a[127:0], imm8[1:0]) - * dst[63:32] := SELECT4(a[127:0], imm8[3:2]) - * dst[95:64] := SELECT4(b[127:0], imm8[5:4]) - * dst[127:96] := SELECT4(b[127:0], imm8[7:6]) - * dst[159:128] := SELECT4(a[255:128], imm8[1:0]) - * dst[191:160] := SELECT4(a[255:128], imm8[3:2]) - * dst[223:192] := SELECT4(b[255:128], imm8[5:4]) - * dst[255:224] := SELECT4(b[255:128], imm8[7:6]) - * dst[MAX:256] := 0 - * - * @param v_a the first input value - * @param v_b the second input value - * @param v_c the third input value - * @param type_bits the number of bits of the data type you want to shuffle - * @return the created node - * */ -expr make_shuffle(const expr_c &v_a, const expr_c &v_b, const int &v_c, - const int &type_bits); - -/** - * Makes an permute node - * Shuffle each 128-bits selected by v_c from v_a and v_b. - * - * _mm256_permute2f128_ps - * DEFINE SELECT4(src1, src2, control) { - * CASE(control[1:0]) OF - * 0: tmp[127:0] := src1[127:0] - * 1: tmp[127:0] := src1[255:128] - * 2: tmp[127:0] := src2[127:0] - * 3: tmp[127:0] := src2[255:128] - * ESAC - * IF control[3] - * tmp[127:0] := 0 - * FI - * RETURN tmp[127:0] - * } - * dst[127:0] := SELECT4(a[255:0], b[255:0], imm8[3:0]) - * dst[255:128] := SELECT4(a[255:0], b[255:0], imm8[7:4]) - * dst[MAX:256] := 0 - - * @param v_a the first input value - * @param v_b the second input value - * @param v_c the third input value - * @param type_bits the number of bits of the data type you want to permute - * @return the created node - * */ -expr make_permute(const expr_c &v_a, const expr_c &v_b, const int &v_c, - const int &type_bits = 128); - -/** - * Makes an permutexvar node - * Using the corresponding bit in idx to Shuffle v. - * eg: - - * _mm512_permutexvar_epi8 - * FOR j := 0 to 63 - * i := j*8 - * id := idx[i+5:i]*8 - * dst[i+7:i] := a[id+7:id] - * ENDFOR - - * @param idx the correspoding index - * @param v the input value - * @param lanes specify the lanes for permutex data. For example: if datatype - is u8 and specify lanes is 8, which means you want to permutex 64bit data in - v. - * @return the created node - * */ -expr make_permutexvar(const expr_c &idx, const expr_c &v, const int lanes = 1); - -/** - * Insert the value into dst at the location specified by imm. Note that if the - * data is more than 128bit, the first parameter needs to be twice the number of - * bits of the second parameter. - * - * ep: _mm512_inserti32x8 - * Operation - * dst[511:0] := a[511:0] - * CASE imm8[0] OF - * 0: dst[255:0] := b[255:0] - * 1: dst[511:256] := b[255:0] - * ESAC - * dst[MAX:512] := 0 - - * @param v_a the first input value - * @param v_b the second input value - * @param imm the location specified value, 0 or 1 - * @return the created node - * */ -expr make_insert(const expr_c &v_a, const expr_c &v_b, const int imm); - -/** - * Extract the value from input specified by imm. - * - * dst[7:0] := (a[127:0] >> (imm[3:0] * 8))[7:0] - * dst[31:8] := 0 - - * @param v_a the input value - * @param imm the location specified value, 0 or 1 - * @param lanes specify the lanes for extracting data. For example: if datatype - is u8 and specify lanes is 8, which means you want to extract 64bit data from - v_a. - * @return the created node - * */ -expr make_extract(const expr_c &v_a, const int imm, const int lanes = 1); - -/** - * Update the 2d value into v_a at the location specified by row and col. - * @param v_a the larger 2D tile - * @param v_b the smaller 2D tile to be inserted to v_a - * @param row the row location for v_b to be inserted to v_a - * @param col the col location for v_b to be inserted to v_a - * @return the created node as the updated value - * */ -expr make_insert(const expr_c &v_a, const expr_c &v_b, const expr_c &row, - const expr_c &col); -/** - * Extract the 2d region from v_a at the location specified by row and col. - * @param v_a the 2D tile value - * @param row the row location for v_b to be inserted to v_a - * @param col the col location for v_b to be inserted to v_a - * @param rows_count the number of rows to be extracted - * @param cols_count the number of cols to be extracted - * @return the created node as the extracted SIMD value of shape [rows x cols] - * */ -expr make_extract(const expr_c &v_a, const expr_c &row, const expr_c &col, - uint32_t rows_count, uint32_t cols_count); - -/** - * Makes an gather node - * Gather elements from memory. - * dst = __mm512{*(addr + indices[0]), *(addr + indices[1]), ...} - * @param addr the base addr value, usually has pointer dtype. - * @param indices the index list, usually use a simd var. - * @return the created node - * */ -expr make_gather(const expr_c &addr, const expr_c &indices); - -/** - * Makes a permutex2var node - * @param v_a the first input value - * @param v_b the second input value - * @param v_c the third input value - * @return the created node - * */ -expr make_permutex2var(const expr_c &v_a, const expr_c &v_b, const expr_c &v_c); - -/** - * Makes a cmp_eq (==) node - * @param left left hand side - * @param right right hand side - * @return the created node - * */ -expr make_cmp_eq(const expr_c &left, const expr_c &right); - -/** - * Makes a cmp_ne (!=) node - * @param left left hand side - * @param right right hand side - * @return the created node - * */ -expr make_cmp_ne(const expr_c &left, const expr_c &right); - -/** - * Makes a cmp_lt (<) node - * @param left left hand side - * @param right right hand side - * @return the created node - * */ -expr make_cmp_lt(const expr_c &left, const expr_c &right); - -/** - * Makes a cmp_le (<=) node - * @param left left hand side - * @param right right hand side - * @return the created node - * */ -expr make_cmp_le(const expr_c &left, const expr_c &right); - -/** - * Makes a cmp_gt (>) node - * @param left left hand side - * @param right right hand side - * @return the created node - * */ -expr make_cmp_gt(const expr_c &left, const expr_c &right); - -/** - * Makes a cmp_ge (>=) node - * @param left left hand side - * @param right right hand side - * @return the created node - * */ -expr make_cmp_ge(const expr_c &left, const expr_c &right); - -/** - * Makes a logic_and (&&) node - * @param left left hand side, should be of boolean type - * @param right right hand side, should be of boolean type - * @return the created node - * */ -expr make_logic_and(const expr_c &left, const expr_c &right); - -/** - * Makes a logic_or (||) node - * @param left left hand side, should be of boolean type - * @param right right hand side, should be of boolean type - * @return the created node - * */ -expr make_logic_or(const expr_c &left, const expr_c &right); - -/** - * Makes a shift left (<<) node - * @param left left hand side, the number to be shift - * @param bits the number of bits to shift - * @return the created node - * */ -expr make_shl(const expr_c &left, const expr_c &bits); - -/** - * Makes a shift right (>>) node - * @param left left hand side, the number to be shift - * @param bits the number of bits to shift - * @return the created node - * */ -expr make_shr(const expr_c &left, const expr_c &bits); - -/** - * Makes a int_and (&) node - * @param left left hand side, should be of boolean type - * @param right right hand side, should be of boolean type - * @return the created node - * */ -expr make_int_and(const expr_c &left, const expr_c &right); - -/** - * Makes a int_or (|) node - * @param left left hand side, should be of boolean type - * @param right right hand side, should be of boolean type - * @return the created node - * */ -expr make_int_or(const expr_c &left, const expr_c &right); - -/** - * Makes a int_xor (^) node - * @param left left hand side, should be of boolean type - * @param right right hand side, should be of boolean type - * @return the created node - * */ -expr make_int_xor(const expr_c &left, const expr_c &right); - -/** - * Makes a node to read field from a struct. - * @param in dynamic tensor, should be a tensor with u8 pointer type - * @param struct_name name string of struct - * @param field_name name enum of field - * @return the created node - * */ -expr make_read_struct(const expr_c &in, const std::string &struct_name, - const int &field_name); - -/** - * Makes a node to write a field to struct. - * @param dyn_tsr dynamic tensor, should be a tensor with u8 pointer type - * @param in data tensor, should be a tensor with pointer type - * @param struct_name name string of struct - * @param field_name name enum of field - * @return the created node - * */ -expr make_write_struct(const expr_c &dyn_tsr, const expr_c &in, - const std::string &struct_name, const int &field_name); - -/** - * Makes a node to get thread group id - * @param par_for_level_id the level of parallel for - * @return the created node - * */ -expr make_get_group_id(uint64_t par_for_level_id); - -/** - * Makes a node to get thread id in group. Specifying the (-1) group will return - * the global thread id - * @param par_for_level_id the level of parallel for - * @return the created node - * */ -expr make_get_group_thread_id(int par_for_level_id); - -/** - * Makes a logic_not (!) node - * @param in the input expression, should be of boolean type - * @return the created node - * */ -expr make_logic_not(const expr_c &in); - -/** - * Makes a select (? :) node - * @param cond the condition expression, should be of boolean type - * @param l obtained value when condition is true - * @param r obtained value when condition is false - * @return the created node - * */ -expr make_select(const expr_c &cond, const expr_c &l, const expr_c &r); - -/** - * Makes a clip(in, clip_min, clip_max) node - * clip(in, clip_min, clip_max) = in when clip_min <= in <= clip_max - * clip(in, clip_min, clip_max) = clip_min when in < clip_min - * clip(in, clip_min, clip_max) = clip_max when in > clip_max - * @param in the input value to be clipped - * @param xmin min value for clip - * @param xmax max value for clip - * @return the created node - * */ -expr make_clip(const expr_c &in, const expr_c &clip_min, - const expr_c &clip_max); - -/** - * Makes a indexing node of multiple dimemsions - * @param ptr the buffer expression, should be a tensor - * @param idx the indices, should be integers - * @param mask the loading mask, currently unused, nullable - * @param length the vector length to index, or 1 as a scalar - * @return the created node - * */ -expr make_indexing(const expr &ptr, const std::vector &idx, - uint16_t length = 1, const expr &mask = expr(), - uint16_t rows = 0); -expr make_indexing(const expr &ptr, std::initializer_list idx, - uint16_t length = 1, const expr &mask = expr(), - uint16_t rows = 0); - -expr make_indexing(const expr_c &ptr, const std::vector &idx, - uint16_t length = 1, const expr_c &mask = expr_c(), - uint16_t rows = 0); - -/** - * Makes a indexing node of single dimemsion - * @param ptr the buffer expression, should be a tensor - * @param idx the index, should be an integer - * @param mask the loading mask, currently unused, nullable - * @param length the vector length to index, or 1 as a scalar - * @return the created node - * */ -expr make_indexing(const expr_c &ptr, const expr_c &idx, uint16_t length = 1, - const expr_c &mask = expr_c(), uint16_t rows = 0); - -/** - * Makes a call node - * @param func the callee - * @param args the arguments - * @return the created node - * */ -expr make_call(const func_t &func, const std::vector &args); -expr make_call(const func_c &func, const std::vector &args); - -/** - * Makes a call node with given callee and arguments. Other members are copied - * from `old` - * @param func_t the callee - * @param args the arguments - * @param old the old call_node - * @return the created node - * */ -expr remake_call(const func_t &func, const std::vector &args, - const call_c &old); -expr remake_call(const func_c &func, const std::vector &args, - const call_c &old); - -/** - * Makes a tensor node - * @param name the name of the tensor - * @param dims the dimensions, should be integers - * @param dtype the elemente type of the tensor - * @param addr_space the address space: CPU/Device - * @param strides stride info for each dim (optional) - * @return the created node - * */ -SC_INTERNAL_API expr make_tensor( - const std::string &name, const std::vector &dims, - sc_data_type_t dtype, address_space addrspace = address_space::automatic, - const std::shared_ptr &init_value = nullptr, - const std::vector &strides = {}); -SC_INTERNAL_API expr make_tensor( - const std::string &name, const std::vector &dims, - sc_data_type_t dtype, address_space addrspace = address_space::automatic, - const std::shared_ptr &init_value = nullptr, - const std::vector &strides = {}); -SC_INTERNAL_API expr make_tensor( - const std::string &name, std::initializer_list dims, - sc_data_type_t dtype, address_space addrspace = address_space::automatic, - const std::shared_ptr &init_value = nullptr, - std::initializer_list strides = std::initializer_list()); - -/** - * Makes a tensor node with user-defined stride - * @param name the name of the tensor - * @param dims the dimensions, should be integers - * @param strides stride info for each dim - * @param dtype the elemente type of the tensor - * @param addr_space the address space: CPU/Device - * @return the created node - * */ -expr make_stensor(const std::string &name, const std::vector &dims, - const std::vector &strides, sc_data_type_t dtype, - address_space addrspace = address_space::automatic, - const std::shared_ptr &init_value = nullptr); -expr make_stensor(const std::string &name, const std::vector &dims, - const std::vector &strides, sc_data_type_t dtype, - address_space addrspace = address_space::automatic, - const std::shared_ptr &init_value = nullptr); - -/** - * Makes a function node - * @param name the function name - * @param params the parameters, should be tensors or vars - * @param body the body of the function - * @param ret_type the return type of the function - * @return the created node - * */ -SC_INTERNAL_API func_t make_func(const std::string &name, - const std::vector ¶ms, stmt body, - sc_data_type_t ret_type); -/** - * @see make_func overloaded function - * */ -SC_INTERNAL_API func_t make_func(const std::string &name, - const std::vector ¶ms, - const stmt_c &body, sc_data_type_t ret_type); - -/** - * Makes an assign statement, the statement is not attached to any builder - * @param var the dest LValue - * @param value the src RValue - * @return the pushed node - * */ -stmt make_assign_unattached(const expr_c &var, const expr_c &value); - -/** - * Makes an stmts statement, the statement is not attached to any builder - * @param seq the sequence of the stmt - * @return the pushed node - * */ -stmt make_stmts_unattached(const std::vector &seq); - -/** - * Makes an if_else, the statement is not attached to any builder - * @param condition_ the condition, should be boolean typed - * @param then_case_ the `then` block - * @param else_case_ the `else` block, nullable. - * @return the pushed node - * */ -stmt make_if_else_unattached(const expr_c &condition, const stmt_c &then_case, - const stmt_c &else_case); - -/** - * Makes an evaluate statement, the statement is not attached to any - * builder - * @param val the expression - * @return the pushed node - * */ -stmt make_evaluate_unattached(const expr_c &val); - -/** - * Makes a return statement, the statement is not attached to any builder - * @param val the expression, nullable if the current function returns - * void_t - * @return the pushed node - * */ -stmt make_returns_unattached(const expr_c &val = expr_c()); - -/** - * Makes a variable def statement, the statement is not attached to any builder - * @param var defined var or tensor - * @param linkage - * @param init init value, nullable - * @return the pushed node - * */ -SC_INTERNAL_API stmt make_var_tensor_def_unattached( - const expr_c &var, linkage linkage = linkage::local, - const expr_c &init = expr_c()); - -/** - * Makes a for_loop, the statement is not attached to any builder - * @param var the iterate variable. The loop-var is expected to - * be used only within the scope of the loop. Should be an integer var - * @param iter_begin the initial value of var_ - * @param iter_end the max bound of the loop-var var_. Can never be reached - * @param step the step of var_ in each iteration. - * @param body the body of the loop - * @param incremental if the loop-var var_ is incremental. Not currently - * used. - * @param kind the kind of the loop. @see for_type - * @param num_threads the number of threads to use in parallel-for. 0 for using - * all avaliable threads in current thread group. If the loop is not parallel, - * it must be 0. - * @return the pushed node - * */ -stmt make_for_loop_unattached(const expr_c &var, const expr_c &iter_begin, - const expr_c &iter_end, const expr_c &step, - const stmt_c &body, bool incremental, - for_type kind, int num_threads = 0); - -// makes a new intrin_call with type_ and intrin_attrs_ copied -intrin_call remake_intrin_call(const intrin_call_c &v, - const std::vector &newargs); - -// makes a func_ptr -expr make_func_addr(func_t v); - -/** - * Makes a phi node - * */ -expr make_phi(const std::vector &values, bool is_loop_phi = false); - -/** - * Makes a x86 gerenal intrinsic node - * */ -expr make_x86_intrin(x86_intrin_type::x86_intrin_type_t type, - const std::vector &args, - const any_map_t &attrs = any_map_t()); - -// makes a new low_level_intrin with newargs and type_ copied -low_level_intrin remake_low_level_intrin(const low_level_intrin_c &v, - const std::vector &newargs); - -/** - * Gets the pointer of an element of the tensor as a view - * @see tensorptr_node - * @param tensor the tensor - * @param idx the indices of the element within the tensor - * @param shape the shape of the resulting view. It can be empty, when - * there is no indexing on the resulting view. If there is an indexing - * on this tensor_ptr, the shape cannot be empty. - * @param is_slice How `index_flatten` pass flatten the indexing on this - * tensor view. For example, we have a tensorptr - * `ptr=&base[offset0,offset1,offset2]`.If `is_slice` is true, indexing - * on this tensor view `ptr[i,j,k]` will be first mapped to indexing on - * the base tensor: `base[i+offset0, j+offset1, k+offset2]`, where - * offsetN is the offset of this tensorptr to the base tensor. Then it - * will flatten the mapped indexing. If `is_slice` is false, - * `index_flatten` pass will treat the view as an independent tensor, - * which shares a part of the memory from the base tensor. So the - * indexing on view `ptr[i,j,k]` will be first flattened to `ptr[i * A + - * j * B + k]`, where `A` and `B` depends on the `shape` field of this - * IR node. Then it will add the flattened index `i * A + j * B + k` to - * the flattened base offset of this tensor ptr `offset0 * C + offset1 * - * D + offset2`, to finally compute the index on the base tensor. - * @return the address of the element in the tensor - * */ -SC_INTERNAL_API expr tensor_ptr(const expr &tensor, - const std::vector &idx, - const std::vector &shape = {}, - bool is_slice = false); -SC_INTERNAL_API expr tensor_ptr(const expr_c &tensor, - const std::vector &idx, - const std::vector &shape = {}, - bool is_slice = false); -SC_INTERNAL_API expr tensor_ptr(const expr &tensor, - std::initializer_list idx, - std::initializer_list shape = {}, - bool is_slice = false); -/** - * Sets the attr of the input IR node - * @tparam TNode should be expr(_c) or stmt(_c) - * @tparam T the value of the attr - * - * @param node the IR node - * @param key the key name of the attr - * @param value the value of the attr - * @return node - * */ -template -TNode with_attr(TNode node, const std::string &key, const T &value) { - node->attr()[key] = value; - return node; -} - -/** - * A builder with scope guard to help set the builder of the current - * thread. It will set the current builder to `this` and remember the old - * builder. After this builder is destructed, it will set the current builder - * back to the old one. Using this builder, we can ensure that - * builder::get_current_builder is a valid pointer or null. - * @see builder_impl_t - * @note Common usage: - * { - * guarded_builder builder; - * // build the IR with builder - * } - * */ -class ir_builder_t : public builder_impl_t { - builder_impl_t *old_; - -public: - ir_builder_t(const ir_builder_t &) = delete; - ir_builder_t() { - old_ = get_current_builder(); - set_current_builder(this); - } - ~ir_builder_t() { set_current_builder(old_); } -}; -} // namespace builder - -#define _BUILDER_MAKE_BIN_OP(OP, NAME) \ - inline expr operator OP(const expr_c &l, const expr_c &r) { \ - return builder::make_##NAME(l, r); \ - } - -_BUILDER_MAKE_BIN_OP(+, add) -_BUILDER_MAKE_BIN_OP(-, sub) -_BUILDER_MAKE_BIN_OP(*, mul) -_BUILDER_MAKE_BIN_OP(/, div) -_BUILDER_MAKE_BIN_OP(%, mod) -_BUILDER_MAKE_BIN_OP(==, cmp_eq) -_BUILDER_MAKE_BIN_OP(!=, cmp_ne) -_BUILDER_MAKE_BIN_OP(<, cmp_lt) -_BUILDER_MAKE_BIN_OP(<=, cmp_le) -_BUILDER_MAKE_BIN_OP(>, cmp_gt) -_BUILDER_MAKE_BIN_OP(>=, cmp_ge) -_BUILDER_MAKE_BIN_OP(&&, logic_and) -_BUILDER_MAKE_BIN_OP(||, logic_or) -_BUILDER_MAKE_BIN_OP(&, int_and) -_BUILDER_MAKE_BIN_OP(|, int_or) -_BUILDER_MAKE_BIN_OP(^, int_xor) -_BUILDER_MAKE_BIN_OP(<<, shl) -_BUILDER_MAKE_BIN_OP(>>, shr) - -inline expr operator!(const expr_c &l) { return builder::make_logic_not(l); } -#undef _BUILDER_MAKE_BIN_OP - -expr copy_attr(const expr_base &ths, expr &&newexpr); -stmt copy_attr(const stmt_base_t &ths, stmt &&newstmt); -func_t copy_attr(const func_base &ths, func_t &&newfunc); - -stmt get_parent_node(const stmt &node); -// If buffer is tptr, return base tensor, If buffer is tensor, return itself -tensor get_real_tensor(const expr &buffer); -// set base tensor of `tptr` with `tsr` -void set_base_tensor(expr &tptr, const expr &tsr); -void add_parent_node(const stmt &s, const stmt &ret); -stmt get_common_parent_node(const stmt &node1, const stmt &node2); - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl -#endif diff --git a/legacy/core/src/compiler/ir/builtin.cpp b/legacy/core/src/compiler/ir/builtin.cpp deleted file mode 100644 index b0fec7930..000000000 --- a/legacy/core/src/compiler/ir/builtin.cpp +++ /dev/null @@ -1,1003 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "builtin.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -SC_MODULE(microkernel.builtin) - -using namespace dnnl::impl::graph::gc::builder; -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -expr get_ir_null() { - return make_expr(UINT64_C(0), datatypes::pointer); -} -expr get_ir_zero_index() { - return make_expr(UINT64_C(0), datatypes::index); -} -bool is_pure_func_call(const expr_c &v) { - if (!v.isa()) { - return false; - } - auto func = v.static_as()->get_prototype(); - return func->attr_ && func->attr_->get_or_else(function_attrs::pure, false); -} - -namespace builtin { -sc_data_type_t infer_output_dtype(sc_data_type_t dtype_A) { - if (dtype_A == datatypes::u8 || dtype_A == datatypes::s8) { - return datatypes::s32; - } - return datatypes::f32; -} - -void print_index(expr v) { - static const func_t print_index_f = - make_func("print_index", {make_var(datatypes::index, "v")}, stmt(), - datatypes::void_t); - _evaluate_call_(print_index_f, std::move(v)); -} - -void print_int(expr v) { - static const func_t print_int_f = make_func( - "print_int", {make_var(datatypes::s32, "v")}, stmt(), datatypes::void_t); - _evaluate_call_(print_int_f, std::move(v)); -} - -void print_float(expr v) { - static const func_t print_float_f = - make_func("print_float", {make_var(datatypes::f32, "v")}, stmt(), - datatypes::void_t); - _evaluate_call_(print_float_f, std::move(v)); -} - -void print_str(expr v) { - static const func_t print_str_f = - make_func("print_str", - {make_var(sc_data_type_t::pointerof(sc_data_etype::U8), "v")}, - stmt(), datatypes::void_t); - _evaluate_call_(print_str_f, std::move(v)); -} - -void print_str(const std::string &v) { - print_str(builder::get_current_builder()->make_str(v)); -} - -void print_str(const char *v) { print_str(std::string(v)); } - -static const func_t &mark_trace_func(const func_t &f) { - f->attr()[function_attrs::is_trace_func] = true; - return f; -} - -expr make_trace(expr func_name, expr in_or_out, expr arg) { - static func_t make_trace_f = mark_trace_func(make_func( - "sc_make_trace", - {make_var(datatypes::s32, "func_name"), - make_var(datatypes::s32, "in_or_out"), make_var(datatypes::s32, "arg")}, - stmt(), datatypes::void_t)); - return make_trace_f(std::move(func_name), std::move(in_or_out), - std::move(arg)); -} - -expr make_trace_kernel(expr func_name, expr in_or_out, expr arg) { - static func_t make_trace_f = mark_trace_func(make_func( - "sc_make_trace_kernel", - {make_var(datatypes::s32, "func_name"), - make_var(datatypes::s32, "in_or_out"), make_var(datatypes::s32, "arg")}, - stmt(), datatypes::void_t)); - return make_trace_f(std::move(func_name), std::move(in_or_out), - std::move(arg)); -} - -func_t get_brgemm_init_func() { - static func_t brgemm_func = _decl_func( - "dnnl_brgemm_init", datatypes::s32, - {_arg_("C", datatypes::pointer), _arg_("M", datatypes::s32), - _arg_("N", datatypes::s32), _arg_("LDC", datatypes::s32), - _arg_("dtypeC", datatypes::s32), _arg_("value", datatypes::f32)}); - return brgemm_func; -} - -void dnnl_brgemm_init(expr C, expr M, expr N, expr LDC, sc_data_type_t dtypeC, - expr value) { - _evaluate_call_(get_brgemm_init_func(), std::move(C), std::move(M), - std::move(N), std::move(LDC), dtypeC.as_etype_int(), - std::move(value)); -} - -static const char *brgemm_names[] = { - "dnnl", - "sc", -}; - -static const char *get_brgemm_name(scflags_t::brgemm_backend_t backend) { - return brgemm_names[static_cast(backend)]; -} - -static const func_t &mark_brgemm_func(const func_t &f) { - f->attr()["is_brgemm_func_with_stream"] = true; - return f; -} - -// returns the kernel creator and kernel caller pair -static std::pair -declare_brgemm_kernel_creator(scflags_t::brgemm_backend_t backend, - brgemm_mode mode, bool has_postop) { - std::stringstream ss; - std::string postfix = has_postop ? "_call_postops" : "_call"; - std::vector> post_args = { - _arg_("postops_data", datatypes::pointer), - _arg_("c_buf", datatypes::pointer)}; - if (mode == brgemm_mode::stride) { - ss << get_brgemm_name(backend) << "_brgemm"; - static func_t creator = _decl_func( - ss.str() + "_func", datatypes::pointer, - {_arg_("M", datatypes::s32), _arg_("N", datatypes::s32), - _arg_("K", datatypes::s32), _arg_("LDA", datatypes::s32), - _arg_("LDB", datatypes::s32), _arg_("LDC", datatypes::s32), - _arg_("stride_a", datatypes::s32), _arg_("stride_b", datatypes::s32), - _arg_("beta", datatypes::f32), _arg_("dtypeA", datatypes::s32), - _arg_("dtypeB", datatypes::s32), - _arg_("brg_attrs", datatypes::pointer), - _arg_("bd_mask", datatypes::pointer), - _arg_("postops_setting", datatypes::pointer)}); - auto caller_args = - std::vector>{_arg_("func", datatypes::pointer), - _arg_("A", datatypes::pointer), - _arg_("B", datatypes::pointer), - _arg_("C", datatypes::pointer), - _arg_("num", datatypes::s32), - _arg_("top_pad", datatypes::pointer), - _arg_("bottom_pad", datatypes::pointer), - _arg_("stream", datatypes::pointer)}; - if (has_postop) { - caller_args.insert(caller_args.end() - 1, post_args.begin(), - post_args.end()); - } - func_t caller = mark_brgemm_func(_decl_func( - ss.str() + postfix, datatypes::void_t, std::move(caller_args))); - return std::pair(creator, caller); - } else { - ss << get_brgemm_name(backend) << "_brgemm_list"; - static func_t creator = _decl_func( - ss.str() + "_func", datatypes::pointer, - {_arg_("M", datatypes::s32), _arg_("N", datatypes::s32), - _arg_("K", datatypes::s32), _arg_("LDA", datatypes::s32), - _arg_("LDB", datatypes::s32), _arg_("LDC", datatypes::s32), - _arg_("beta", datatypes::f32), _arg_("dtypeA", datatypes::s32), - _arg_("dtypeB", datatypes::s32), - _arg_("brg_attrs", datatypes::pointer), - _arg_("bd_mask", datatypes::pointer), - _arg_("postops_setting", datatypes::pointer)}); - auto caller_args = - std::vector>{_arg_("func", datatypes::pointer), - _arg_("A", datatypes::pointer), - _arg_("B", datatypes::pointer), - _arg_("C", datatypes::pointer), - _arg_("num", datatypes::s32), - _arg_("stride_a", datatypes::s32), - _arg_("stride_b", datatypes::s32), - _arg_("len", datatypes::s32), - _arg_("dtypeA", datatypes::s32), - _arg_("dtypeB", datatypes::s32), - _arg_("top_pad", datatypes::pointer), - _arg_("bottom_pad", datatypes::pointer), - _arg_("stream", datatypes::pointer)}; - if (has_postop) { - caller_args.insert(caller_args.end() - 1, post_args.begin(), - post_args.end()); - } - func_t caller = mark_brgemm_func(_decl_func( - ss.str() + postfix, datatypes::void_t, std::move(caller_args))); - return std::pair(creator, caller); - } -} - -std::pair get_brgemm_creator_and_call_func( - brgemm_mode mode, scflags_t::brgemm_backend_t backend, bool has_postop) { -#define DEF_FUNC(back, list_stride) \ - if (mode == brgemm_mode::list_stride && \ - backend == scflags_t::brgemm_backend_t::back) { \ - static std::pair f0 = \ - declare_brgemm_kernel_creator(backend, mode, false); \ - static std::pair f1 = \ - declare_brgemm_kernel_creator(backend, mode, true); \ - return has_postop ? f1 : f0; \ - } - // we need a static variable each branch to ensure there will be no - // duplicated decl for the same func_t. - DEF_FUNC(dnnl, stride) - DEF_FUNC(dnnl, addr_list) -#undef DEF_FUNC - assert(0 && "Unreachable"); - return std::pair(); -} - -// returns the kernel creator and kernel caller pair -static func_t declare_brgemm_update_funcs(scflags_t::brgemm_backend_t backend, - brgemm_mode mode, bool init) { - std::stringstream ss; - if (mode == brgemm_mode::stride) { - ss << get_brgemm_name(backend) << "_brgemm_"; - if (init) { - ss << "init_"; - } - ss << "update"; - auto update_args = std::vector>{ - _arg_("A", datatypes::pointer), - _arg_("B", datatypes::pointer), - _arg_("C", datatypes::pointer), - _arg_("num", datatypes::s32), - _arg_("M", datatypes::s32), - _arg_("N", datatypes::s32), - _arg_("K", datatypes::s32), - _arg_("LDA", datatypes::s32), - _arg_("LDB", datatypes::s32), - _arg_("LDC", datatypes::s32), - _arg_("stride_a", datatypes::s32), - _arg_("stride_b", datatypes::s32), - _arg_("dtypeA", datatypes::s32), - _arg_("dtypeB", datatypes::s32), - _arg_("brg_attrs", datatypes::pointer), - _arg_("bd_mask", datatypes::pointer), - _arg_("postops_setting", datatypes::pointer), - _arg_("top_pad", datatypes::pointer), - _arg_("bottom_pad", datatypes::pointer), - _arg_("postops_data", datatypes::pointer), - _arg_("c_buf", datatypes::pointer), - _arg_("stream", datatypes::pointer)}; - - func_t update = mark_brgemm_func( - _decl_func(ss.str(), datatypes::s32, std::move(update_args))); - return update; - } else { - ss << get_brgemm_name(backend) << "_brgemm_"; - if (init) { - ss << "init_"; - } - ss << "list_update"; - auto update_args = std::vector>{ - _arg_("A", datatypes::pointer), - _arg_("B", datatypes::pointer), - _arg_("C", datatypes::pointer), - _arg_("num", datatypes::s32), - _arg_("M", datatypes::s32), - _arg_("N", datatypes::s32), - _arg_("K", datatypes::s32), - _arg_("LDA", datatypes::s32), - _arg_("LDB", datatypes::s32), - _arg_("LDC", datatypes::s32), - _arg_("stride_a", datatypes::s32), - _arg_("stride_b", datatypes::s32), - _arg_("len", datatypes::s32), - _arg_("dtypeA", datatypes::s32), - _arg_("dtypeB", datatypes::s32), - _arg_("brg_attrs", datatypes::pointer), - _arg_("bd_mask", datatypes::pointer), - _arg_("postops_setting", datatypes::pointer), - _arg_("top_pad", datatypes::pointer), - _arg_("bottom_pad", datatypes::pointer), - _arg_("postops_data", datatypes::pointer), - _arg_("c_buf", datatypes::pointer), - _arg_("stream", datatypes::pointer)}; - func_t brgemm_func = mark_brgemm_func( - _decl_func(ss.str(), datatypes::s32, std::move(update_args))); - return brgemm_func; - } -} - -func_t get_brgemm_call_range_func(brgemm_mode mode) { - if (mode == brgemm_mode::stride) { - auto update_args = - std::vector>{_arg_("func", datatypes::pointer), - _arg_("M_real", datatypes::s32), - _arg_("N_real", datatypes::s32), - _arg_("K_real", datatypes::s32), - _arg_("A", datatypes::pointer), - _arg_("B", datatypes::pointer), - _arg_("C", datatypes::pointer), - _arg_("num", datatypes::s32), - _arg_("top_pad", datatypes::pointer), - _arg_("bottom_pad", datatypes::pointer), - _arg_("stream", datatypes::pointer)}; - - static func_t update = mark_brgemm_func(_decl_func( - "dnnl_brgemm_call_range", datatypes::s32, std::move(update_args))); - return update; - } else { - auto update_args = - std::vector>{_arg_("func", datatypes::pointer), - _arg_("M_real", datatypes::s32), - _arg_("N_real", datatypes::s32), - _arg_("K_real", datatypes::s32), - _arg_("A", datatypes::pointer), - _arg_("B", datatypes::pointer), - _arg_("C", datatypes::pointer), - _arg_("num", datatypes::s32), - _arg_("stride_a", datatypes::s32), - _arg_("stride_b", datatypes::s32), - _arg_("len", datatypes::s32), - _arg_("dtypeA", datatypes::s32), - _arg_("dtypeB", datatypes::s32), - _arg_("top_pad", datatypes::pointer), - _arg_("bottom_pad", datatypes::pointer), - _arg_("stream", datatypes::pointer)}; - static func_t brgemm_func = mark_brgemm_func(_decl_func( - "dnnl_brgemm_list_call_range", datatypes::s32, std::move(update_args))); - return brgemm_func; - } -} - -std::pair -get_brgemm_update_funcs(brgemm_mode mode, scflags_t::brgemm_backend_t backend) { -#define DEF_FUNC(back) \ - if (mode == brgemm_mode::stride && \ - backend == scflags_t::brgemm_backend_t::back) { \ - static std::pair f = std::pair( \ - declare_brgemm_update_funcs(backend, mode, false), \ - declare_brgemm_update_funcs(backend, mode, true)); \ - return f; \ - } -#define DEF_LIST_FUNC(back) \ - if (mode == brgemm_mode::addr_list && \ - backend == scflags_t::brgemm_backend_t::back) { \ - static std::pair f = std::pair( \ - declare_brgemm_update_funcs(backend, mode, false), \ - declare_brgemm_update_funcs(backend, mode, true)); \ - return f; \ - } - // we need a static variable each branch to ensure there will be no - // duplicated decl for the same func_t. - DEF_FUNC(dnnl) - DEF_LIST_FUNC(dnnl) -#undef DEF_FUNC -#undef DEF_LIST_FUNC - assert(0 && "Unreachable"); - return std::pair(); -} - -evaluate brgemm_init_update( - const expr &A, const expr &B, const expr &C, const expr &num, const expr &M, - const expr &N, const expr &K, const expr &LDA, const expr &LDB, - const expr &LDC, const expr &stride_a, const expr &stride_b, - const sc_data_type_t &dtypeA, const sc_data_type_t &dtypeB, - const sc_brgemm_attrs_t &brg_attrs, const sc_brgemm_bd_mask_t &bd_mask, - const expr &bd_mask_idx, const int &bd_mask_set_num, const expr &top_pad, - const expr &bottom_pad, const sc_brgemm_postops_setting_t &postops_set, - const std::vector &postops_data, const expr &c_buf) { - return builder::get_current_builder() - ->brgemm(A, B, C, num, M, N, K, LDA, LDB, LDC, stride_a, stride_b, - postops_data, c_buf, bd_mask_idx, top_pad, bottom_pad, - {brgemm_args::cpu_t{true}, dtypeA, dtypeB, - infer_output_dtype(dtypeA), brg_attrs, bd_mask, bd_mask_set_num, - postops_set}) - .checked_as(); -} - -evaluate brgemm_init_update_allow_fusion( - const expr &A, const expr &B, const expr &C, const expr &num, const expr &M, - const expr &N, const expr &K, const expr &LDA, const expr &LDB, - const expr &LDC, const expr &stride_a, const expr &stride_b, - const sc_data_type_t &dtypeA, const sc_data_type_t &dtypeB, - const sc_brgemm_attrs_t &brg_attrs, const sc_brgemm_bd_mask_t &bd_mask, - const expr &bd_mask_idx, const int &bd_mask_set_num, const expr &top_pad, - const expr &bottom_pad, const sc_brgemm_postops_setting_t &postops_set, - const std::vector &postops_data, const expr &c_buf) { - auto ret = brgemm_init_update(A, B, C, num, M, N, K, LDA, LDB, LDC, stride_a, - stride_b, dtypeA, dtypeB, brg_attrs, bd_mask, - bd_mask_idx, bd_mask_set_num, top_pad, - bottom_pad, postops_set, postops_data, c_buf) - .checked_as(); - builder::get_current_builder() - ->get_current_scope() - .as_seq() - .back() - .checked_as() - ->value_.checked_as() - ->intrin_attrs_->set(intrin_attr::allow_brgemm_fusion, true); - return ret; -} - -evaluate brgemm_update( - const expr &A, const expr &B, const expr &C, const expr &num, const expr &M, - const expr &N, const expr &K, const expr &LDA, const expr &LDB, - const expr &LDC, const expr &stride_a, const expr &stride_b, - const sc_data_type_t &dtypeA, const sc_data_type_t &dtypeB, - const sc_brgemm_attrs_t &brg_attrs, const sc_brgemm_bd_mask_t &bd_mask, - const expr &bd_mask_idx, const int &bd_mask_set_num, const expr &top_pad, - const expr &bottom_pad, const sc_brgemm_postops_setting_t &postops_set, - const std::vector &postops_data, const expr &c_buf) { - return builder::get_current_builder() - ->brgemm(A, B, C, num, M, N, K, LDA, LDB, LDC, stride_a, stride_b, - postops_data, c_buf, bd_mask_idx, top_pad, bottom_pad, - {brgemm_args::cpu_t{false}, dtypeA, dtypeB, - infer_output_dtype(dtypeA), brg_attrs, bd_mask, bd_mask_set_num, - postops_set}) - .checked_as(); -} - -evaluate brgemm_list_update( - const expr &A, const expr &B, const expr &C, const expr &num, const expr &M, - const expr &N, const expr &K, const expr &LDA, const expr &LDB, - const expr &LDC, const expr &stride_a, const expr &stride_b, - const expr &len, const sc_data_type_t &dtypeA, const sc_data_type_t &dtypeB, - const sc_brgemm_attrs_t &brg_attrs, const sc_brgemm_bd_mask_t &bd_mask, - const expr &bd_mask_idx, const int &bd_mask_set_num, const expr &top_pad, - const expr &bottom_pad, const sc_brgemm_postops_setting_t &postops_set, - const std::vector &postops_data, const expr &c_buf) { - A->attr()[attr_keys::no_index2var] = true; - A->attr()["list_brgemm_arg"] = true; - B->attr()[attr_keys::no_index2var] = true; - B->attr()["list_brgemm_arg"] = true; - return builder::get_current_builder() - ->list_brgemm( - A, B, C, num, M, N, K, LDA, LDB, LDC, stride_a, stride_b, len, - postops_data, c_buf, bd_mask_idx, top_pad, bottom_pad, - brgemm_args::extra_args_t(brgemm_args::cpu_t{false}, dtypeA, dtypeB, - infer_output_dtype(dtypeA), brg_attrs, - bd_mask, bd_mask_set_num, postops_set)) - .checked_as(); -} - -evaluate brgemm_init_list_update( - const expr &A, const expr &B, const expr &C, const expr &num, const expr &M, - const expr &N, const expr &K, const expr &LDA, const expr &LDB, - const expr &LDC, const expr &stride_a, const expr &stride_b, - const expr &len, const sc_data_type_t &dtypeA, const sc_data_type_t &dtypeB, - const sc_brgemm_attrs_t &brg_attrs, const sc_brgemm_bd_mask_t &bd_mask, - const expr &bd_mask_idx, const int &bd_mask_set_num, const expr &top_pad, - const expr &bottom_pad, const sc_brgemm_postops_setting_t &postops_set, - const std::vector &postops_data, const expr &c_buf) { - A->attr()[attr_keys::no_index2var] = true; - A->attr()["list_brgemm_arg"] = true; - B->attr()[attr_keys::no_index2var] = true; - B->attr()["list_brgemm_arg"] = true; - return builder::get_current_builder() - ->list_brgemm( - A, B, C, num, M, N, K, LDA, LDB, LDC, stride_a, stride_b, len, - postops_data, c_buf, bd_mask_idx, top_pad, bottom_pad, - brgemm_args::extra_args_t(brgemm_args::cpu_t{true}, dtypeA, dtypeB, - infer_output_dtype(dtypeA), brg_attrs, - bd_mask, bd_mask_set_num, postops_set)) - .checked_as(); -} - -void brgemm_init(expr C, expr M, expr N, expr LDC, sc_data_type_t dtypeC, - expr value) { - dnnl_brgemm_init(std::move(C), std::move(M), std::move(N), std::move(LDC), - dtypeC, std::move(value)); -} - -func_t get_mem_set_func() { - static func_t memzerofunc = - _decl_func("memset", datatypes::pointer, - {_arg_("ptr", datatypes::pointer), _arg_("v", datatypes::s32), - _arg_("len", datatypes::index)}); - return memzerofunc; -} - -void mem_zero(expr C, const expr &size, sc_data_type_t dtype) { - _evaluate_call_(get_mem_set_func(), std::move(C), 0, - size * utils::get_sizeof_type(dtype)); -} - -func_t get_brgemm_postops_data_init_func() { - static func_t data_init_func = _decl_func( - "dnnl_brgemm_postops_data_init", datatypes::void_t, - {_arg_("dnnl_data", datatypes::pointer), - _arg_("bias", datatypes::pointer), _arg_("scales", datatypes::pointer), - _arg_("binary_post_ops_rhs", datatypes::pointer), - _arg_("oc_logical_off", datatypes::index), - _arg_("dst_row_logical_off", datatypes::index), - _arg_("data_C_ptr_", datatypes::pointer), - _arg_("first_mb_matrix_addr_off", datatypes::index), - _arg_("a_zp_compensations", datatypes::pointer), - _arg_("b_zp_compensations", datatypes::pointer), - _arg_("c_zp_values", datatypes::pointer), - _arg_("skip_accumulation", datatypes::boolean), - _arg_("zp_a_val", datatypes::s32), - _arg_("do_only_comp", datatypes::boolean), - _arg_("do_only_zp_a_val", datatypes::boolean)}); - return data_init_func; -} - -std::vector create_initialed_postops_data() { - std::vector data(brgemm::postops_data_init_func_nargs, get_ir_null()); - auto false_node = make_expr(UINT64_C(0), datatypes::boolean); - // oc_logical_off - data[3] = get_ir_zero_index(); - // dst_row_logical_off - data[4] = get_ir_zero_index(); - // first_mb_matrix_addr_off - data[6] = get_ir_zero_index(); - // skip_accumulation - data[10] = false_node; - // zp_a_val - data[11] = make_expr(UINT64_C(0), datatypes::s32); - // do_only_comp - data[12] = false_node; - // do_only_zp_a_val - data[13] = false_node; - return data; -} - -// dynamic query format function call -expr call_matmul_core_query_format( - const expr &tb, const expr &out0, const expr &in0, const expr &in1, - const expr &ori_in0, const expr &ori_in1, const expr &out_format0, - const expr &in_format0, const expr &in_format1, const expr &ori_in_format0, - const expr &ori_in_format1, const expr &out_size, const expr &kernel, - const expr &impl) { - static func_t matmul_core_query_f = - make_func("query_format_matmul_core_op", - {make_var(datatypes::pointer, "op_table"), - make_var(datatypes::pointer, "out"), - make_var(datatypes::pointer, "inp0"), - make_var(datatypes::pointer, "inp1"), - make_var(datatypes::pointer, "ori_inp0"), - make_var(datatypes::pointer, "ori_inp1"), - make_var(datatypes::pointer, "out_fmt"), - make_var(datatypes::pointer, "inp_fmt0"), - make_var(datatypes::pointer, "inp_fmt1"), - make_var(datatypes::pointer, "ori_inp_fmt0"), - make_var(datatypes::pointer, "ori_inp_fmt1"), - make_var(datatypes::pointer, "out_size"), - make_var(datatypes::pointer, "kernel"), - make_var(datatypes::pointer, "impl")}, - stmt(), datatypes::void_t); - return matmul_core_query_f(tb, out0, in0, in1, ori_in0, ori_in1, out_format0, - in_format0, in_format1, ori_in_format0, - ori_in_format1, out_size, kernel, impl); -} - -expr call_managed_matmul_core_query_format( - const expr &tb, const expr &out0, const expr &in0, const expr &in1, - const expr &ori_in0, const expr &ori_in1, const expr &out_format0, - const expr &in_format0, const expr &in_format1, const expr &ori_in_format0, - const expr &ori_in_format1, const expr &out_size, const expr &kernel, - const expr &impl) { - static func_t mmm_query_f = - make_func("query_format_managed_matmul_core_op", - {make_var(datatypes::pointer, "op_table"), - make_var(datatypes::pointer, "out"), - make_var(datatypes::pointer, "inp0"), - make_var(datatypes::pointer, "inp1"), - make_var(datatypes::pointer, "ori_inp0"), - make_var(datatypes::pointer, "ori_inp1"), - make_var(datatypes::pointer, "out_fmt"), - make_var(datatypes::pointer, "inp_fmt0"), - make_var(datatypes::pointer, "inp_fmt1"), - make_var(datatypes::pointer, "ori_inp_fmt0"), - make_var(datatypes::pointer, "ori_inp_fmt1"), - make_var(datatypes::pointer, "out_size"), - make_var(datatypes::pointer, "kernel"), - make_var(datatypes::pointer, "impl")}, - stmt(), datatypes::void_t); - return mmm_query_f(tb, out0, in0, in1, ori_in0, ori_in1, out_format0, - in_format0, in_format1, ori_in_format0, ori_in_format1, - out_size, kernel, impl); -} - -expr call_conv_fwd_core_query_format( - const expr &tb, const expr &out0, const expr &in0, const expr &in1, - const expr &ori_in0, const expr &ori_in1, const expr &out_format0, - const expr &in_format0, const expr &in_format1, const expr &ori_in_format0, - const expr &ori_in_format1, const expr &out_size, const expr &kernel, - const expr &impl) { - static func_t conv_fwd_core_query_f = - make_func("query_format_conv_fwd_core_op", - {make_var(datatypes::pointer, "op_table"), - make_var(datatypes::pointer, "out"), - make_var(datatypes::pointer, "inp0"), - make_var(datatypes::pointer, "inp1"), - make_var(datatypes::pointer, "ori_inp0"), - make_var(datatypes::pointer, "ori_inp1"), - make_var(datatypes::pointer, "out_fmt"), - make_var(datatypes::pointer, "inp_fmt0"), - make_var(datatypes::pointer, "inp_fmt1"), - make_var(datatypes::pointer, "ori_inp_fmt0"), - make_var(datatypes::pointer, "ori_inp_fmt1"), - make_var(datatypes::pointer, "out_size"), - make_var(datatypes::pointer, "kernel"), - make_var(datatypes::pointer, "impl")}, - stmt(), datatypes::void_t); - return conv_fwd_core_query_f( - tb, out0, in0, in1, ori_in0, ori_in1, out_format0, in_format0, in_format1, - ori_in_format0, ori_in_format1, out_size, kernel, impl); -} - -expr call_unary_fusible_op_query_format( - const expr &tb, const expr &out0, const expr &in0, const expr &out_format0, - const expr &in_format0, const expr &out_size, const expr &kernel) { - static func_t unary_query_f = make_func( - "query_format_unary_fusible_op", - {make_var(datatypes::pointer, "op_table"), - make_var(datatypes::pointer, "out"), make_var(datatypes::pointer, "inp"), - make_var(datatypes::pointer, "out_fmt"), - make_var(datatypes::pointer, "inp_fmt"), - make_var(datatypes::pointer, "out_size"), - make_var(datatypes::pointer, "kernel")}, - stmt(), datatypes::void_t); - return unary_query_f(tb, out0, in0, out_format0, in_format0, out_size, - kernel); -} - -expr call_padding_op_query_format(const expr &tb, const expr &out0, - const expr &in0, const expr &out_format0, - const expr &in_format0, const expr &out_size, - const expr &kernel) { - static func_t padding_query_f = make_func( - "query_format_padding_op", - {make_var(datatypes::pointer, "op_table"), - make_var(datatypes::pointer, "out"), make_var(datatypes::pointer, "inp"), - make_var(datatypes::pointer, "out_fmt"), - make_var(datatypes::pointer, "inp_fmt"), - make_var(datatypes::pointer, "out_size"), - make_var(datatypes::pointer, "kernel")}, - stmt(), datatypes::void_t); - return padding_query_f(tb, out0, in0, out_format0, in_format0, out_size, - kernel); -} - -expr call_pooling_op_query_format(const expr &tb, const expr &out0, - const expr &in0, const expr &out_format0, - const expr &in_format0, const expr &out_size, - const expr &kernel) { - static func_t pooling_query_f = make_func( - "query_format_pooling_op", - {make_var(datatypes::pointer, "op_table"), - make_var(datatypes::pointer, "out"), make_var(datatypes::pointer, "inp"), - make_var(datatypes::pointer, "out_fmt"), - make_var(datatypes::pointer, "inp_fmt"), - make_var(datatypes::pointer, "out_size"), - make_var(datatypes::pointer, "kernel")}, - stmt(), datatypes::void_t); - return pooling_query_f(tb, out0, in0, out_format0, in_format0, out_size, - kernel); -} - -expr call_binary_fusible_op_query_format( - const expr &tb, const expr &out0, const expr &in0, const expr &in1, - const expr &out_format0, const expr &in_format0, const expr &in_format1, - const expr &out_size, const expr &kernel) { - static func_t binary_query_f = - make_func("query_format_binary_fusible_op", - {make_var(datatypes::pointer, "op_table"), - make_var(datatypes::pointer, "out"), - make_var(datatypes::pointer, "inp0"), - make_var(datatypes::pointer, "inp1"), - make_var(datatypes::pointer, "out_fmt"), - make_var(datatypes::pointer, "inp_fmt0"), - make_var(datatypes::pointer, "inp_fmt1"), - make_var(datatypes::pointer, "out_size"), - make_var(datatypes::pointer, "kernel")}, - stmt(), datatypes::void_t); - return binary_query_f(tb, out0, in0, in1, out_format0, in_format0, in_format1, - out_size, kernel); -} - -expr call_reorder_op_query_format(const expr &tb, const expr &out0, - const expr &in0, const expr &out_format0, - const expr &in_format0, const expr &out_size, - const expr &kernel, const expr &impl) { - static func_t reorder_query_f = make_func( - "query_format_reorder_op", - {make_var(datatypes::pointer, "op_table"), - make_var(datatypes::pointer, "out"), make_var(datatypes::pointer, "inp"), - make_var(datatypes::pointer, "out_fmt"), - make_var(datatypes::pointer, "inp_fmt"), - make_var(datatypes::pointer, "out_size"), - make_var(datatypes::pointer, "kernel"), - make_var(datatypes::pointer, "impl")}, - stmt(), datatypes::void_t); - return reorder_query_f(tb, out0, in0, out_format0, in_format0, out_size, - kernel, impl); -} - -expr call_reduce_op_query_format(const expr &tb, const expr &out0, - const expr &in0, const expr &out_format0, - const expr &in_format0, const expr &out_size, - const expr &kernel) { - static func_t reorder_query_f = make_func( - "query_format_reduce_op", - {make_var(datatypes::pointer, "op_table"), - make_var(datatypes::pointer, "out"), make_var(datatypes::pointer, "inp"), - make_var(datatypes::pointer, "out_fmt"), - make_var(datatypes::pointer, "inp_fmt"), - make_var(datatypes::pointer, "out_size"), - // make_var(datatypes::s32, "rd_axis"), - make_var(datatypes::pointer, "kernel")}, - stmt(), datatypes::void_t); - return reorder_query_f(tb, out0, in0, out_format0, in_format0, out_size, - kernel); -} - -expr call_tensor_view_op_query_format(const expr &tb, const expr &out0, - const expr &in0, const expr &out_format0, - const expr &in_format0, - const expr &out_size, - const expr &kernel) { - static func_t reorder_query_f = make_func( - "query_format_tensor_view_op", - {make_var(datatypes::pointer, "op_table"), - make_var(datatypes::pointer, "out"), make_var(datatypes::pointer, "inp"), - make_var(datatypes::pointer, "out_fmt"), - make_var(datatypes::pointer, "inp_fmt"), - make_var(datatypes::pointer, "out_size"), - make_var(datatypes::pointer, "kernel")}, - stmt(), datatypes::void_t); - return reorder_query_f(tb, out0, in0, out_format0, in_format0, out_size, - kernel); -} - -expr call_select_op_query_format(const expr &tb, const expr &out0, - const expr &in0, const expr &in1, - const expr &in2, const expr &out_format0, - const expr &in_format0, const expr &in_format1, - const expr &in_format2, const expr &out_size, - const expr &kernel) { - static func_t select_query_f = - make_func("query_format_select_op", - {make_var(datatypes::pointer, "op_table"), - make_var(datatypes::pointer, "out"), - make_var(datatypes::pointer, "inp0"), - make_var(datatypes::pointer, "inp1"), - make_var(datatypes::pointer, "inp2"), - make_var(datatypes::pointer, "out_fmt"), - make_var(datatypes::pointer, "inp0_fmt"), - make_var(datatypes::pointer, "inp1_fmt"), - make_var(datatypes::pointer, "inp2_fmt"), - make_var(datatypes::pointer, "out_size"), - make_var(datatypes::pointer, "kernel")}, - stmt(), datatypes::void_t); - return select_query_f(tb, out0, in0, in1, in2, out_format0, in_format0, - in_format1, in_format2, out_size, kernel); -} - -expr call_fused_op_query_combined(const expr &tb, const expr &combined_keys, - const expr &combined_algs, - const expr &each_op_num_key, - const expr &op_num, const expr &kernel) { - static func_t fused_query_f = - make_func("query_combined_fused_op", - {make_var(datatypes::pointer, "op_table"), - make_var(datatypes::pointer, "combined_keys"), - make_var(datatypes::pointer, "combined_algs"), - make_var(datatypes::pointer, "each_op_num_key"), - make_var(datatypes::s32, "op_num"), - make_var(datatypes::pointer, "kernel")}, - stmt(), datatypes::void_t); - return fused_query_f(tb, combined_keys, combined_algs, each_op_num_key, - op_num, kernel); -} - -expr call_cal_blocking_dims(const expr &placeholder, const expr &format) { - static func_t cal_blocking_f = - make_func("calculate_blocking_dims", - {make_var(datatypes::pointer, "placeholder"), - make_var(datatypes::pointer, "format")}, - stmt(), datatypes::index); - return cal_blocking_f(placeholder, format); -} - -expr call_get_matmul_dyn_cfg_single(const expr &in, const expr &is_batch) { - static func_t matmul_dyn_cfg_f = - make_func("get_matmul_dyn_cfg_single", - {make_var(datatypes::s32, "in"), - make_var(datatypes::boolean, "is_batch")}, - stmt(), datatypes::s32); - return matmul_dyn_cfg_f(in, is_batch); -} - -static func_t set_pure_function(func_t f) { - f->attr()[function_attrs::pure] = true; - return f; -} - -func_t get_thread_id_func() { - static func_t func = - set_pure_function(_decl_func("sc_get_thread_id", datatypes::s32, {})); - return func; -} - -func_t get_is_in_parallel_func() { - static func_t func = - set_pure_function(_decl_func("sc_is_in_parallel", datatypes::s32, {})); - return func; -} - -func_t get_barrier_arrive_func() { - static func_t func = _decl_func("sc_arrive_at_barrier", datatypes::void_t, - {_arg_("b", datatypes::pointer), - _arg_("idle_func", datatypes::pointer), - _arg_("idle_args", datatypes::pointer)}); - return func; -} - -func_t get_init_barrier_func() { - static func_t func = - _decl_func("sc_init_barrier", datatypes::void_t, - {_arg_("b", datatypes::pointer), _arg_("num", datatypes::s32), - _arg_("thread_cnt", datatypes::index)}); - return func; -} - -func_t get_set_idle_func_managed_func() { - static func_t f = - builder::make_func("sc_set_idle_func_managed", - {builder::make_var(datatypes::pointer, "funcptr"), - builder::make_var(datatypes::pointer, "arg1")}, - stmt(), datatypes::void_t); - return f; -} - -func_t get_tls_amx_buffer_func() { - static func_t f = - set_pure_function(_decl_func("sc_get_tls_amx_buffer", datatypes::pointer, - {_arg_("stream", datatypes::pointer)})); - return f; -} - -// get the sum of elements with indices [0,1,...,gid] of the -// array: [A,A,..., A, B, B,...,B], where the count of element "A" is num_A -static expr generate_balance211_job_id_base(const expr &gid, const expr &num_A, - const expr &val_A, - const expr &val_B) { - return builder::make_select(gid <= num_A, gid * val_A, - num_A * val_A + (gid - num_A) * val_B); -} - -uint64_t generate_balance211( - int num_threads, const expr &start_e, const expr &end_e, const expr &step_e, - const expr &gid, const std::function &namer, - expr *out_start, expr *out_len, expr *out_end, std::vector *out_seq) { - // the util function to define a var with init value - auto def_var = [&](const char *name, const expr &init_v, - sc_data_type_t dtype = datatypes::index) { - auto ret = builder::make_var(dtype, namer ? namer(name) : name); - auto def = builder::make_var_tensor_def_unattached( - ret, linkage::local, do_cast_and_fold(init_v)); - if (out_seq) { - out_seq->emplace_back(def); - } else { - builder::get_current_builder()->emit(def); - } - return ret; - }; - // the util function to define the expressions to return - auto make_output_vars = [&](expr &my_begin, const expr &cur_jobs) { - my_begin = do_cast_and_fold(my_begin); - *out_start = def_var("_start", my_begin); - - expr out_len_e; - if (out_end || out_len) { - out_len_e = def_var("_len", do_cast_and_fold(cur_jobs * step_e)); - } - if (out_end) { - expr my_end = *out_start + out_len_e; - my_end = do_cast_and_fold(my_end); - *out_end = def_var("_end", my_end); - } - if (out_len) { - *out_len = out_len_e; - } - }; - expr the_tid, my_jobs_e, my_jobs_2_e; - if (start_e.isa() && end_e.isa() && - step_e.isa()) { - // if is constant-for (in most cases) - uint64_t end = get_const_as_int(end_e.static_as()); - uint64_t begin = get_const_as_int(start_e.static_as()); - uint64_t step = get_const_as_int(step_e.static_as()); - auto len = end - begin; - auto num_jobs = utils::divide_and_ceil(len, step); - uint64_t my_jobs = utils::divide_and_ceil(num_jobs, num_threads); - COMPILE_ASSERT(my_jobs > 0, "Bad number of jobs"); - if (num_jobs % num_threads == 0) { - // fast path. the jobs is divisible by the thread number - *out_start = def_var("_start", gid * (my_jobs * step) + begin); - if (out_end) { - *out_end = def_var("_end", *out_start + (my_jobs * step)); - } - if (out_len) { - *out_len = my_jobs * step; - } - return num_threads; - } - uint64_t my_jobs_2 = my_jobs - 1; - // number of threads doing my_jobs work - uint64_t num_thread_larger_work = num_jobs - my_jobs_2 * num_threads; - // number of threads doing my_jobs - 1 work - uint64_t num_thread_less_work = num_threads - num_thread_larger_work; - // the loop is divisible with num_thread_less_work parts - // and each part has same number of works - // the loop can be further "split" into outer and inner loops and - // the outer loop may be merged with another loop - uint64_t num_split = - math_utils::get_gcd(num_thread_larger_work, num_threads); - if (num_split > 1UL) { - // e.g. 22 jobs and 8 threads - // my_jobs=3 my_jobs_2=2 - // num_thread_larger_work = 6 - // num_split = gcd(num_thread_larger_work, num_threads) = 2 - // num_thread_each_split = 4 - // num_thread_larger_work_per_split = 6/2 = 3 - // cur_jobs for each thread: 3,3,3,2 | 3,3,3,2 <<< 2 splits - uint64_t num_thread_each_split = num_threads / num_split; - uint64_t num_thread_larger_work_per_split = - num_thread_larger_work / num_split; - expr id_in_split = def_var("id_in_split", gid % num_thread_each_split); - expr is_large_work = id_in_split < num_thread_larger_work_per_split; - expr cur_jobs = builder::make_select(is_large_work, my_jobs, my_jobs_2); - // for gid=5, base_job_id = 11 - expr base_job_id = (gid / num_thread_each_split) * (num_jobs / num_split); - // for gid=5, job_id_offset = 3 - expr job_id_offset; - if (num_thread_larger_work_per_split == num_thread_each_split - 1) { - job_id_offset = id_in_split * my_jobs; - } else { - job_id_offset = generate_balance211_job_id_base( - id_in_split, num_thread_larger_work_per_split, my_jobs, my_jobs_2); - } - expr my_begin = start_e + (base_job_id + job_id_offset) * step_e; - make_output_vars(my_begin, cur_jobs); - return num_split; - } - // fallback way: non-divisible loop - the_tid = num_thread_larger_work; - my_jobs_e = my_jobs; - my_jobs_2_e = my_jobs_2; - - } else { - auto &end = end_e; - auto &begin = start_e; - auto &step = step_e; - auto len = end - begin; - auto num_jobs = def_var("num_jobs", (len + step - 1) / step); - my_jobs_e = - def_var("my_jobs", (num_jobs + (num_threads - 1)) / num_threads); - // assert(my_jobs > 0); - my_jobs_2_e = def_var("my_jobs2", my_jobs_e - 1); - the_tid = def_var("the_tid", num_jobs - my_jobs_2_e * num_threads); - } - expr cur_jobs = builder::make_select(gid < the_tid, my_jobs_e, my_jobs_2_e); - expr my_begin = start_e + generate_balance211_job_id_base( - gid, the_tid, my_jobs_e, my_jobs_2_e) * - step_e; - make_output_vars(my_begin, cur_jobs); - return 0; -} - -} // namespace builtin -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/builtin.hpp b/legacy/core/src/compiler/ir/builtin.hpp deleted file mode 100644 index 15aac7f7c..000000000 --- a/legacy/core/src/compiler/ir/builtin.hpp +++ /dev/null @@ -1,505 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_BUILTIN_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_BUILTIN_HPP - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -SC_INTERNAL_API expr get_ir_null(); -SC_INTERNAL_API expr get_ir_zero_index(); -SC_INTERNAL_API bool is_pure_func_call(const expr_c &v); - -namespace builtin { - -/** - * Infer output dtype based on dtype of input A for brgemm - * */ -SC_INTERNAL_API sc_data_type_t infer_output_dtype(sc_data_type_t dtype_A); - -/** - * Generates a call node to print_index, and wrap the call node with - * evaluate node. Also declares the print_index function in the builder - * */ -void print_index(expr v); - -/** - * Generates a call node to print_int, and wrap the call node with - * evaluate node. Also declares the print_int function in the builder - * */ -void print_int(expr v); - -/** - * Generates a call node to print_float, and wrap the call node with - * evaluate node. Also declares the print_float function in the builder - * */ -void print_float(expr v); - -/** - * Generates a call node to print_str function, and wrap the call node - * with evaluate node. Also declares the print_str function in the builder - * */ -void print_str(expr v); - -/** - * Generates a string and pass the string to a call node to print_str function, - * and wrap the call node with evaluate node. Also declares the print_str - * function in the builder - * */ -void print_str(const std::string &v); - -/** - * Generates a string and pass the string to a call node to print_str function, - * and wrap the call node with evaluate node. Also declares the print_str - * function in the builder - * */ -void print_str(const char *v); - -/** - * Generates a evaluate_call to sc_make_trace - * @param func_id the function id, s32 - * @param in_or_out s32, if 0, this is the entry trace of the function. if 1, - * this is the exit trace of the function - * @param arg the argument in the trace - * */ -expr make_trace(expr func_name, expr in_or_out, expr arg); - -/** - * Generates a evaluate_call to sc_make_trace_kernel - * @param func_id the function id, s32 - * @param in_or_out s32, if 0, this is the entry trace of the function. if 1, - * this is the exit trace of the function - * @param arg the argument in the trace - * */ -expr make_trace_kernel(expr func_name, expr in_or_out, expr arg); - -// Create a initialized postops data vector. Its length matches the number of -// dnnl postop data init func args -SC_INTERNAL_API std::vector create_initialed_postops_data(); - -/** - * Generates a call node to dnnl_brgemm_init, and wrap the call node - * with evaluate node. Also declares the dnnl_brgemm_init function in - * the builder - * - * @param C pointer (float) - * @param M s32 - * @param N s32 - * @param LDC s32 - * @param dtypeC sc_data_type_t - * @param value f32 - * */ -void dnnl_brgemm_init(expr C, expr M, expr N, expr LDC, sc_data_type_t dtypeC, - expr value); - -/** - * Generates a generate call node to brgemm_init_update. - * - * @param A pointer (float) - * @param B pointer (float) - * @param C pointer (float) - * @param num s32 - * @param M s32 - * @param N s32 - * @param K s32 - * @param LDA s32 - * @param LDB s32 - * @param LDC s32 - * @param stride_a s32 - * @param stride_b s32 - * @param dtypeA sc_data_type_t - * @param dtypeB sc_data_type_t - * @param brg_attrs any_map - * @param bd_mask bd_mask - * @param bd_mask_idx bd_mask idx for same brgemm with different bd_mask - * @param bd_mask_set_num num of different bd_mask for same brgemm - * @param brg_postops_setting postops_setting - * @param brg_postops_data postops_data - * @param brg_c_buf c_buf - * */ -evaluate brgemm_init_update( - const expr &A, const expr &B, const expr &C, const expr &num, const expr &M, - const expr &N, const expr &K, const expr &LDA, const expr &LDB, - const expr &LDC, const expr &stride_a, const expr &stride_b, - const sc_data_type_t &dtypeA, const sc_data_type_t &dtypeB, - const sc_brgemm_attrs_t &brg_attrs = sc_brgemm_attrs_t(), - const sc_brgemm_bd_mask_t &bd_mask = sc_brgemm_bd_mask_t(), - const expr &bd_mask_idx = get_ir_zero_index(), - const int &bd_mask_set_num = 1, const expr &top_pad = get_ir_null(), - const expr &bottom_pad = get_ir_null(), - const sc_brgemm_postops_setting_t &brg_postops_setting = - sc_brgemm_postops_setting_t(), - const std::vector &brg_postops_data = create_initialed_postops_data(), - const expr &brg_c_buf = get_ir_null()); - -/** - * Generates a generate call node to brgemm_init_update. If you want to use - * brgemm fusion, please use this interface. Notice that brgemm fusion needs all - * calculation of reduce axis done. - * - * @param A pointer (float) - * @param B pointer (float) - * @param C pointer (float) - * @param num s32 - * @param M s32 - * @param N s32 - * @param K s32 - * @param LDA s32 - * @param LDB s32 - * @param LDC s32 - * @param stride_a s32 - * @param stride_b s32 - * @param dtypeA sc_data_type_t - * @param dtypeB sc_data_type_t - * @param brg_attrs any_map - * @param bd_mask bd_mask - * @param bd_mask_idx bd_mask idx for same brgemm with different bd_mask - * @param bd_mask_set_num num of different bd_mask for same brgemm - * @param top_pad pointer, indicate the first padding num on M dimension(bd) - * @param bottom_pad pointer, indicate the last padding num on M dimension(bd) - * @param brg_postops_setting postops_setting - * @param brg_postops_data postops_data - * @param brg_c_buf c_buf - * */ -evaluate brgemm_init_update_allow_fusion( - const expr &A, const expr &B, const expr &C, const expr &num, const expr &M, - const expr &N, const expr &K, const expr &LDA, const expr &LDB, - const expr &LDC, const expr &stride_a, const expr &stride_b, - const sc_data_type_t &dtypeA, const sc_data_type_t &dtypeB, - const sc_brgemm_attrs_t &brg_attrs = sc_brgemm_attrs_t(), - const sc_brgemm_bd_mask_t &bd_mask = sc_brgemm_bd_mask_t(), - const expr &bd_mask_idx = get_ir_zero_index(), - const int &bd_mask_set_num = 1, const expr &top_pad = get_ir_null(), - const expr &bottom_pad = get_ir_null(), - const sc_brgemm_postops_setting_t &brg_postops_setting = - sc_brgemm_postops_setting_t(), - const std::vector &brg_postops_data = create_initialed_postops_data(), - const expr &brg_c_buf = get_ir_null()); - -/** - * Generates a generate call node to brgemm_init_f32. - * - * @param C pointer (void) - * @param M s32 - * @param N s32 - * @param LDC s32 - * @param dtypeC sc_data_type_t - * @param value f32 - * */ -void brgemm_init(expr C, expr M, expr N, expr LDC, sc_data_type_t dtypeC, - expr value); - -/** - * Generates a generate call node to brgemm_update. - * - * @param A pointer (void) - * @param B pointer (void) - * @param C pointer (void) - * @param num s32 - * @param M s32 - * @param N s32 - * @param K s32 - * @param LDA s32 - * @param LDB s32 - * @param LDC s32 - * @param stride_a s32 - * @param stride_b s32 - * @param dtypeA sc_data_type_t - * @param dtypeB sc_data_type_t - * @param brg_attrs any_map - * @param bd_mask bd_mask - * @param bd_mask_idx bd_mask idx for same brgemm with different bd_mask - * @param bd_mask_set_num num of different bd_mask for same brgemm - * @param top_pad pointer, indicate the first padding num on M dimension(bd) - * @param bottom_pad pointer, indicate the first padding num on M dimension(bd) - * @param brg_postops_setting postops_setting - * @param brg_postops_data postops_data - * @param brg_c_buf c_buf - * */ -evaluate brgemm_update( - const expr &A, const expr &B, const expr &C, const expr &num, const expr &M, - const expr &N, const expr &K, const expr &LDA, const expr &LDB, - const expr &LDC, const expr &stride_a, const expr &stride_b, - const sc_data_type_t &dtypeA, const sc_data_type_t &dtypeB, - const sc_brgemm_attrs_t &brg_attrs = sc_brgemm_attrs_t(), - const sc_brgemm_bd_mask_t &bd_mask = sc_brgemm_bd_mask_t(), - const expr &bd_mask_idx = get_ir_zero_index(), - const int &bd_mask_set_num = 1, const expr &top_pad = get_ir_null(), - const expr &bottom_pad = get_ir_null(), - const sc_brgemm_postops_setting_t &brg_postops_setting = - sc_brgemm_postops_setting_t(), - const std::vector &brg_postops_data = create_initialed_postops_data(), - const expr &brg_c_buf = get_ir_null()); - -/** - * Generates a generate call node to brgemm_list_update. - * - * @param A pointer (void) - * @param B pointer (void) - * @param C pointer (void) - * @param num s32 - * @param M s32 - * @param N s32 - * @param K s32 - * @param lda s32 - * @param ldb s32 - * @param ldc s32 - * @param stride_a s32 - * @param stride_b s32 - * @param len s32 (len of addr list, each addr list contains num addrs which - * inferred via stride_a and strid_b) - * @param dtypeA sc_data_type_t - * @param dtypeB sc_data_type_t - * @param brg_attrs any_map - * @param bd_mask bd_mask - * @param bd_mask_idx bd_mask idx for same brgemm with different bd_mask - * @param bd_mask_set_num num of different bd_mask for same brgemm - * @param top_pad pointer, indicate the first padding num on M dimension(bd) - * @param bottom_pad pointer, indicate the first padding num on M dimension(bd) - * @param brg_postops_setting postops_setting - * @param brg_postops_data postops_data - * @param brg_c_buf c_buf - * */ -evaluate brgemm_list_update( - const expr &A, const expr &B, const expr &C, const expr &num, const expr &M, - const expr &N, const expr &K, const expr &lda, const expr &ldb, - const expr &ldc, const expr &stride_a, const expr &stride_b, - const expr &len, const sc_data_type_t &dtypeA, const sc_data_type_t &dtypeB, - const sc_brgemm_attrs_t &brg_attrs = sc_brgemm_attrs_t(), - const sc_brgemm_bd_mask_t &bd_mask = sc_brgemm_bd_mask_t(), - const expr &bd_mask_idx = get_ir_zero_index(), - const int &bd_mask_set_num = 1, const expr &top_pad = get_ir_null(), - const expr &bottom_pad = get_ir_null(), - const sc_brgemm_postops_setting_t &brg_postops_setting = - sc_brgemm_postops_setting_t(), - const std::vector &brg_postops_data = create_initialed_postops_data(), - const expr &brg_c_buf = get_ir_null()); - -/** - * Generates a generate call node to brgemm_init_list_update. - * - * @param A pointer (void) - * @param B pointer (void) - * @param C pointer (void) - * @param num s32 - * @param M s32 - * @param N s32 - * @param K s32 - * @param lda s32 - * @param ldb s32 - * @param ldc s32 - * @param stride_a s32 - * @param stride_b s32 - * @param len s32 (len of addr list, each addr list contains num addrs which - * inferred via stride_a and strid_b) - * @param dtypeA sc_data_type_t - * @param dtypeB sc_data_type_t - * @param brg_attrs any_map - * @param bd_mask bd_mask - * @param bd_mask_idx bd_mask idx for same brgemm with different bd_mask - * @param bd_mask_set_num num of different bd_mask for same brgemm - * @param top_pad pointer, indicate the first padding num on M dimension(bd) - * @param bottom_pad pointer, indicate the first padding num on M dimension(bd) - * @param brg_postops_setting postops_setting - * @param brg_postops_data postops_data - * @param brg_c_buf c_buf - * */ -evaluate brgemm_init_list_update( - const expr &A, const expr &B, const expr &C, const expr &num, const expr &M, - const expr &N, const expr &K, const expr &lda, const expr &ldb, - const expr &ldc, const expr &stride_a, const expr &stride_b, - const expr &len, const sc_data_type_t &dtypeA, const sc_data_type_t &dtypeB, - const sc_brgemm_attrs_t &brg_attrs = sc_brgemm_attrs_t(), - const sc_brgemm_bd_mask_t &bd_mask = sc_brgemm_bd_mask_t(), - const expr &bd_mask_idx = get_ir_zero_index(), - const int &bd_mask_set_num = 1, const expr &top_pad = get_ir_null(), - const expr &bottom_pad = get_ir_null(), - const sc_brgemm_postops_setting_t &brg_postops_setting = - sc_brgemm_postops_setting_t(), - const std::vector &brg_postops_data = create_initialed_postops_data(), - const expr &brg_c_buf = get_ir_null()); - -/** - * Generates a generate call node to mem_zero. - * use 'memset' to initialize a specified buffer to zero, size of which equals - * to size*sizeof(dtype). - * - * @param C pointer (void) - * @param size index - * @param dtype sc_data_type_t - * */ -void mem_zero(expr C, const expr &size, sc_data_type_t dtype); -func_t get_mem_set_func(); - -/** - * Generates a call node to convert multiple buffers(like scales, bias) to one - * dnnl postop data type struct. - * - * @return the created func. - * */ -func_t get_brgemm_postops_data_init_func(); - -enum class brgemm_mode { - // offset doesn't used for now. - // offset, - stride, - addr_list, -}; - -// returns pair -std::pair get_brgemm_creator_and_call_func( - brgemm_mode mode, scflags_t::brgemm_backend_t backend, bool has_postop); - -// returns pair -std::pair -get_brgemm_update_funcs(brgemm_mode mode, scflags_t::brgemm_backend_t backend); -func_t get_brgemm_call_range_func(brgemm_mode mode); - -// dynamic query format function evaluation at runtime. -expr call_matmul_core_query_format( - const expr &tb, const expr &out0, const expr &in0, const expr &in1, - const expr &ori_in0, const expr &ori_in1, const expr &out_format0, - const expr &in_format0, const expr &in_format1, const expr &ori_in_format0, - const expr &ori_in_format1, const expr &out_size, const expr &kernel, - const expr &impl = get_ir_null()); -expr call_managed_matmul_core_query_format( - const expr &tb, const expr &out0, const expr &in0, const expr &in1, - const expr &ori_in0, const expr &ori_in1, const expr &out_format0, - const expr &in_format0, const expr &in_format1, const expr &ori_in_format0, - const expr &ori_in_format1, const expr &out_size, const expr &kernel, - const expr &impl = get_ir_null()); -expr call_conv_fwd_core_query_format( - const expr &tb, const expr &out0, const expr &in0, const expr &in1, - const expr &ori_in0, const expr &ori_in1, const expr &out_format0, - const expr &in_format0, const expr &in_format1, const expr &ori_in_format0, - const expr &ori_in_format1, const expr &out_size, const expr &kernel, - const expr &impl = get_ir_null()); -expr call_unary_fusible_op_query_format( - const expr &tb, const expr &out0, const expr &in0, const expr &out_foramt0, - const expr &in_format0, const expr &out_size, const expr &kernel); -expr call_padding_op_query_format(const expr &tb, const expr &out0, - const expr &in0, const expr &out_foramt0, - const expr &in_format0, const expr &out_size, - const expr &kernel); -expr call_pooling_op_query_format(const expr &tb, const expr &out0, - const expr &in0, const expr &out_foramt0, - const expr &in_format0, const expr &out_size, - const expr &kernel); -expr call_binary_fusible_op_query_format( - const expr &tb, const expr &out0, const expr &in0, const expr &in1, - const expr &out_format0, const expr &in_format0, const expr &in_format1, - const expr &out_size, const expr &kernel); -// reorder need to query its impl alg. -expr call_reorder_op_query_format(const expr &tb, const expr &out0, - const expr &in0, const expr &out_foramt0, - const expr &in_format0, const expr &out_size, - const expr &kernel, - const expr &impl = get_ir_null()); -expr call_reduce_op_query_format(const expr &tb, const expr &out0, - const expr &in0, const expr &out_foramt0, - const expr &in_format0, const expr &out_size, - const expr &kernel); -expr call_tensor_view_op_query_format(const expr &tb, const expr &out0, - const expr &in0, const expr &out_format0, - const expr &in_format0, - const expr &out_size, const expr &kernel); -expr call_select_op_query_format(const expr &tb, const expr &out0, - const expr &in0, const expr &in1, - const expr &in2, const expr &out_format0, - const expr &in_format0, const expr &in_format1, - const expr &in_format2, const expr &out_size, - const expr &kernel); -expr call_fused_op_query_combined(const expr &tb, const expr &combined_keys, - const expr &combined_algs, - const expr &each_op_num_key, - const expr &op_num, const expr &kernel); -expr call_cal_blocking_dims(const expr &placeholder, const expr &format); -// Get single config by input shape of matmul. -expr call_get_matmul_dyn_cfg_single(const expr &in, const expr &is_batch); - -// gets the IR func for get_thread_id. @see thread_pool_table::get_thread_id -func_t get_thread_id_func(); - -// gets the IR func for is_in_parallel. @see thread_pool_table::is_in_parallel -func_t get_is_in_parallel_func(); - -// gets the IR func for gc::runtime::enter_barrier -func_t get_barrier_arrive_func(); - -// gets the IR func for gc::runtime::init_barrier -func_t get_init_barrier_func(); - -// gets the IR func for sc_set_idle_func_managed -func_t get_set_idle_func_managed_func(); - -func_t get_tls_amx_buffer_func(); - -func_t get_brgemm_init_func(); - -/** - * Generates the IR to do work-dispatch of balance211 - to parallelize a loop of - * [start,end) with loop iterator step = "step" using "num_threads" threads. - * Given the above parameters and the thread id, this helper function builds the - * IR to calculate the share of the workload in [out_start, out_end) - * - * @param num_threads the number of threads - * @param start the start of the loop - * @param end the end of the loop, not inclusive - * @param step the setp of the loop - * @param tid the current thread id - * @param namer the optional callback function to customize the internal - * variable name. It takes a parameter of the base variable name. A typical - * implementation of this function may transform the input name "a" into a_{N}, - * where N is an internal counter. If is null, use the default namer. - * @param out_start outputs the IR node for the start of the workload of the - * thread - * @param out_len optionally outputs the IR node for the length of the workload - * of the thread. If is null, this function will not output IR for this - * @param out_end optionally outputs the IR node for the end of the workload of - * the thread. If is null, this function will not output IR for this - * @param out_seq If it is not null, append the generated statements into it. If - * is is null, generate the statements to the current IR builder. - * @return the maximal number of splits of the threads, which threads can be - * grouped into, and each group of threads does the same amount of jobs. The - * return value should be a factor of num_threads. e.g. If the number of threads - * is 18 and the function returns 6, it means that the 18 threads can be further - * grouped into sub-groups with 2, 3, or 6 threads. The function can return 0, - * indicating that the loop boundary are not constants. - * */ -uint64_t generate_balance211( - int num_threads, const expr &start, const expr &end, const expr &step, - const expr &tid, const std::function &namer, - expr *out_start, expr *out_len = nullptr, expr *out_end = nullptr, - std::vector *out_seq = nullptr); - -} // namespace builtin -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/content_hash.cpp b/legacy/core/src/compiler/ir/content_hash.cpp deleted file mode 100644 index ed389b758..000000000 --- a/legacy/core/src/compiler/ir/content_hash.cpp +++ /dev/null @@ -1,85 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "content_hash.hpp" -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -std::size_t content_hash_t::operator()(const constant_c &k) const { - std::size_t ret = static_cast(k->dtype_); - if (k->dtype_.is_pointer()) { - for (auto &v : k->value_) { - ret = ret * 23 + v.s64; - } - return ret; - } - switch (get_etype_category(k->dtype_)) { - case CATE_FLOAT: - for (auto &v : k->value_) { - ret = ret * 23 + std::hash()(v.f32); - } - break; - case CATE_INT: - case CATE_UINT: - for (auto &v : k->value_) { - ret = ret * 23 + v.s64; - } - break; - default: - break; - } - return ret; -} - -std::size_t content_hash_t::operator()(const expr &k) const { - return content_hash_t()(k); -} - -std::size_t content_hash_t::operator()(const expr_c &k) const { - std::size_t ret; - switch (k->node_type_) { - case sc_expr_type::constant: - ret = content_hash_t()(k.static_as()); - break; - case sc_expr_type::tensor: - ret = std::hash()(k); - break; - default: - throw std::runtime_error("Unsupported node type!"); - break; - } - return ret; -} - -bool content_equals_t::operator()(const expr_c &a, - const expr_c &b) const { -#if !SC_GNUC_VERSION_LT(7) && !defined(_MSC_VER) - // use cached ir_comparer because it is a complex class - // we will auto-reset after compare, so the cmper_ is unchanged after this - // function call, as if it is "const" - return const_cast(cmper_).compare(a, b); -#else - ir_comparer cmper_; - return cmper_.compare(a, b); -#endif -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/content_hash.hpp b/legacy/core/src/compiler/ir/content_hash.hpp deleted file mode 100644 index f905a392e..000000000 --- a/legacy/core/src/compiler/ir/content_hash.hpp +++ /dev/null @@ -1,91 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_CONTENT_HASH_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_CONTENT_HASH_HPP - -#include "ir_comparer.hpp" -#include "sc_expr.hpp" -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -template struct content_hash_t {}; - -template <> struct content_hash_t { - std::size_t operator()(const constant_c &k) const; -}; - -template <> struct content_hash_t { - std::size_t operator()(const expr &k) const; -}; - -template <> struct content_hash_t { - std::size_t operator()(const expr_c &k) const; -}; - -template struct content_hash_t> { - std::size_t operator()(const std::vector &k) const { - auto h = content_hash_t(); - std::size_t ret = 0; - for (auto &v : k) { - ret = ret * 23 + h(v); - } - return ret; - } -}; - -template struct content_equals_t {}; - -template <> struct content_equals_t { -#if !SC_GNUC_VERSION_LT(7) && !defined(_MSC_VER) - // Old version of gcc will produce error upon content_hash_map construction - ir_comparer cmper_; -#endif - bool operator()(const expr_c &a, const expr_c &b) const; -}; - -template -struct content_equals_t> - : content_equals_t> {}; - -template struct content_equals_t> { - bool operator()(const std::vector &a, const std::vector &b) const { - content_equals_t eq; - if (a.size() != b.size()) { - return false; - } - for (unsigned i = 0; i < a.size(); i++) { - if (!eq(a[i], b[i])) - return false; - } - return true; - } -}; - -template -using content_hash_map = - std::unordered_map, content_equals_t>; -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/easy_build.cpp b/legacy/core/src/compiler/ir/easy_build.cpp deleted file mode 100644 index 211829234..000000000 --- a/legacy/core/src/compiler/ir/easy_build.cpp +++ /dev/null @@ -1,138 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "easy_build.hpp" -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace builder { -for_range_simulator_t range(const std::string &name, for_loop &out, expr min, - expr extent, expr step, for_type type, - int num_threads) { - return for_range_simulator_t(builder::get_current_builder(), &out, name, - std::move(min), std::move(extent), - std::move(step), type, num_threads); -} - -for_range_simulator_t range_nobind(const std::string &name, expr min, - expr extent, expr step, for_type type, - int num_threads) { - return for_range_simulator_t(builder::get_current_builder(), nullptr, name, - std::move(min), std::move(extent), - std::move(step), type, num_threads); -} - -for_range_simulator_t range(for_loop &out, expr min, expr extent, expr step, - for_type type, int num_threads) { - return for_range_simulator_t(builder::get_current_builder(), &out, - "!!!unamed", std::move(min), std::move(extent), - std::move(step), type, num_threads); -} - -for_range_simulator_t range(expr min, expr extent, expr step, for_type type, - int num_threads) { - return for_range_simulator_t(builder::get_current_builder(), nullptr, - "!!!unamed", std::move(min), std::move(extent), - std::move(step), type, num_threads); -} - -func_simulator_t _make_func_simulator(const std::string &name, func_t *outfunc, - sc_data_type_t dtype, - std::vector> &&args) { - std::vector flattened; - for (auto &a : args) { - for (auto &arg : a) { - flattened.emplace_back(std::move(arg)); - } - } - return func_simulator_t(name, outfunc, dtype, std::move(flattened)); -} - -std::vector _make_arg(const char *name, sc_data_type_t dtype, - const std::vector &args) { - expr ret; - if (args.empty()) { - ret = builder::make_var(dtype, name); - } else { - std::vector dims; - dims.reserve(args.size()); - for (auto i : args) { - dims.emplace_back(i); - } - ret = builder::make_tensor(name, dims, dtype); - } - return std::vector{ret}; -} - -std::vector -_make_arg(const char *name, sc_data_type_t dtype, - std::initializer_list args) { // NOLINT, - // We must use unsigned long here to let g++ and MSVC to correctly let UL - // number literals find correct overload version of function. - expr ret; - if (args.size() == 0) { - ret = builder::make_var(dtype, name); - } else { - std::vector dims; - dims.reserve(args.size()); - for (auto i : args) { - dims.emplace_back(i); - } - ret = builder::make_tensor(name, dims, dtype); - } - return std::vector{ret}; -} - -std::vector _make_arg(const char *name, sc_data_type_t dtype, - const std::vector &args) { - expr ret; - if (args.empty()) { - ret = builder::make_var(dtype, name); - } else { - ret = builder::make_tensor(name, args, dtype); - } - return std::vector{ret}; -} - -std::vector _make_arg(const char *name, sc_data_type_t dtype, - std::initializer_list args) { - return _make_arg(name, dtype, std::vector(args)); -} - -std::vector _make_arg(const char *name, sc_data_type_t dtype) { - return std::vector{builder::make_var(dtype, name)}; -} - -func_t _decl_func(const std::string &name, sc_data_type_t dtype, - std::vector> &&args) { - std::vector flattened; - for (auto &a : args) { - for (auto &arg : a) { - flattened.emplace_back(std::move(arg)); - } - } - return builder::make_func(name, flattened, stmt(), dtype); -} - -} // namespace builder -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/easy_build.hpp b/legacy/core/src/compiler/ir/easy_build.hpp deleted file mode 100644 index cbf756587..000000000 --- a/legacy/core/src/compiler/ir/easy_build.hpp +++ /dev/null @@ -1,598 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_EASY_BUILD_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_EASY_BUILD_HPP - -#include "builder.hpp" -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -struct context_t; - -// the assignment overload for utils::bind_vector_to_args -namespace utils { -template <> struct SC_INTERNAL_API bind_assigner_t { - static void assign(expr::lvalue_proxy_t &dst, const expr &src) { - dst.data_ = src; - dst.require_remake_ = false; - } -}; -} // namespace utils - -namespace builder { -struct SC_INTERNAL_API scope_mgr_t { - builder::builder_impl_t *ctx_; - using callback_type = std::function; - callback_type on_pop_; - scope_mgr_t(builder::builder_impl_t *ctx, callback_type on_pop) - : ctx_(ctx), on_pop_(std::move(on_pop)) { - ctx->push_scope(); - } - scope_mgr_t(scope_mgr_t &&other) - : ctx_(other.ctx_), on_pop_(std::move(other.on_pop_)) { - other.ctx_ = nullptr; - } - ~scope_mgr_t() { - if (ctx_) { - on_pop_(ctx_, ctx_->pop_scope()); - } - } -}; - -/** - * This class builds a for-loop node with RAII of C++. - * It provides an iterator which can only iterate once. - * Users should use range-based-for to iterate on this iterator - * The iterator returns the "var expr" of the for-loop to generate - * e.g. - * for (auto i: range(0, 100, "i")) { - * buf[i] = buf[i] + 1; - * } - * - * At the end of the scope, this class will push a for-loop node in current - * build - * */ -struct SC_INTERNAL_API for_range_simulator_t { - expr var_; - builder::builder_impl_t *ctx_; - expr min_; - expr extent_; - expr step_; - for_type type_; - for_loop *out_; - int num_threads_; - struct for_range_iterator_t { - expr var_; - bool consumed_; - expr::lvalue_proxy_t operator*() const { - return expr::lvalue_proxy_t(var_, false); - } - - for_range_iterator_t &operator++() { - consumed_ = true; - return *this; - } - - bool operator!=(for_range_iterator_t &other) const { - return consumed_ != other.consumed_; - } - - for_range_iterator_t(expr var) : var_(std::move(var)), consumed_(false) {} - for_range_iterator_t() : consumed_(true) {} - }; - - for_range_iterator_t begin() const { return for_range_iterator_t(var_); } - - for_range_iterator_t end() { return for_range_iterator_t(); } - - for_range_simulator_t(builder::builder_impl_t *ctx, for_loop *out, - const std::string &name, expr min, expr extent, - expr step, for_type type, int num_threads) - : var_(builder::make_var(datatypes::index, name)), ctx_(ctx), - min_(std::move(min)), extent_(std::move(extent)), - step_(std::move(step)), type_(type), out_(out), - num_threads_(num_threads) { - ctx->push_scope(); - } - - for_range_simulator_t(const for_range_simulator_t &other) = delete; - for_range_simulator_t(for_range_simulator_t &&other) - : var_(std::move(other.var_)), ctx_(other.ctx_), - min_(std::move(other.min_)), extent_(std::move(other.extent_)), - step_(std::move(other.step_)), type_(other.type_), out_(other.out_), - num_threads_(other.num_threads_) {} - - ~for_range_simulator_t() { - if (!var_.defined()) - return; - auto bb = ctx_->pop_scope(); - auto st = ctx_->push_for_loop(var_, min_, extent_, step_, bb, true, type_, - num_threads_); - if (out_) { - *out_ = st.checked_as(); - } - } -}; - -SC_INTERNAL_API for_range_simulator_t range(const std::string &name, - for_loop &out, expr min, - expr extent, expr step = expr(1), - for_type type = for_type::NORMAL, - int num_threads = 0); -SC_INTERNAL_API for_range_simulator_t range_nobind( - const std::string &name, expr min, expr extent, expr step = expr(1), - for_type type = for_type::NORMAL, int num_threads = 0); -SC_INTERNAL_API for_range_simulator_t range(for_loop &out, expr min, - expr extent, expr step = expr(1), - for_type type = for_type::NORMAL, - int num_threads = 0); -SC_INTERNAL_API for_range_simulator_t range(expr min, expr extent, - expr step = expr(1), - for_type type = for_type::NORMAL, - int num_threads = 0); - -/** - * Builds a for-loop. Takes arguments: - * the loop var (will create a C++ variable and a var node of the same name) - * iter_begin - * iter_end - * step (optional) - * loop type (optional) - * See range_nobind(). e.g. - * _for_(i, 0, 100) {...} - * */ -#define _for_(IDX, ...) \ - for (auto IDX : \ - ::dnnl::impl::graph::gc::builder::range_nobind(#IDX, __VA_ARGS__)) - -/** - * Builds a for-loop and returns the for-loop node to an output variable. - * Takes arguments: - * output variable: the output for_loop C++ variable. The variable will be - * set after the scope of the for loop - * the loop var (will create a C++ variable and a var node of the same name) - * iter_begin - * iter_end - * step (optional) - * loop type (optional) - * See range(). e.g. - * for_loop li; - * _named_for_(li, i, 0, 100) {...} - * */ -#define _named_for_(OUT, IDX, ...) \ - for (auto IDX : \ - ::dnnl::impl::graph::gc::builder::range(#IDX, OUT, __VA_ARGS__)) - -struct SC_INTERNAL_API nested_for_ranges_t { - std::vector loops_; - unsigned cur_var_ = 0; - expr get_var() { - assert(cur_var_ < loops_.size()); - return loops_[cur_var_++].var_; - } - bool consumed_; - nested_for_ranges_t(nested_for_ranges_t &other) = delete; - nested_for_ranges_t(std::vector &&loops) - : loops_(std::move(loops)), consumed_(false) {} - nested_for_ranges_t(nested_for_ranges_t &&other) - : loops_(std::move(other.loops_)), consumed_(other.consumed_) {} - - template - nested_for_ranges_t(Args... args) : consumed_(false) { - utils::args_to_vector(loops_, std::move(args)...); - } - - void step() { consumed_ = true; } - ~nested_for_ranges_t() { - // destruct inner loop first - while (!loops_.empty()) - loops_.pop_back(); - } - std::vector get_vars() { - std::vector ret; - for (auto &l : loops_) { - ret.emplace_back(l.var_); - } - return ret; - } -}; - -/** - * Builds nested for loops - * params: the for_range_simulator_t(s) of the loops. Create - * for_range_simulator_t by range()/range_nobind() functions - * */ -#define _nested_for_(...) \ - for (auto _0_nested_for = \ - ::dnnl::impl::graph::gc::builder::nested_for_ranges_t(__VA_ARGS__); \ - !_0_nested_for.consumed_; _0_nested_for.step()) - -/** - * Binds a loop-variable in the nested for to a expr variable in C++ - * Calling this macro several times will bind the loop variable from - * the outer to the inner. Can only be used in a nested-for - * Also sets the name of the loop-variable. - * */ -#define _iter_var_(v) \ - expr v = _0_nested_for.get_var(); \ - (v).checked_as()->name_ = #v - -/** - * This class builds a if-then-else node with RAII of C++. - * if_simulator_t will generate an iterator with "block_num" inside. - * Calling "++" on if_iterator_t will increase "block_num" by one. - * Using "*" on if_iterator_t will generate a std::pair, where - * the "scope_mgr_t" is the scope guard. When the "then/else" scope is done, - * the "scope_mgr_t" will register the generated basic_block_t to - * if_simulator_t.{true_block, false_block}. The second element of - "*if_iterator_t" - * is the current "block_num", which can help decide whether we are in - * "then block" or "else block". When range-based-for scope is done, - * if_simulator_t will be destructed and make and if-else node with registerd - * true/false blocks. The "_if_" macro wraps the underlying range-based-for. - * It expands like: - { - if_simulator_t _simu = if_simulator_t(builder::get_current(), cond); - if_iterator_t itr = _simu.begin() - for (;itr != _simu.end(); itr++) { - std::pair _scope = *itr; - if (scope.second == 0) { - // true block - } else { - // false block - } - // _scope is destoryed here. Will register "true block" or "false block" - // in _simu - } - // _simu is destoryed here. The if-then-else is generated - } -*/ -struct SC_INTERNAL_API if_simulator_t { - stmt true_block_; - stmt false_block_; - builder::builder_impl_t *ctx_; - expr cond_; - - struct if_iterator_t { - int block_num_; - if_simulator_t *if_scope_; - std::pair operator*() { - return std::make_pair( - scope_mgr_t(if_scope_->ctx_, - [this](builder::builder_impl_t *ctx, const stmt &s) { - if (block_num_ == 0) { - if_scope_->true_block_ = s; - } else { - if_scope_->false_block_ = s; - } - }), - block_num_); - } - - if_iterator_t &operator++() { - block_num_++; - return *this; - } - - bool operator!=(if_iterator_t &other) const { - return block_num_ != other.block_num_; - } - - if_iterator_t(if_simulator_t *if_scope) - : block_num_(0), if_scope_(if_scope) {} - if_iterator_t() : block_num_(2), if_scope_(nullptr) {} - }; - - if_iterator_t begin() { return if_iterator_t(this); } - - if_iterator_t end() { return if_iterator_t(); } - - if_simulator_t(builder::builder_impl_t *ctx, expr cond) - : ctx_(ctx), cond_(std::move(cond)) {} - - if_simulator_t(if_iterator_t &other) = delete; - ~if_simulator_t() { - if (!true_block_.defined() || !false_block_.defined()) { - SC_WARN << "Cannot generate if statements due to undefined " - "true_block/false_block for if_simulator, could be " - "caused by early destruction from assertion failure"; - return; - } - stmt false_block = - false_block_.checked_as()->seq_.empty() ? stmt() : false_block_; - builder::get_current_builder()->push_if_else(cond_, true_block_, - false_block); - } -}; - -/** - * Builds an if_else node. Takes the parameter of the condition - * as an expr - * */ -#define _if_(...) \ - for (auto &&__if_scope__ : ::dnnl::impl::graph::gc::builder::if_simulator_t( \ - ::dnnl::impl::graph::gc::builder::get_current_builder(), \ - (__VA_ARGS__))) \ - if (__if_scope__.second == 0) -#define _else_ else - -struct SC_INTERNAL_API func_simulator_t { - operator bool() const { return true; } - std::vector vargs_; - func_t *outfunc_; - std::string name_; - sc_data_type_t dtype_; - func_simulator_t(func_simulator_t &&other) - : vargs_(std::move(other.vargs_)), outfunc_(other.outfunc_), - name_(std::move(other.name_)), dtype_(other.dtype_) { - other.outfunc_ = nullptr; - } - - func_simulator_t(const std::string &name, func_t *outfunc, - sc_data_type_t dtype, std::vector &&vargs) - : vargs_(std::move(vargs)), outfunc_(outfunc), name_(name), - dtype_(dtype) { - get_current_builder()->push_scope(); - } - ~func_simulator_t() { - if (!outfunc_) - return; - auto bb = get_current_builder()->pop_scope(); - *outfunc_ = make_func(name_, vargs_, bb, dtype_); - } -}; - -SC_INTERNAL_API func_simulator_t _make_func_simulator( - const std::string &name, func_t *outfunc, sc_data_type_t dtype, - std::vector> &&args); - -SC_INTERNAL_API std::vector -_make_arg(const char *name, sc_data_type_t dtype, const std::vector &args); -SC_INTERNAL_API std::vector -_make_arg(const char *name, sc_data_type_t dtype, - std::initializer_list args); // NOLINT, -// We must use unsigned long here to let g++ and MSVC to correctly let UL number -// literals find correct overload version of function. -SC_INTERNAL_API std::vector _make_arg(const char *name, - sc_data_type_t dtype, - const std::vector &args); - -SC_INTERNAL_API std::vector _make_arg(const char *name, - sc_data_type_t dtype, - std::initializer_list args); - -SC_INTERNAL_API std::vector _make_arg(const char *name, - sc_data_type_t dtype); - -SC_INTERNAL_API func_t _decl_func(const std::string &name, sc_data_type_t dtype, - std::vector> &&args); - -/** - * Defines a function node named "NAME" and create a C++ variable of func with - * the same name. arguments: - * DTYPE: the return type - * NAME: the function name (and the func variable name in C++) - * the argument definitions: see _arg_ below - * e.g. _function_(datatypes::s32, funcA, - * _arg_("buffer", datatypes::f32, {100,200})) { - * ... - * } - * */ -#define _function_(DTYPE, NAME, ...) \ - ::dnnl::impl::graph::gc::func_t NAME; \ - if (auto _0_func__ = ::dnnl::impl::graph::gc::builder::_make_func_simulator( \ - #NAME, &NAME, DTYPE, \ - std::vector>{ \ - __VA_ARGS__})) -/** - * Declares a function node named "NAME" and create a C++ variable of func with - * the same name. This macro will not define a function. It declares an extern - * function. - * arguments: - * DTYPE: the return type - * NAME: the function name (and the func variable name in C++) - * the argument definitions: see _arg_ below - * */ -#define _decl_func_(DTYPE, NAME, ...) \ - ::dnnl::impl::graph::gc::func_t NAME = \ - ::dnnl::impl::graph::gc::builder::_decl_func(#NAME, DTYPE, \ - {__VA_ARGS__}) - -/** - * Declares an argument of function node. - * arguments: - * name: the name of the argument - * dtype: the sc_data_type_t of the argument - * dimemsions: the integer dimensions of the argument. Wrapped by {}. - * Can be empty {} or omitted, meaning this argument is a scalar. - * e.g. - * _arg_("len", datatypes::s32) // a scalar arg of s32 - * _arg_("buffer", datatypes::f32, {100,200}) // a tensor arg of f32 - * */ -#define _arg_(...) ::dnnl::impl::graph::gc::builder::_make_arg(__VA_ARGS__) - -/** - * An std::vector of arguments to the function node. - * e.g. - * std::vector args = {make_var(...)}; - * _varg_(args) - * */ -#define _varg_(...) (__VA_ARGS__) - -/** - * Binds all arguments in the function definition to C++ expr variables - * e.g. _bind_(a,b,c); - * */ -#define _bind_(...) \ - ::dnnl::impl::graph::gc::expr::lvalue_proxy_t __VA_ARGS__; \ - ::dnnl::impl::graph::gc::utils::bind_vector_to_args<0>(_0_func__.vargs_, \ - __VA_ARGS__); - -/** - * Defines a variable within the current scope - * arguments: - * NAME: the name of the variable, should not be quoted - * DTYPE: sc_data_type_t of the variable - * */ -#define _var_(NAME, DTYPE) \ - ::dnnl::impl::graph::gc::expr::lvalue_proxy_t NAME( \ - ::dnnl::impl::graph::gc::builder::make_var(DTYPE, #NAME), false); \ - ::dnnl::impl::graph::gc::builder::get_current_builder() \ - ->push_var_tensor_def(NAME, linkage::local); - -/** - * Defines a variable within the current scope with specified name - * arguments: - * NAME: the name of the variable, should not be quoted - * VAR_NAME: the name of the tir variable, should be quoted - * DTYPE: sc_data_type_t of the variable - * */ -#define _named_var_(NAME, VAR_NAME, DTYPE) \ - ::dnnl::impl::graph::gc::expr::lvalue_proxy_t NAME( \ - ::dnnl::impl::graph::gc::builder::make_var(DTYPE, VAR_NAME), false); \ - ::dnnl::impl::graph::gc::builder::get_current_builder() \ - ->push_var_tensor_def(NAME, linkage::local); - -/** - * Defines a variable within the current scope - * arguments: - * NAME: the name of the variable, should not be quoted - * DTYPE: sc_data_type_t of the variable - * LINKAGE: the linkage - * INIT: the initial value - * */ -#define _var_ex_(NAME, DTYPE, ...) \ - ::dnnl::impl::graph::gc::expr::lvalue_proxy_t NAME( \ - ::dnnl::impl::graph::gc::builder::make_var(DTYPE, #NAME), false); \ - ::dnnl::impl::graph::gc::builder::get_current_builder() \ - ->push_var_tensor_def(NAME, __VA_ARGS__); - -/** - * Defines a variable within the current scope with init value - * arguments: - * NAME: the name of the variable, should not be quoted - * DTYPE: sc_data_type_t of the variable - * INIT: the initial value - * */ -#define _var_init_(NAME, DTYPE, INIT) \ - _var_ex_(NAME, DTYPE, ::dnnl::impl::graph::gc::linkage::local, INIT) - -#define _var_init_copy_(NAME, DTYPE, INIT) \ - _var_init_(NAME, DTYPE, INIT); \ - NAME##_ = NAME; - -/** - * Defines a private linkage global variable within the current scope - * arguments: - * MODULE: the ir_module - * NAME: the name of the variable, should not be quoted - * DTYPE: sc_data_type_t of the variable - * INIT: the initial value - * */ -#define _module_var_(MODULE, NAME, DTYPE, INIT) \ - ::dnnl::impl::graph::gc::expr::lvalue_proxy_t NAME( \ - (MODULE)->make_global_var(DTYPE, #NAME, linkage::private_global, INIT), \ - false); - -/** - * Defines a public linkage global variable within the current scope - * arguments: - * MODULE: the ir_module - * NAME: the name of the variable, should not be quoted - * DTYPE: sc_data_type_t of the variable - * INIT: the initial value - * */ -#define _global_var_(MODULE, NAME, DTYPE, INIT) \ - ::dnnl::impl::graph::gc::expr::lvalue_proxy_t NAME( \ - (MODULE)->make_global_var(DTYPE, #NAME, linkage::public_global, INIT), \ - false); - -/** - * Defines a private linkage global tensor within the current scope - * arguments: - * MODULE: the ir_module - * NAME: the name of the tensor, should not be quoted - * DTYPE: sc_data_type_t of the tensor - * INIT: the initial value - * */ -#define _module_tensor_(MODULE, NAME, DTYPE, ...) \ - ::dnnl::impl::graph::gc::expr::lvalue_proxy_t NAME( \ - MODULE->make_global_tensor(DTYPE, #NAME, {__VA_ARGS__}, \ - linkage::private_global), \ - false); - -/** - * Defines a public linkage global tensor within the current scope - * arguments: - * MODULE: the ir_module - * NAME: the name of the tensor, should not be quoted - * DTYPE: sc_data_type_t of the tensor - * INIT: the initial value - * */ -#define _global_tensor_(MODULE, NAME, DTYPE, ...) \ - ::dnnl::impl::graph::gc::expr::lvalue_proxy_t NAME( \ - MODULE->make_global_tensor(DTYPE, #NAME, {__VA_ARGS__}, \ - linkage::public_global), \ - false); - -/** - * Defines a tensor within the current scope - * arguments: - * NAME: the name of the tensor, should not be quoted - * DTYPE: sc_data_type_t of the tensor - * dimemsions: the dimemsions - * e.g. _tensor_(buffer, datatypes::f32, 100, 200) - * */ -#define _tensor_(NAME, DTYPE, ...) \ - ::dnnl::impl::graph::gc::expr::lvalue_proxy_t NAME( \ - ::dnnl::impl::graph::gc::builder::make_tensor( \ - #NAME, std::vector{__VA_ARGS__}, DTYPE), \ - false); \ - ::dnnl::impl::graph::gc::builder::get_current_builder() \ - ->push_var_tensor_def(NAME, linkage::local); - -/** - * Creates and reserve a function call - * */ -#define _evaluate_call_(NAME, ...) \ - ::dnnl::impl::graph::gc::builder::get_current_builder()->push_evaluate( \ - ::dnnl::impl::graph::gc::builder::make_call( \ - NAME, std::vector<::dnnl::impl::graph::gc::expr>{__VA_ARGS__})); - -/** - * Creates a returns statement - * */ -#define _return_(...) \ - ::dnnl::impl::graph::gc::builder::get_current_builder()->push_returns( \ - __VA_ARGS__) - -} // namespace builder -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/function_pass.hpp b/legacy/core/src/compiler/ir/function_pass.hpp deleted file mode 100644 index 08b55bbf2..000000000 --- a/legacy/core/src/compiler/ir/function_pass.hpp +++ /dev/null @@ -1,51 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_FUNCTION_PASS_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_FUNCTION_PASS_HPP - -#include "pass_info_macros.hpp" -#include "sc_function.hpp" -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -struct tir_pass_dependency_t; - -/** - * The base abstruct class of all function passes - * */ -class function_pass_t { -public: - virtual func_c operator()(func_c f) = 0; - virtual ~function_pass_t() = default; - virtual const char *get_name() const { return nullptr; } -#ifndef NDEBUG - virtual void get_dependency_info(tir_pass_dependency_t &out) const; -#endif -}; - -using function_pass_ptr = std::unique_ptr; -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/graph/analysis/analysis.hpp b/legacy/core/src/compiler/ir/graph/analysis/analysis.hpp deleted file mode 100644 index 9dac108bc..000000000 --- a/legacy/core/src/compiler/ir/graph/analysis/analysis.hpp +++ /dev/null @@ -1,35 +0,0 @@ -/******************************************************************************* - * Copyright 2021-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_ANALYSIS_ANALYSIS_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_ANALYSIS_ANALYSIS_HPP - -#include "../graph.hpp" - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -// check whether open quantized optimzation passes. -void analysis_quantized(sc_graph_t &graph, - const context_ptr &ctx = get_default_context()); -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/graph/analysis/analysis_quantized.cpp b/legacy/core/src/compiler/ir/graph/analysis/analysis_quantized.cpp deleted file mode 100644 index 1db50b920..000000000 --- a/legacy/core/src/compiler/ir/graph/analysis/analysis_quantized.cpp +++ /dev/null @@ -1,35 +0,0 @@ -/******************************************************************************* - * Copyright 2021-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "../traits.hpp" -#include "analysis.hpp" - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -void analysis_quantized(sc_graph_t &graph, const context_ptr &ctx) { - for (auto &op : graph.ops_) { - if (op->op_name_.find("quantize") != std::string::npos) { - graph.attrs_[sc_graph_t::attr_key_t::quantize] = true; - break; - } - } -} -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/anchor_loop_generator.cpp b/legacy/core/src/compiler/ir/graph/anchor_loop_generator.cpp deleted file mode 100644 index 09fb4447e..000000000 --- a/legacy/core/src/compiler/ir/graph/anchor_loop_generator.cpp +++ /dev/null @@ -1,407 +0,0 @@ -/* - * Copyright (C) 2025 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions - * and limitations under the License. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "anchor_loop_generator.hpp" -#include "fusible_op_utils.hpp" -#include "fusion_anchor.hpp" -#include "mixed_partition.hpp" -#include "utils.hpp" -#include "visitor.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -SC_MODULE(graph.outer_loop_gen); - -for_loop get_next_inner_loop(const for_loop &cur_loop) { - if (cur_loop->body_.isa()) { - return cur_loop->body_.checked_as(); - } else if (cur_loop->body_.isa() && - cur_loop->body_.checked_as()->seq_.size() == 1 && - cur_loop->body_.checked_as()->seq_[0].isa()) { - return cur_loop->body_.checked_as()->seq_[0].checked_as(); - } - return for_loop(); -} - -static bool axis_can_be_sort(sc_graph_t &graph) { - return is_optimized_sub_graph(graph) && - std::all_of( - graph.ops_.begin(), graph.ops_.end(), [](const sc_op_ptr &op) { - return (!op->isa() && - !op->isa()) || - op->attrs_.get_or_else(op_attr_key::no_fuse, false); - }); -} - -typedef std::vector (*loop_sort_rule_func)(const context_ptr &, - const std::vector &, - sc_graph_t &, - const graph_tensor_ptr &); - -/** - * Move loop axis of reduce axis to inner. - * - * E.g. loop axis is {0, 1, 2, 3}, rd_axis is {1, 2}, after func, we get loop - * axis {0, 3, 1, 2} - * */ -static std::vector -move_reduce_axis_to_inner(const context_ptr &ctx, - const std::vector &in_axis, sc_graph_t &graph, - const graph_tensor_ptr &base_gt) { - if (graph.is_dynamic() || !axis_can_be_sort(graph)) { - return in_axis; - } - auto run_threads = runtime_config_t::get().get_num_threads(); - std::vector out_axis(in_axis.begin(), in_axis.end()); - op_visitor_t vis = op_visitor_t::dfs_topology_sort(graph.ops_.size()); - bool can_move = true; - vis.visit_graph(graph, [&](op_visitor_t *vis, const sc_op_ptr &node) { - std::vector reduce_axis; - if (auto reduce_node = node->dyn_cast()) { - reduce_axis = reduce_node->get_rd_axis(); - } else if (auto reduce_node = node->dyn_cast()) { - reduce_axis = reduce_node->get_rd_axis(); - } else { - return; - } - // if using single core and the reduce op is the only op in this - // partition, don't reorder the loops - // we move the reduce axis for parallelism with a cost of losing - // cache locality. For single core configuration, we don't do this - // moving - if (run_threads == 1 && graph.ops_.size() == 3) { - // 3 ops for input, reduce and output - can_move = false; - return; - } - std::sort(reduce_axis.begin(), reduce_axis.end()); - if (reduce_axis.back() >= static_cast(in_axis.size())) { - can_move = false; - return; - } - for (auto raxis : reduce_axis) { - auto rend = std::remove(out_axis.begin(), out_axis.end(), raxis); - assert(rend + 1 == out_axis.end()); - *rend = raxis; - } - }); - return can_move ? out_axis : in_axis; -} - -/** - * Move last channel axis to outer if pooling format is NXC, in order to acheive - * better loop parallelism - * */ -static std::vector -move_pooling_axis_to_outer(const context_ptr &ctx, - const std::vector &in_axis, sc_graph_t &graph, - const graph_tensor_ptr &base_gt) { - if (graph.is_dynamic() && !axis_can_be_sort(graph)) { - return in_axis; - } - - if (!is_optimized_sub_graph(graph)) - return in_axis; - auto run_threads = runtime_config_t::get().get_num_threads(); - // auto skip - if (run_threads == 1) - return in_axis; - std::vector out_axis(in_axis.begin(), in_axis.end()); - bool use_vectorized = false; - if (std::any_of( - graph.ops_.begin(), graph.ops_.end(), - [&ctx, &run_threads, &use_vectorized](const sc_op_ptr &node) { - if (!node->isa()) - return false; - auto &detail = node->get_inputs()[0]->details_; - // skip if not a plain format - if (detail.get_format().is_blocking()) - return false; - auto shape = detail.get_blocking_dims(); - auto pool = node->dyn_cast(); - auto channel_axis = pool->get_channel_axis(); - COMPILE_ASSERT(channel_axis.size() == 1, "plain format is expected") - // skip if not channel last - if (pool->get_channel_axis()[0] != - (static_cast(shape.size()) - 1)) - return false; - auto &last_dim = shape.back(); - auto vector_lanes = vectorize_step(ctx, detail.dtype_.type_code_); - use_vectorized = - (last_dim / vector_lanes && last_dim % vector_lanes == 0); - if (use_vectorized) { - shape.back() = std::max((int64_t)1, shape.back() / vector_lanes); - } - auto pool_axis = pool->get_real_pooling_axis(); - int parallel_num = 1; - // calculate possible parallelism - for (int i = 0; i < static_cast(shape.size()); i++) { - if (std::find(pool_axis.begin(), pool_axis.end(), i) != - pool_axis.end()) - continue; - parallel_num *= shape[i]; - } - // check parallel_num - return parallel_num >= run_threads; - })) { - // move last channel axis to outer - auto &last_channel_ax = out_axis.back(); - out_axis.insert(out_axis.begin() + 1, last_channel_ax); - if (!use_vectorized) - out_axis.pop_back(); - return out_axis; - } else { - return in_axis; - } -} - -/** - * Satisfy continuous access of input tensor include vectorization on last axis - * and ensure size of each load is more than cache line. - * - * E.g. loop axis = {1, 3, 4, 0, 2} - * - * IF input tensor(origin shape) is f32(32, 4, 16, 8, 16), last axis is 16 - * which fills up a cache line, after func we get loop axis = {1, 3, 0, 2, 4}. - * IF input tensor(origin shape) is f32{32, 4, 16, 8, 8}, after func we get loop - * axis = {1, 0, 2, 3, 4} - * */ -static std::vector continuous_access_satisfaction( - const context_ptr &ctx, const std::vector &in_axis, sc_graph_t &graph, - const graph_tensor_ptr &base_gt) { - if (!axis_can_be_sort(graph)) { - return in_axis; - } - auto base_dims = base_gt->details_.get_blocking_dims_expr(graph); - assert(in_axis.size() == base_dims.size()); - constexpr int cache_line_size = 64; - int fill_up_dim = static_cast(base_dims.size()) - 1; - int dtype_size = utils::get_sizeof_type(base_gt->details_.dtype_); - int cur_load_size = base_dims[fill_up_dim].isa() - ? get_expr_as_int(base_dims[fill_up_dim]) - : 1; - while (fill_up_dim > 0 && cur_load_size * dtype_size < cache_line_size) { - fill_up_dim--; - cur_load_size = - cur_load_size * (base_dims[fill_up_dim].isa() - ? get_expr_as_int(base_dims[fill_up_dim]) - : 1); - } - // input tensor is too small that can not fill up a cache line. - // No need to change loop axis. - if (fill_up_dim == 0) { - if (!graph.is_dynamic()) { - return in_axis; - } else { - fill_up_dim = static_cast(base_dims.size()) - 1; - } - } - std::vector out_axis(in_axis.begin(), in_axis.end()); - for (int i = fill_up_dim; i < static_cast(base_dims.size()); i++) { - auto rend = std::remove(out_axis.begin(), out_axis.end(), i); - *rend = i; - } - return out_axis; -} - -static std::vector loop_sort_rules = { - move_reduce_axis_to_inner, continuous_access_satisfaction, - move_pooling_axis_to_outer}; - -anchor_loop_generator_t::anchor_loop_generator_t( - const graph_tensor_ptr &base_gt) - : body_generator_base_t(nullptr, {}, {}), base_gt_(base_gt) {} - -sc_graph_t &get_owner_graph_from_gt(const graph_tensor_ptr >) { - // search producer firstly - auto op = gt->producer_owner_; - if (!op->owner_graph_) { - // if not found, search in users - for (auto &user : gt->uses_) { - op = user.second.get(); - if (op->owner_graph_) { - break; - } - } - } - COMPILE_ASSERT(op->owner_graph_, "No owner graph found, please check") - return op->get_owner_graph(); -} - -bool anchor_loop_generator_t::create_outer_loop_anchor( - fusion_anchor_mgr_t *fmgr, const context_ptr &ctx) const { - COMPILE_ASSERT(fmgr, "fusion anchor mgr should not be null") - auto &g = get_owner_graph_from_gt(base_gt_); - // query binding axis - query_binding_axis(g); - auto base_dims = base_gt_->details_.get_blocking_dims_expr(g); - auto numdims = base_dims.size(); - assert(numdims > 0); - std::vector loop_vars; - slice_range cur_tsr_slice; - std::vector loop_axis; - loop_vars.reserve(numdims); - cur_tsr_slice.reserve(numdims); - loop_axis.reserve(numdims); - - auto bld = builder::get_current_builder(); - // will create numdims loop vars but uses numdims - 1 because user may sort - // loop axis - for (size_t i = 0; i < numdims; i++) { - bld->push_scope(); - loop_vars.emplace_back(builder::make_var( - datatypes::index, std::string("__itr_") + std::to_string(i))); - // outer loops should have tensor slice of length=1 - cur_tsr_slice.emplace_back(std::make_pair(loop_vars.back(), expr(1))); - loop_axis.push_back(static_cast(i)); - } - // sort loop axis with rules - for (auto &sort_rule : loop_sort_rules) { - loop_axis = sort_rule(ctx, loop_axis, g, base_gt_); - } - - // generate anchors from inner to outer - if (numdims > 1) { - // get ax for last dim - int max_ax = *std::max_element(loop_axis.begin(), loop_axis.end()); - // if duplicated ax found, it means that loop generator needs to split - // loop for last dim vectorization - bool need_split_loop = - (std::count(loop_axis.begin(), loop_axis.end(), max_ax) == 2); - auto lanes = vectorize_step(ctx, base_gt_->details_.dtype_.type_code_); - // set last dim slice range for lanes - if (need_split_loop) { - cur_tsr_slice[max_ax] = std::make_pair(loop_vars[max_ax] * lanes, lanes); - } - for (size_t i = 0; i < numdims - 1; i++) { - // loop num is current dimension index - auto loop_num = loop_axis[numdims - i - 1]; - // upper loop num - auto upper_loop_num = loop_axis[numdims - i - 2]; - // set full tensor range for loop_num dimension - cur_tsr_slice[loop_num] = std::make_pair(0, base_dims[loop_num]); - fmgr->create_fusion_anchor(slice_map{{base_gt_.get(), {cur_tsr_slice}}}); - auto body = bld->pop_scope(); - auto loop = - bld->push_for_loop(loop_vars[upper_loop_num], 0, - (need_split_loop && upper_loop_num == max_ax) - ? do_cast_and_fold(base_dims[max_ax] / lanes) - : base_dims[upper_loop_num], - 1, body, true, for_type::NORMAL); - // bind outer loops with axis hint - bind_loop_axis(base_gt_, loop, upper_loop_num, true); - } - } else { - COMPILE_ASSERT(numdims == 1, "only 1 dims is expected") - if (base_dims[0].isa()) { - auto only_dim = get_const_as_int(base_dims[0].static_as()); - auto lanes = vectorize_step(ctx, base_gt_->details_.dtype_.type_code_); - if ((only_dim % lanes == 0) && (only_dim > lanes)) { - cur_tsr_slice[0].second = expr((int)lanes); - fmgr->create_fusion_anchor( - slice_map{{base_gt_.get(), {cur_tsr_slice}}}); - auto body = bld->pop_scope(); - auto loop = - bld->push_for_loop(loop_vars[0], 0, base_dims[0], expr((int)lanes), - body, true, for_type::NORMAL); - // bind outer loops with axis hint - bind_loop_axis(base_gt_, loop, 0, true); - } - } - } - // create outer-most anchor - cur_tsr_slice[loop_axis[0]] = std::make_pair(0, base_dims[loop_axis[0]]); - fmgr->create_fusion_anchor(slice_map{{base_gt_.get(), {cur_tsr_slice}}}); - return true; -} - -bool anchor_loop_generator_t::create_inner_loop_anchor( - fusion_anchor_mgr_t *fmgr, const fusion_anchor_ptr &parent_fanchor) const { - COMPILE_ASSERT(parent_fanchor, "parent anchor must be set") - // do not support multi-slice - if (parent_fanchor->fsmap_.get(base_gt_).size() != 1) - return false; - auto bld = builder::get_current_builder(); - std::vector loop_vars; - std::vector inner_anchor_axis; - slice_range inner_slice; - // will create numdims loop vars but uses numdims - 1 because user may sort - // loop axis - auto &range = parent_fanchor->fsmap_.get(base_gt_)[0]; - auto is_valid_constant_range = [](const std::pair &range_i) { - return (range_i.second.isa() && - get_expr_as_int(range_i.second) > 1); - }; - size_t valid_range_end = 0; - for (int64_t i = range.size() - 1; i >= 0; --i) { - if (is_valid_constant_range(range[i])) { - valid_range_end = i; - break; - } - } - for (size_t i = 0; i < range.size(); i++) { - if (is_valid_constant_range(range[i]) && (i < valid_range_end)) { - bld->push_scope(); - loop_vars.emplace_back(builder::make_var( - datatypes::index, std::string("__inner_itr_") + std::to_string(i))); - inner_slice.emplace_back(std::make_pair(loop_vars.back(), expr(1))); - inner_anchor_axis.emplace_back(i); - } else { - inner_slice.emplace_back(std::make_pair(expr(0), range[i].second)); - } - } - - auto inner_anchor_num = inner_anchor_axis.size(); - if (!inner_anchor_num) - return false; - // generate anchors from inner to outer - for (int64_t i = static_cast(inner_anchor_num) - 1; i >= 0; i--) { - auto loop_num = inner_anchor_axis[i]; - fmgr->create_fusion_anchor(slice_map{{base_gt_.get(), {inner_slice}}}, - parent_fanchor); - auto body = bld->pop_scope(); - auto loop = bld->push_for_loop(loop_vars[i], 0, range[loop_num].second, 1, - body, true, for_type::NORMAL); - if (i == 0) { - loop->attr()[stmt_attr_key::merge_loop] = true; - // bind outer loops with axis hint - bind_loop_axis(base_gt_, loop, loop_num, true); - } - inner_slice[loop_num] = range[loop_num]; - } - return true; -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/anchor_loop_generator.hpp b/legacy/core/src/compiler/ir/graph/anchor_loop_generator.hpp deleted file mode 100644 index 8f2357b47..000000000 --- a/legacy/core/src/compiler/ir/graph/anchor_loop_generator.hpp +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright (C) 2025 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions - * and limitations under the License. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_ANCHOR_LOOP_GENERATOR_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_ANCHOR_LOOP_GENERATOR_HPP -#include "graph.hpp" -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -struct fusion_anchor_t; - -class anchor_loop_generator_t : public body_generator_base_t { -private: - // decide which one is base graph tensor - graph_tensor_ptr base_gt_; - -public: - anchor_loop_generator_t(const graph_tensor_ptr &base_gt); - config_ptr get_default_config(context_ptr ctx) const override { - return nullptr; - } - - bool generate(context_ptr ctx, const void *config, fusion_anchor_mgr_t *fmgr, - const std::vector &inputs, - const std::vector &outputs, - std::vector &loops) const override { - return false; - }; - - bool create_outer_loop_anchor(fusion_anchor_mgr_t *fmgr, - const context_ptr &ctx) const; - - bool create_inner_loop_anchor( - fusion_anchor_mgr_t *fmgr, - const std::shared_ptr &parent_fanchor) const; - - void schedule_loops(context_ptr ctx, const void *config, stmt body, - std::vector &fors) const override {}; - float get_gflop() const override { return 0; } -}; - -for_loop get_next_inner_loop(const for_loop &cur_loop); - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/graph/binding_axis.cpp b/legacy/core/src/compiler/ir/graph/binding_axis.cpp deleted file mode 100644 index 8a3328080..000000000 --- a/legacy/core/src/compiler/ir/graph/binding_axis.cpp +++ /dev/null @@ -1,368 +0,0 @@ -/******************************************************************************* - * Copyright 2023-2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "binding_axis.hpp" -#include "fusible_op.hpp" -#include "fusible_op_utils.hpp" -#include "graph.hpp" -#include "pass/pass.hpp" -#include "visitor.hpp" -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -// only hash topology of graph, excluding attr and so on -graph_identity get_identity_by_topology(const sc_graph_t &g) { - graph_identity identity; - size_t seed = 0; - op_visitor_t vis = op_visitor_t::bfs_topology_sort(g.ops_.size()); - vis.visit_graph(g, - [&seed, &identity](op_visitor_t *vis, const sc_op_ptr &op) { - for (auto &in : op->get_inputs()) { - hash_combine(seed, in->details_.hash()); - identity.op_lts_.emplace_back(in->details_); - } - for (auto &out : op->get_outputs()) { - hash_combine(seed, out->details_.hash()); - identity.op_lts_.emplace_back(out->details_); - } - hash_combine(seed, op->op_name_); - identity.op_names_.emplace_back(op->op_name_); - }); - identity.hash_ = seed; - return identity; -} - -bool graph_identity::operator==(const graph_identity &other) const { - return hash_ == other.hash_ && op_names_ == other.op_names_ && - op_lts_ == other.op_lts_; -} - -void query_binding_axis(sc_graph_t &g) { - // get hash of graph - auto g_identity = get_identity_by_topology(g); - // get global map cache of graph - auto cache = g.attrs_.get_or_null( - binding_axis_attr::global_map_cache); - // auto skip, when cache is found and dont need to update - if (cache && cache->identity_ == g_identity) { - return; - } - // make shared ptr of global map - auto shared_map_ptr = std::make_shared(); - // global map - auto &gax_map = *shared_map_ptr; - // query function - auto query_based_on_gt = [&gax_map](const graph_tensor_ptr >) { - auto key = (uintptr_t)gt.get(); - // avoid repeated query - if (gax_map.find(key) != gax_map.end()) - return; - // skip any format - if (gt->details_.get_format().is_any()) - return; - // set initial axis - int rank = gt->details_.get_plain_dims().size(); - binding_axis_map bd_ax_map; - binding_axis init_axis; - init_axis.reserve(rank); - for (int i = 0; i < rank; i++) { - init_axis.emplace_back(std::vector{i}); - } - bd_ax_map.get(gt) = init_axis; - - // start node producer recursively call pre-infer binding axis - auto &producer_op = gt->producer_owner_; - if (auto cur = - producer_op->dyn_cast()) { - cur->pre_infer_binding_axis(bd_ax_map); - } - - // start node user recursively call infer binding axis - for (auto &user : gt->uses_) { - if (auto cur = - user.second->dyn_cast()) { - cur->infer_binding_axis(bd_ax_map); - } - } - - // tranform key type - std::unordered_map value_map; - std::transform(bd_ax_map.datamap_.begin(), bd_ax_map.datamap_.end(), - std::inserter(value_map, value_map.end()), - [](const std::pair &kv) { - return std::make_pair((uintptr_t)kv.first, kv.second); - }); - - // insert to global map - gax_map[key] = std::move(value_map); - }; - - // visit all graph tensor - for (auto &op : g.ops_) { - if (op->isa() || op->isa() || op->isa()) - continue; - for (auto &inp : op->get_inputs()) { - query_based_on_gt(inp); - } - for (auto &out : op->get_outputs()) { - query_based_on_gt(out); - } - } - - // attach to graph attr - g.attrs_[binding_axis_attr::global_map_cache] = - global_map_cache{shared_map_ptr, g_identity}; -} - -std::shared_ptr get_binding_axis_map(sc_graph_t &g) { - auto cache = g.attrs_.get_or_null( - binding_axis_attr::global_map_cache); - return cache ? cache->global_map_ptr_ : nullptr; -} - -void bind_loop_axis(const graph_tensor_ptr >, const for_loop &loop, - const std::vector &axis, bool is_block) { - auto &owner_graph = gt->producer_owner_->owner_graph_; - COMPILE_ASSERT(owner_graph, "No owner graph is found") - // get global map ptr - auto global_ptr = get_binding_axis_map(*owner_graph); - COMPILE_ASSERT(global_ptr, "No global mapping ptr found, please check") - // set attr - loop->attr()[binding_axis_attr::loop_hint] = loop_binding_axis_hint{ - global_ptr, (uintptr_t)gt.get(), - is_block ? transform_axis_blocking2plain(gt->details_, axis) : axis}; -} - -void bind_loop_axis(const graph_tensor_ptr >, const stmt &loop, int axis, - bool is_block) { - COMPILE_ASSERT(loop.isa(), - "loop node is expected, but got " << loop) - bind_loop_axis(gt, loop.static_as(), std::vector{axis}, - is_block); -} - -void bind_loop_axis(const graph_tensor_ptr >, - const std::vector &loops, - const std::vector &axis, bool is_block) { - COMPILE_ASSERT(loops.size() == axis.size(), - "axis size should be equal to loop size, but got " - << axis.size() << " VS " << loops.size()) - for (size_t i = 0; i < loops.size(); i++) { - bind_loop_axis(gt, loops[i], axis[i], is_block); - } -} - -std::vector -transform_binding_axis_with_indices(const binding_axis &axis, - const std::vector &indices) { - std::vector ret; - for (auto &ax : indices) { - COMPILE_ASSERT(ax < (int)axis.size(), - "ax exceed size: " << ax << " >= " << axis.size()) - ret.insert(ret.end(), axis[ax].begin(), axis[ax].end()); - } - - // check if empty to make g++12 happy - if (!ret.empty()) { - // sort axis - std::sort(ret.begin(), ret.end()); - // erase possible repeated elements in axis B - ret.erase(std::unique(ret.begin(), ret.end()), ret.end()); - } - return ret; -} - -bool get_aligned_binding_axis(const loop_binding_axis_hint &hint_a, - const loop_binding_axis_hint &hint_b, - std::vector &axis_a, - std::vector &axis_b) { - // reset - axis_a.clear(); - axis_b.clear(); - - COMPILE_ASSERT(hint_a.global_map_ptr_ && hint_b.global_map_ptr_, - "global mapping ptr is null") - // global mapping ptr must be equal, which means they are quried from the - // same graph - if (hint_a.global_map_ptr_ != hint_b.global_map_ptr_) { - return false; - } - - // if two keys are same - if (hint_a.key_ == hint_b.key_) { - axis_a = hint_a.axis_; - axis_b = hint_b.axis_; - return true; - } - // get binding axis of A from hint A - axis_a = hint_a.axis_; - // get global map - auto &global_map = *hint_a.global_map_ptr_; - - // get mapping from view of hint B - if (global_map.find(hint_b.key_) == global_map.end()) { - return false; - } - auto &map_b = global_map[hint_b.key_]; - - // query binding axis of A from hint B - if (map_b.find(hint_a.key_) == map_b.end()) { - return false; - } - auto &value_a = map_b[hint_a.key_]; - // use axis_a as indices to get axis_b from value_a - axis_b = transform_binding_axis_with_indices(value_a, hint_b.axis_); - - return true; -} - -bool get_aligned_binding_axis_twice(const loop_binding_axis_hint &hint_a, - const loop_binding_axis_hint &hint_b, - std::vector &axis_a, - std::vector &axis_b) { - return get_aligned_binding_axis(hint_a, hint_b, axis_a, axis_b) || - get_aligned_binding_axis(hint_b, hint_a, axis_b, axis_a); -} - -bool check_loop_binding_axis(const for_loop_node_t *loop_a, - const for_loop_node_t *loop_b) { - // get hint A and B respectively - auto hint_a = any_map_t::fetch_or_null( - loop_a->attr_.get(), binding_axis_attr::loop_hint); - auto hint_b = any_map_t::fetch_or_null( - loop_b->attr_.get(), binding_axis_attr::loop_hint); - if (!hint_a || !hint_b) - return false; - - std::vector axis_a, axis_b; - if (get_aligned_binding_axis_twice(*hint_a, *hint_b, axis_a, axis_b)) { - return axis_a == axis_b; - } else { - return false; - } -} - -bool check_loop_binding_axis(const for_loop &loop_a, const for_loop &loop_b) { - return check_loop_binding_axis(loop_a.get(), loop_b.get()); -} - -int check_loop_binding_axis(const std::vector &loop_a, - const std::vector &loop_b, - int64_t check_loop_size) { - size_t gcs = std::min(loop_a.size(), loop_b.size()); - if (check_loop_size >= 0) { - gcs = std::min(gcs, (size_t)check_loop_size); - } - int aligned_loop_num = 0; - for (size_t i = 0; i < gcs; i++) { - if (check_loop_binding_axis(loop_a[i], loop_b[i])) { - aligned_loop_num++; - } else { - break; - } - } - return aligned_loop_num; -} - -bool check_loop_has_axis(const for_loop &loop, const graph_tensor_ptr >, - const std::vector &axis) { - // get hint - auto hint = any_map_t::fetch_or_null( - loop->attr_.get(), binding_axis_attr::loop_hint); - if (!hint) - return false; - // get binding axis - auto &self_axis = hint->axis_; - // get global map - COMPILE_ASSERT(hint->global_map_ptr_, "global mapping ptr is null") - auto &global_map = *(hint->global_map_ptr_); - - // get mapping based on self hint key - if (global_map.find(hint->key_) == global_map.end()) - return false; - auto &self_map = global_map[hint->key_]; - - // get target key - auto target_key = (uintptr_t)gt.get(); - // query binding axis - if (self_map.find(target_key) == self_map.end()) - return false; - auto &target_binding_axis = self_map[target_key]; - // get target_axis based on self_axis from view of target graph tensor - std::vector target_axis = - transform_binding_axis_with_indices(target_binding_axis, self_axis); - - // check any element of argument `axis` would appear among target axis - return std::any_of(axis.begin(), axis.end(), [&target_axis](const int &ax) { - return std::find(target_axis.begin(), target_axis.end(), ax) != - target_axis.end(); - }); -} - -void copy_binding_axis_hint(const for_loop_node_t *ths, - for_loop_node_t *other) { - auto ptr = any_map_t::fetch_or_null( - ths->attr_.get(), binding_axis_attr::loop_hint); - if (ptr) { - auto loop_hint = *ptr; - other->attr().set(binding_axis_attr::loop_hint, loop_hint); - } -} - -void fuse_binding_axis_hint(const for_loop_node_t *ths, - const for_loop_node_t *other) { - auto ths_hint_ptr = any_map_t::fetch_or_null( - ths->attr_.get(), binding_axis_attr::loop_hint); - auto other_hint_ptr = any_map_t::fetch_or_null( - other->attr_.get(), binding_axis_attr::loop_hint); - if (!ths_hint_ptr || !other_hint_ptr) - return; - - loop_binding_axis_hint hint_ths = *ths_hint_ptr, hint_other = *other_hint_ptr; - - auto merge_axis = [](const std::vector &axis_a, - const std::vector &axis_b) { - auto ret = axis_a; - ret.insert(ret.end(), axis_b.begin(), axis_b.end()); - // check if empty to make g++12 happy - if (!ret.empty()) { - std::sort(ret.begin(), ret.end()); - ret.erase(std::unique(ret.begin(), ret.end()), ret.end()); - } - return ret; - }; - - std::vector axis_ths, axis_other; - if (get_aligned_binding_axis(hint_ths, hint_other, axis_ths, axis_other)) { - auto merged_axis = merge_axis(axis_ths, axis_other); - hint_ths.axis_ = merged_axis; - } else if (get_aligned_binding_axis(hint_other, hint_ths, axis_other, - axis_ths)) { - auto merged_axis = merge_axis(axis_ths, axis_other); - hint_other.axis_ = merged_axis; - } -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/binding_axis.hpp b/legacy/core/src/compiler/ir/graph/binding_axis.hpp deleted file mode 100644 index 586a80a8f..000000000 --- a/legacy/core/src/compiler/ir/graph/binding_axis.hpp +++ /dev/null @@ -1,114 +0,0 @@ -/******************************************************************************* - * Copyright 2023-2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_BINDING_AXIS_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_BINDING_AXIS_HPP - -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -namespace binding_axis_attr { -// global binding axis map cache, attached on the graph -constexpr const char *global_map_cache = "global_map_cache"; -// loop binding axis hint, attached on the for loop node -constexpr const char *loop_hint = "loop_hint"; -}; // namespace binding_axis_attr - -class sc_graph_t; -struct graph_tensor; -struct logical_tensor_t; - -using binding_axis = std::vector>; -template struct gt_map_t; -using binding_axis_map = gt_map_t; - -using global_binding_axis_map = - std::unordered_map>; - -// The identity based on the topology of graph -struct graph_identity { - std::vector op_names_; - std::vector op_lts_; - size_t hash_; - // compare graph identity - bool operator==(const graph_identity &other) const; -}; - -struct global_map_cache { - // global map ptr - std::shared_ptr global_map_ptr_; - // ensure consistency of global mapping ptr - graph_identity identity_; -}; - -struct loop_binding_axis_hint { - // global map ptr - std::shared_ptr global_map_ptr_; - // key: the address of graph tensor that generate loops - uintptr_t key_; - // axis of graph tensor's plain dims - std::vector axis_; -}; - -// query binding axis based on give graph and set result on the attr of graph -void query_binding_axis(sc_graph_t &g); - -// the standart binding axis interface -void bind_loop_axis(const std::shared_ptr >, - const for_loop &loop, const std::vector &axis, - bool is_block = false); -// friendly to development -void bind_loop_axis(const std::shared_ptr >, const stmt &loop, - int axis, bool is_block = false); -void bind_loop_axis(const std::shared_ptr >, - const std::vector &loops, - const std::vector &axis, bool is_block = false); - -// compare loop with binding axis -bool check_loop_binding_axis(const for_loop_node_t *loop_a, - const for_loop_node_t *loop_b); -bool check_loop_binding_axis(const for_loop &loop_a, const for_loop &loop_b); -// compare loop with binding axis and return aligned loop num -int check_loop_binding_axis(const std::vector &loop_a, - const std::vector &loop_b, - int64_t check_loop_size = -1); - -// check whether the given loop has axis of graph tensor -bool check_loop_has_axis(const for_loop &loop, - const std::shared_ptr >, - const std::vector &axis); - -/* used for loop transform */ -// copy binding axis hint when loop split -void copy_binding_axis_hint(const for_loop_node_t *ths, for_loop_node_t *other); -// merge binding axis hint when loop fuse -void fuse_binding_axis_hint(const for_loop_node_t *ths, - const for_loop_node_t *other); - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/graph/brgemm_fusion.cpp b/legacy/core/src/compiler/ir/graph/brgemm_fusion.cpp deleted file mode 100644 index 569d0dcd2..000000000 --- a/legacy/core/src/compiler/ir/graph/brgemm_fusion.cpp +++ /dev/null @@ -1,363 +0,0 @@ -/******************************************************************************* - * Copyright 2021-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "brgemm_fusion.hpp" -#include "../builder.hpp" -#include "../intrinsics.hpp" -#include "../pass/ir_copy.hpp" -#include "../viewer.hpp" -#include "fusion_anchor.hpp" -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -class valid_brgemm_finder_t : public ir_viewer_t { -public: - using ir_viewer_t::dispatch; - using ir_viewer_t::view; - // currenly only support one valid brgemm in fusion manager - expr get_valid_brgemm_node() const { - if (valid_brgemm_count_ == 1) { - return valid_brgemm_node_; - } - return expr(); - } - void operator()(stmt_c v) { ir_viewer_t::dispatch(std::move(v)); } - void view(intrin_call_c v) override { - if (v->type_ == intrin_type::brgemm || - v->type_ == intrin_type::list_brgemm) { - if (v->intrin_attrs_->get_or_else(intrin_attr::allow_brgemm_fusion, - false)) { - valid_brgemm_count_++; - valid_brgemm_node_ = v.remove_const(); - } - } - } - -private: - int valid_brgemm_count_ = 0; - expr valid_brgemm_node_; -}; - -class brgemm_inplace_replacer_t : public ir_inplace_visitor_t { -public: - using ir_inplace_visitor_t::dispatch_impl; - using ir_inplace_visitor_t::visit_impl; - brgemm_inplace_replacer_t(std::unordered_map &rmap, - const expr &out_tsr) - : rmap_(rmap), out_tsr_(out_tsr) {} - stmt operator()(stmt v) { - return ir_inplace_visitor_t::dispatch_impl(std::move(v)); - } - expr visit_impl(intrin_call v) override { - auto itr = rmap_.find(v); - if (itr != rmap_.end()) { - changed_ = true; - return itr->second; - } - return v; - } - stmt visit_impl(stmts v) override { - std::vector seq; - seq.reserve(v->seq_.size()); - bool has_out_tsr = false; - for (auto &st : v->seq_) { - auto newst = dispatch_impl(st); - if (newst.isa() && - newst.static_as()->var_.ptr_same(out_tsr_)) { - seq.insert(seq.begin(), newst); - } else { - seq.emplace_back(newst); - } - } - v->seq_ = seq; - return v; - } - -private: - std::unordered_map &rmap_; - // output tsr for define node search and upshift. - expr out_tsr_; -}; - -// Check if the shape is valid for brgemm. -// Valid cases: -// [m, n] + [1, n] -// [m, n] + [m, n] -// [m, n] + [1, 1] -// Invalid cases: -// [m, n] + [m, 1] -// [m, n] + [m, 2n] -static bool is_brgemm_valid_shape(const std::vector &extra_in_shape) { - if (extra_in_shape.size() < 2) { - return false; - } - // brgemm not support [m, 1] broadcast. - auto n = extra_in_shape[extra_in_shape.size() - 1]; - auto m = extra_in_shape[extra_in_shape.size() - 2]; - if (n.isa() && get_expr_as_int(n) == UINT64_C(1) && - !(m.isa() && get_expr_as_int(m) == UINT64_C(1))) { - return false; - } - for (size_t i = 0; i < extra_in_shape.size() - 2; i++) { - auto &shape = extra_in_shape[i]; - if (!(shape.isa() && get_expr_as_int(shape) == INT64_C(1))) { - return false; - } - } - return true; -} - -static sc_data_etype get_expr_etype(expr in) { - while (in.isa()) { - in = in.static_as()->base_->ptr_; - } - assert(in.isa()); - return etypes::get_pointer_element(in.static_as()->dtype_.type_code_); -} - -static bool register_bias(brgemm_fusion_register *reg, sc_data_type_t dtype, - const expr &bias) { - if (reg->data_.at(brgemm::bias)->equals(get_ir_null())) { - brgemm::bias_op_t op(dtype.as_etype()); - reg->setting_.emplace_back(op); - reg->data_.at(brgemm::bias) = bias; - return true; - } - return false; -} - -static bool register_scales(brgemm_fusion_register *reg, const expr &scales) { - if (reg->data_.at(brgemm::scales)->equals(get_ir_null())) { - brgemm::scale_op_t op; - reg->setting_.emplace_back(op); - reg->data_.at(brgemm::scales) = scales; - return true; - } - return false; -} - -static bool register_zp(brgemm_fusion_register *reg, const expr &compen, - const brgemm::alg_kind_t &alg) { - brgemm::postop_data_kind dkind; - if (alg == brgemm::a_zp) { - dkind = brgemm::a_zp_compensations; - } else if (alg == brgemm::b_zp) { - dkind = brgemm::b_zp_compensations; - } else { - assert(alg == brgemm::c_zp); - dkind = brgemm::c_zp_values; - } - if (reg->data_.at(dkind)->equals(get_ir_null())) { - brgemm::zp_op_t op(alg); - reg->setting_.emplace_back(op); - reg->data_.at(dkind) = compen; - return true; - } - return false; -} - -static bool register_unary(brgemm_fusion_register *reg, - brgemm::alg_kind_t alg_kind, const sc_op_ptr &op) { - // todo: currenly only add relu for test. - switch (alg_kind) { - case brgemm::alg_kind_t::eltwise_relu: { - brgemm::elt_op_t op2(alg_kind, 1.f, 0.f); - reg->setting_.emplace_back(op2); - return true; - } - default: - return false; - } -} - -static bool register_binary(brgemm_fusion_register *reg, - brgemm::alg_kind_t alg_kind, const sc_op_ptr &op, - const expr &extra_in, - const std::vector &extra_in_shape) { - if (!is_brgemm_valid_shape(extra_in_shape)) { - return false; - } - if (!reg->data_.at(brgemm::binary_post_ops_rhs)->equals(get_ir_null())) { - return false; - } - sc_dims dims = get_expr_to_dims(extra_in_shape); - assert(dims.size() >= 2); - std::vector int_dims(dims.end() - 2, dims.end()); - auto dtype = get_expr_etype(extra_in); - // todo: currently only support add/sub/mul/div. - switch (alg_kind) { - case brgemm::binary_add: - case brgemm::binary_sub: - case brgemm::binary_mul: - case brgemm::binary_div: { - brgemm::bin_op_t op2(alg_kind, int_dims.data(), dtype); - reg->setting_.emplace_back(op2); - reg->data_.at(brgemm::binary_post_ops_rhs) = extra_in; - return true; - } - default: - return false; - } -} - -// always register success. output dtype setting is always the first setting op. -static bool register_out_dtype(brgemm_fusion_register *reg, - const expr &output) { - auto dtype = get_expr_etype(output); - if (!utils::is_one_of(dtype, sc_data_etype::U8, sc_data_etype::S8, - sc_data_etype::S32, sc_data_etype::BF16, - sc_data_etype::F32)) { - return false; - } - if (reg->setting_.empty() || - reg->setting_[0].empty_op_.alg_ != brgemm::out_dtype) { - brgemm::out_op_t op(dtype); - reg->setting_.insert(reg->setting_.begin(), op); - } else { - reg->setting_[0].out_op_.dtype_ = dtype; - } - return true; -} - -static void update_data_c_ptr(brgemm_fusion_register *reg, expr output) { - if (!reg->data_.at(brgemm::binary_post_ops_rhs)->equals(get_ir_null())) { - reg->data_.at(brgemm::data_C_ptr) = std::move(output); - } -} - -bool brgemm_fusion_register::register_op_infos( - const sc_op_ptr &op, const expr &output, const expr &extra_in, - const std::vector &extra_in_shape) { - if (setting_.size() >= brgemm::postops_setting_t::max_postops_num) { - SC_WARN << "Op " << op->op_name_ - << " can not be inserted into brgemm fusion as the number of " - "postops is full: " - << setting_.size(); - return false; - } - auto brg_op = op->dyn_cast(); - // can not fuse in brgemm, return false - if (!brg_op || !brg_op->fuse_in_brgemm_ || - brg_op->alg_kind_ == brgemm::alg_kind_undef) { - return false; - } - bool status = false; - if (brg_op->alg_kind_ == brgemm::bias_add) { - status = - register_bias(this, op->get_inputs()[1]->details_.dtype_, extra_in); - } else if (brg_op->alg_kind_ == brgemm::out_scales) { - status = register_scales(this, extra_in); - } else if (brg_op->alg_kind_ == brgemm::out_dtype) { - status = register_out_dtype(this, output); - } else if (brg_op->alg_kind_ == brgemm::a_zp) { - status = register_zp(this, extra_in, brgemm::a_zp); - } else if (brg_op->alg_kind_ == brgemm::b_zp) { - status = register_zp(this, extra_in, brgemm::b_zp); - } else if (brg_op->alg_kind_ == brgemm::c_zp) { - status = register_zp(this, extra_in, brgemm::c_zp); - } else if (brg_op->alg_kind_ >= brgemm::eltwise_begin && - brg_op->alg_kind_ <= brgemm::eltwise_end) { - status = register_unary(this, brg_op->alg_kind_, op); - } else if (brg_op->alg_kind_ >= brgemm::binary_begin && - brg_op->alg_kind_ <= brgemm::binary_end) { - status = - register_binary(this, brg_op->alg_kind_, op, extra_in, extra_in_shape); - } else { - COMPILE_ASSERT(false, "Unsupported brgemm fusion op kind."); - } - if (status) { - last_out_ = output; - status = register_out_dtype(this, output); - update_data_c_ptr(this, output); - } - COMPILE_ASSERT(setting_.size() <= brgemm::postops_setting_t::max_postops_num, - "Current number of postops(" - << setting_.size() << ") should be less than max number(" - << brgemm::postops_setting_t::max_postops_num << ")."); - return status; -} - -bool brgemm_fusion_register::can_register_brgemm_fusion(const stmt &body) { - valid_brgemm_finder_t finder; - finder(body); - valid_brgemm_node_ = finder.get_valid_brgemm_node(); - return valid_brgemm_node_.defined(); -} - -void brgemm_fusion_register::reset() { - can_register_next_ = true; - last_out_ = expr(); - valid_brgemm_node_ = expr(); - setting_.clear(); - data_ = builtin::create_initialed_postops_data(); -} - -stmt brgemm_fusion_register::remake_brgemm_intrinsic_by_fusion( - stmt body, expr c_buf) const { - COMPILE_ASSERT(valid_brgemm_node_.defined(), - "Should return valid remake brgemm."); - auto node = valid_brgemm_node_.checked_as(); - assert(node->type_ == intrin_type::brgemm || - node->type_ == intrin_type::list_brgemm); - auto extra_args = node->intrin_attrs_->get( - intrin_attr::brgemm_extras); - assert(extra_args.postops_setting_.empty()); - if (setting_.empty()) { - return body; - } - extra_args.postops_setting_ = setting_; - assert(last_out_.isa()); - extra_args.dtype_C_ = last_out_.static_as() - ->base_->ptr_.static_as() - ->elem_dtype_; - int basic_arg_size = node->type_ == intrin_type::brgemm - ? brgemm_args::NUM_BASIC_ARGS_STRIDE - : brgemm_args::NUM_BASIC_ARGS_LIST; - // layout of node->args (full args): - // | basic_args | postops_data list(11 elems) | c_buf | bdmask_idx - auto new_args = std::vector(node->args_.begin(), - node->args_.begin() + basic_arg_size); - new_args.insert(new_args.end(), data_.begin(), data_.end()); - if (!c_buf.defined()) { - c_buf = get_ir_null(); - } - new_args.emplace_back(c_buf); - new_args.emplace_back(node->args_.back()); - new_args[brgemm_args::C] = last_out_; - assert(new_args.size() == node->args_.size()); - auto new_node = copy_attr( - *node, make_expr( - node->type_, new_args, - any_map_t{{intrin_attr::brgemm_extras, extra_args}, - {intrin_attr::allow_brgemm_fusion, - node->intrin_attrs_->get_or_else( - intrin_attr::allow_brgemm_fusion, false)}})); - std::unordered_map rmap = {{node, new_node}}; - brgemm_inplace_replacer_t replacer( - rmap, last_out_.static_as()->base_->ptr_); - return replacer(body); -} -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/brgemm_fusion.hpp b/legacy/core/src/compiler/ir/graph/brgemm_fusion.hpp deleted file mode 100644 index 95987158d..000000000 --- a/legacy/core/src/compiler/ir/graph/brgemm_fusion.hpp +++ /dev/null @@ -1,50 +0,0 @@ -/******************************************************************************* - * Copyright 2021-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_BRGEMM_FUSION_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_BRGEMM_FUSION_HPP -#include "graph.hpp" -#include -#include -#include -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -// When fusion manager executing `do_compute_blocks` pass, check if it can be -// registered to brgemm inside fusion, otherwise do trival computation. -struct brgemm_fusion_register { - // register in brgemm fusion needs op infos, extra input tensor shape and - // index, op output tensor index. - bool register_op_infos( - const sc_op_ptr &op, const expr &output, - const expr &extra_in = get_ir_null(), - const std::vector &extra_in_shape = std::vector()); - // only have one valid brgemm - bool can_register_brgemm_fusion(const stmt &body); - stmt remake_brgemm_intrinsic_by_fusion(stmt body, - expr c_buf = get_ir_null()) const; - void reset(); - bool can_register_next_ = true; - expr last_out_; - expr valid_brgemm_node_; - sc_brgemm_postops_setting_t setting_; - std::vector data_ = builtin::create_initialed_postops_data(); -}; -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl -#endif diff --git a/legacy/core/src/compiler/ir/graph/driver.cpp b/legacy/core/src/compiler/ir/graph/driver.cpp deleted file mode 100644 index 462982a18..000000000 --- a/legacy/core/src/compiler/ir/graph/driver.cpp +++ /dev/null @@ -1,356 +0,0 @@ -/******************************************************************************* - * Copyright 2021-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "driver.hpp" -#include "pass/pass.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef _MSC_VER -#include -#define getprocessid GetCurrentProcessId -#else -#include -#define getprocessid getpid -#endif - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -SC_MODULE(graph.driver) - -basic_graph_pass_ptr create_graph_pass(const std::string &name, - pass_func func_t, - const std::vector &required, - pass_type type, sc_opt_level opt_level, - bool enabled) { - return std::make_shared(func_t, name, required, type, - opt_level, enabled); -} - -static std::vector -filter_passes_by_opt_level(const std::vector &passes, - sc_opt_level opt_level) { - std::vector ret; - for (auto &p : passes) { - if (p->opt_level_ <= opt_level) { - ret.emplace_back(p); - } - } - return ret; -} - -static std::tuple, - std::vector> -create_default_graph_flow(const context_ptr &ctx) { - std::vector pre_tune_passes, post_tune_passes; - pre_tune_passes.push_back(create_graph_pass( - "eliminate_zero_shaped_tensors", eliminate_zero_shaped_tensors, {}, - pass_type::pre_tune, sc_opt_level::lv0, true)); - pre_tune_passes.push_back( - create_graph_pass("analysis_quantized", analysis_quantized, {}, - pass_type::analysis, sc_opt_level::lv0, true)); - pre_tune_passes.push_back(create_graph_pass( - "annotate_fusion_break", quantize::annotate_fusion_break, {}, - pass_type::pre_tune, sc_opt_level::lv2, true)); - pre_tune_passes.push_back( - create_graph_pass("annotate_config", annotate_config, {}, - pass_type::pre_tune, sc_opt_level::lv2, true)); - pre_tune_passes.push_back(create_graph_pass("graph_inline", graph_inline, {}, - pass_type::pre_tune, - sc_opt_level::lv0, true)); - pre_tune_passes.push_back( - create_graph_pass("constant_optimization", constant_optimization, {}, - pass_type::pre_tune, sc_opt_level::lv0, true)); - pre_tune_passes.push_back(create_graph_pass( - "quantized_info_propagation", quantize::quantize_info_propagation, {}, - pass_type::pre_tune, sc_opt_level::lv0, true)); - pre_tune_passes.push_back(create_graph_pass( - "quantized_graph_reschedule", quantize::graph_reschedule, {}, - pass_type::pre_tune, sc_opt_level::lv0, true)); - pre_tune_passes.push_back(create_graph_pass("fpmath_mode", fpmath_mode, {}, - pass_type::pre_tune, - sc_opt_level::lv0, true)); - // should be executed after graph reschedule, and before quantize_inline - pre_tune_passes.push_back( - create_graph_pass("rl_conv_weight_transform", rl_conv_weight_transform, - {}, pass_type::pre_tune, sc_opt_level::lv2, true)); - pre_tune_passes.push_back( - create_graph_pass("const_folding", graph_constant_input_folding, {}, - pass_type::pre_tune, sc_opt_level::lv2, true)); - pre_tune_passes.push_back(create_graph_pass("flatten_conv", flatten_conv, {}, - pass_type::pre_tune, - sc_opt_level::lv2, true)); - pre_tune_passes.push_back( - create_graph_pass("dynamic_graph_transform", dynamic_graph_transform, {}, - pass_type::pre_tune, sc_opt_level::lv2, true)); - pre_tune_passes.push_back( - create_graph_pass("quantize_inline", quantize::quantize_inline, {}, - pass_type::pre_tune, sc_opt_level::lv0, true)); - pre_tune_passes.push_back( - create_graph_pass("elemtwise_bcast_swap", elemwise_bcast_swap, {}, - pass_type::pre_tune, sc_opt_level::lv1, true)); - pre_tune_passes.push_back( - create_graph_pass("permute_propagation", permute_propagation, {}, - pass_type::pre_tune, sc_opt_level::lv0, true)); - pre_tune_passes.push_back(create_graph_pass( - "quantize_op_compensation", quantize::calculate_op_compensation, {}, - pass_type::pre_tune, sc_opt_level::lv0, true)); - pre_tune_passes.push_back( - create_graph_pass("broadcast_transform", broadcast_transform, {}, - pass_type::pre_tune, sc_opt_level::lv0, true)); - pre_tune_passes.push_back(create_graph_pass( - "elemwise_dimension_alignment", elemwise_dimension_alignment, {}, - pass_type::pre_tune, sc_opt_level::lv0, true)); - pre_tune_passes.push_back(create_graph_pass( - "shape_relationship_binding", shape_relationship_binding, {}, - pass_type::pre_tune, sc_opt_level::lv0, true)); - - // ------------------ post_tune ------------------------------------------- - post_tune_passes.push_back( - create_graph_pass("const_folding", graph_constant_input_folding, {}, - pass_type::post_tune, sc_opt_level::lv0, true)); - post_tune_passes.push_back( - create_graph_pass("div_bcast_transform", div_bcast_transform, {}, - pass_type::post_tune, sc_opt_level::lv2, true)); - post_tune_passes.push_back(create_graph_pass("pre_padding", pre_padding, {}, - pass_type::post_tune, - sc_opt_level::lv2, true)); - post_tune_passes.push_back( - create_graph_pass("layout_propagation", layout_propagation, {}, - pass_type::post_tune, sc_opt_level::lv0, true)); - post_tune_passes.push_back( - create_graph_pass("tensor_view_transform", tensor_view_transform, {}, - pass_type::post_tune, sc_opt_level::lv0, true)); - post_tune_passes.push_back( - create_graph_pass("const_folding", graph_constant_input_folding, {}, - pass_type::post_tune, sc_opt_level::lv0, true)); - post_tune_passes.push_back(create_graph_pass("graph_simplify", graph_simplify, - {}, pass_type::post_tune, - sc_opt_level::lv2, true)); - post_tune_passes.push_back( - create_graph_pass("global_reschedule", global_reschedule, {}, - pass_type::post_tune, sc_opt_level::lv1, true)); - post_tune_passes.push_back( - create_graph_pass("intrusive_opt_level", intrusive_opt_level, {}, - pass_type::post_tune, sc_opt_level::lv0, true)); - post_tune_passes.push_back( - create_graph_pass("partial_reduce_replace", partial_reduce_replace, {}, - pass_type::post_tune, sc_opt_level::lv2, true)); - // fix-me(brgemm-fuse): recover the following when postop is fixed -#if 0 - post_tune_passes.push_back(create_graph_pass("brgemm_fusion_transform", - brgemm_fusion_transform, {}, pass_type::post_tune, true)); -#endif - post_tune_passes.push_back( - create_graph_pass("const_folding", graph_constant_input_folding, {}, - pass_type::post_tune, sc_opt_level::lv0, true)); - if (ctx->flags_.concat_optimization_) { - post_tune_passes.push_back(create_graph_pass("merge_concats", merge_concats, - {}, pass_type::post_tune, - sc_opt_level::lv2, true)); - } - post_tune_passes.push_back( - create_graph_pass("const_folding_and_share", - graph_constant_input_folding_and_share_constants, {}, - pass_type::post_tune, sc_opt_level::lv1, true)); - post_tune_passes.push_back( - create_graph_pass("graph_code_cache", graph_code_cache, {}, - pass_type::post_tune, sc_opt_level::lv1, true)); - post_tune_passes.push_back( - create_graph_pass("inplace_transform", inplace_transform, {}, - pass_type::post_tune, sc_opt_level::lv0, true)); - post_tune_passes.push_back( - create_graph_pass("padded_mask_mark", padded_mask_mark, {}, - pass_type::post_tune, sc_opt_level::lv0, true)); - post_tune_passes.push_back( - create_graph_pass("mixed_partition", mixed_partition, {}, - pass_type::post_tune, sc_opt_level::lv1, true)); - if (ctx->flags_.concat_optimization_) { - post_tune_passes.push_back( - create_graph_pass("graph_concat_optimize", graph_concat_memory_planning, - {}, pass_type::post_tune, sc_opt_level::lv2, true)); - } - // filter passes by opt level - pre_tune_passes = - filter_passes_by_opt_level(pre_tune_passes, ctx->flags_.opt_level_); - post_tune_passes = - filter_passes_by_opt_level(post_tune_passes, ctx->flags_.opt_level_); - - // get passes map - std::unordered_map passes_map; - std::transform(pre_tune_passes.begin(), pre_tune_passes.end(), - std::inserter(passes_map, passes_map.end()), - [](const basic_graph_pass_ptr &pass) { - return std::make_pair(pass->name_, pass); - }); - - std::transform(post_tune_passes.begin(), post_tune_passes.end(), - std::inserter(passes_map, passes_map.end()), - [](const basic_graph_pass_ptr &pass) { - return std::make_pair(pass->name_, pass); - }); - // get pass's dependies and reset enabled_. - for (auto &kv : passes_map) { - if (kv.second->enabled_) { - for (const std::string &require : kv.second->requires_) { - passes_map[require]->enabled_ = true; - } - } - } - return std::make_tuple(pre_tune_passes, post_tune_passes); -} - -std::tuple, std::vector> -get_graph_passes(const context_ptr &ctx) { - return create_default_graph_flow(ctx); -} - -void run_graph_passes(sc_graph_t &graph, const context_ptr &ctx, - const std::vector &passes, - bool allow_cache) { - bool need_time = utils::compiler_configs_t::get().print_pass_time_; - bool need_result = utils::compiler_configs_t::get().print_pass_result_; - for (auto &pass : passes) { - if (pass->enabled_) { - auto timer = utils::create_scoped_timer( - need_time, [&pass](utils::time_duration dur) { - std::string name = std::string("graph.driver.time.") + pass->name_; - SC_MODULE_INFO2(name.c_str()) - << "took " - << std::chrono::duration_cast(dur) - .count() - << " us"; - }); - pass->func_(graph, ctx); - if (allow_cache && pass->type_ == pass_type::post_tune && - pass->name_ == "graph_code_cache") { - if (graph.attrs_.has_key("graph_code_cache")) { - break; - } - } - if (need_result) { - std::string name = std::string("graph.driver.debug.") + pass->name_; - if (auto stream = runtime::get_info_logging_stream(name.c_str())) { - *stream.stream_ << "IR after this pass:\n"; - print_graph(graph, *stream.stream_, true, true, true, true); - } - } - } - } -} - -std::unordered_map> -create_op_map(sc_graph_t &lg, sc_graph_t &rg) { - assert(lg.ops_.size() == rg.ops_.size()); - auto op_size = lg.ops_.size(); - std::unordered_map> op_map; - for (auto i = 0UL; i < op_size; i++) { - op_map[lg.ops_[i]] = std::vector({rg.ops_[i]}); - op_map[rg.ops_[i]] = std::vector({lg.ops_[i]}); - } - return op_map; -} - -sc_graph_t dynamic_shape_infer_preprocess(const sc_graph_t &graph, - const context_ptr &ctx) { - sc_graph_t copy = copy_graph(graph); - graph_inline(copy, ctx); - constant_optimization(copy, ctx); - return copy; -} - -void graph_driver(sc_graph_t &graph, const context_ptr &ctx, - const graph_config *in_cfg, graph_config *out_cfg, - int batch_size, int repeat, int64_t timeout, - tuner_creator *tune_creator, - std::vector *pre_tune_pass, - std::vector *post_tune_pass, - bool allow_cache) { - bool need_tuning = timeout != 0; - sc_graph_t graph_cpy; - - // save origin graph(tuning) / load config(no tune) - if (need_tuning) { - graph_cpy = copy_graph(graph); - graph.attrs_["temp.op_map"] = create_op_map(graph, graph_cpy); - } else { - SC_MODULE_INFO << "Use default config"; - } - - auto passes_tuple = get_graph_passes(ctx); - const std::vector *prepass = - pre_tune_pass ? pre_tune_pass : &std::get<0>(passes_tuple); - const std::vector *postpass = - post_tune_pass ? post_tune_pass : &std::get<1>(passes_tuple); - // run pre_processing passes - run_graph_passes(graph, ctx, *prepass, true); - - // run post tune passes - run_graph_passes(graph, ctx, *postpass, true); -} - -void graph_driver(sc_graph_t &graph, int batch_size, int repeat, - const context_ptr &ctx) { - graph_config *pincfg = nullptr; - graph_config *poutcfg = nullptr; - tuner_creator *ptun_creator = nullptr; - int64_t real_timeout = 0; - sc_graph_t orig_graph; - if (poutcfg) { - orig_graph = copy_graph(graph); - } - graph_driver(graph, ctx, pincfg, poutcfg, batch_size, repeat, real_timeout, - ptun_creator); -} - -void graph_driver_before_fusion(sc_graph_t &graph, const context_ptr &ctx) { - analysis_quantized(graph, ctx); - graph_inline(graph, ctx); - constant_optimization(graph, ctx); - quantize::quantize_info_propagation(graph, ctx); - - quantize::graph_reschedule(graph, ctx); - quantize::quantize_inline(graph, ctx); - - elemwise_bcast_swap(graph, ctx); - shape_relationship_binding(graph, ctx); - permute_propagation(graph, ctx); - - quantize::calculate_op_compensation(graph, ctx); - elemwise_dimension_alignment(graph, ctx); - layout_propagation(graph, ctx); - - tensor_view_transform(graph, ctx); - graph_simplify(graph, ctx); - global_reschedule(graph, ctx); - partial_reduce_replace(graph, ctx); - graph_constant_input_folding(graph, ctx); -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/driver.hpp b/legacy/core/src/compiler/ir/graph/driver.hpp deleted file mode 100644 index 7f5312d9a..000000000 --- a/legacy/core/src/compiler/ir/graph/driver.hpp +++ /dev/null @@ -1,145 +0,0 @@ -/******************************************************************************* - * Copyright 2021-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_DRIVER_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_DRIVER_HPP - -#include "analysis/analysis.hpp" -#include "transform/transform.hpp" -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -enum class pass_type { analysis, pre_tune, post_tune }; - -using pass_func = void (*)(sc_graph_t &, const context_ptr &); -struct basic_graph_pass_t { -public: - std::string name_; - std::vector requires_; - pass_func func_; - pass_type type_; - sc_opt_level opt_level_; // allowed minimum opt level. - bool enabled_; // for debug and tuning - basic_graph_pass_t(pass_func func, const std::string &name, - const std::vector &required, pass_type type, - sc_opt_level opt_level = sc_opt_level::lv3, - bool enabled = true) - : name_(name), requires_(required), func_(func), type_(type), - opt_level_(opt_level), enabled_(enabled) {} -}; - -using basic_graph_pass_ptr = std::shared_ptr; - -basic_graph_pass_ptr -create_graph_pass(const std::string &name, pass_func func, - const std::vector &required, pass_type type, - sc_opt_level opt_level = sc_opt_level::lv3, - bool enabled = true); - -// Return: std::vector, represents all passes run order. -// 1. If adding a new pass, developer need to define this pass will be put which -// pos in vector container. -// 2. If this pass has dependent passes, please set`requires_`, for example -// `create_pass("pass_0", pass_0_func, {"pass_1"}, pass_type::xxxx)` -// 3. Each pass's enabled_ field is defaulted true. In general, the pass's -// enabled_ is true. If pass has dependencies and executed by sepcial situations -// like: assuming having analaysis_xxx pass return tue, a pass will be opened -// (enabled_ sets true) and the passes it depends on also need to be opened. -SC_API std::tuple, - std::vector> -get_graph_passes(const context_ptr &ctx); - -/** - * @param graph orginal graph - * @param ctx the context - * Return: a deep copy of the original graph that runs preprocess passes for - * dynamic_infer_shape - * */ -SC_API sc_graph_t dynamic_shape_infer_preprocess( - const sc_graph_t &graph, const context_ptr &ctx = get_default_context()); - -/** - * @param graph orginal graph - * @param tuner_batch the number of configs which is - * generated from a call to get_next_config_batch(). The executor may - * generate executables from a batch of configs in parallel. - * @param repeat the times to repeatedly run the same executable. - * @param ctx if developer doesn't set context, use default context. - * The param: tuner_batch and repeat may be replaced by `timed out`. - * */ -SC_API void graph_driver(sc_graph_t &graph, int tuner_batch, int repeat, - const context_ptr &ctx = get_default_context()); - -struct tuner_creator; -struct graph_config; - -/** - * Runs post tune passes, tunes the graph and runs post tune passes - * @param graph orginal graph - * @param ctx the context - * @param in_cfg if tuning is off and if not null, takes this parameter as the - * config for the graph. If is null, will use default configs - * @param out_cfg if tuning is on and if not null, returns the tuned graph - * config to this pointer. - * @param tuner_batch the number of configs which is - * generated from a call to get_next_config_batch(). The executor may - * generate executables from a batch of configs in parallel. - * @param repeat the times to repeatedly run the same executable. - * @param timeout the timeout of the tuning. Will check if time runs out after - * each tuner batch. If set to negative, there will be no time limit. If set to - * 0, then the tuning will be turned off - * @param tune_creator the creator for the tuner. if null, use default tuner - * settings - * @param pre_tune_pass the graph passes before running tuner. if null, use - * default passes - * @param post_tune_pass the graph passes after running tuner. if null, use - * default passes - * @param allow_cache allow reusing cached code for the graph - * */ -SC_API void -graph_driver(sc_graph_t &graph, const context_ptr &ctx = get_default_context(), - const graph_config *in_cfg = nullptr, - graph_config *out_cfg = nullptr, int tuner_batch = 0, - int repeat = 0, int64_t timeout = 0, - tuner_creator *tune_creator = nullptr, - std::vector *pre_tune_pass = nullptr, - std::vector *post_tune_pass = nullptr, - bool allow_cache = false); - -// util function to create mapping of ops in the copied graph -std::unordered_map> -create_op_map(sc_graph_t &lg, sc_graph_t &rg); -void run_graph_passes(sc_graph_t &graph, const context_ptr &ctx, - const std::vector &passes, - bool allow_cache = false); - -// get graph driver result before fusion, usually used for unit test -void graph_driver_before_fusion(sc_graph_t &graph, const context_ptr &ctx); - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/graph/dynamic_dispatch_key.cpp b/legacy/core/src/compiler/ir/graph/dynamic_dispatch_key.cpp deleted file mode 100644 index 7e410e59b..000000000 --- a/legacy/core/src/compiler/ir/graph/dynamic_dispatch_key.cpp +++ /dev/null @@ -1,393 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "dynamic_dispatch_key.hpp" -#include "dynamic_internal_info.hpp" -#include "dynamic_lower_info.hpp" -#include "dynamic_utils.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -#define DISPATCH_KEY_MAX_THRESHOLD UINT64_C(64) -SC_MODULE(graph.dynamic_dispatch_key) - -bool op_dispatch_key_t::operator==(const op_dispatch_key_t &other) const { - return var_block_ == other.var_block_ && - in_out_formats_ == other.in_out_formats_ && impl_ == other.impl_; -} - -bool op_dispatch_key_t::operator!=(const op_dispatch_key_t &other) const { - return !(*this == other); -} - -void op_dispatch_key_t::set_op_dispatch_key(const sc_op_ptr &node, - const context_ptr &ctx) const { - if (auto tunable_node = node->dyn_cast()) { - tunable_node->set_config_by_key(*this, ctx); - } - auto &inputs = node->get_inputs(); - auto &outputs = node->get_outputs(); - int idx = 0; - for (auto &in : inputs) { - in->details_.set_format(in_out_formats_[idx++]); - } - for (auto &out : outputs) { - out->details_.set_format(in_out_formats_[idx++]); - } - node->info_.cur_impl_ = impl_; -} - -std::vector -op_dispatch_key_t::convert_to_runtime_format_vec() const { - std::vector outs(in_out_formats_.size()); - bool var_block_empty = var_block_.empty(); - assert(var_block_empty || var_block_.size() == in_out_formats_.size()); - for (size_t i = 0; i < in_out_formats_.size(); i++) { - sc_dim block0 = 0, block1 = 0; - if (!var_block_empty && !var_block_[i].empty()) { - block0 = var_block_[i][0]; - block1 = var_block_[i][1]; - } else { - block0 = in_out_formats_[i].blocks_[0]; - block1 = in_out_formats_[i].blocks_[1]; - } - outs[i] = runtime::dispatch_key( - static_cast(in_out_formats_[i].format_code_), block0, block1, - impl_, in_out_formats_[i].is_plain()); - } - return outs; -} - -void impl_op_dispatch_key_t::set_op_dispatch_key(const sc_op_ptr &node, - const context_ptr &ctx) const { - if (auto tun_node = node->dyn_cast()) { - tun_node->set_internal_config_by_key(*this, ctx); - } else { - throw std::runtime_error("Unimplement"); - } -} - -std::vector -impl_op_dispatch_key_t::convert_to_runtime_format_vec() const { - std::vector outs( - repeat_, runtime::dispatch_key(UINT64_C(0), 0, 0, impl_, false)); - return outs; -} - -std::vector -combined_op_dispatch_key_t::convert_to_runtime_format_vec() const { - std::vector ret; - for (auto &key : keys_) { - auto v = key.convert_to_runtime_format_vec(); - ret.insert(ret.end(), v.begin(), v.end()); - } - return ret; -} - -bool combined_op_dispatch_key_t::operator==( - const combined_op_dispatch_key_t &other) const { - return keys_ == other.keys_; -} - -bool combined_op_dispatch_key_t::operator!=( - const combined_op_dispatch_key_t &other) const { - return !(*this == other); -} - -void combined_op_dispatch_key_t::set_op_dispatch_key( - const sc_op_ptr &node, const context_ptr &ctx) const { - assert(node->isa()); - node->stc_cast()->update_internal_graph_format(*this, ctx); -} - -bool dispatch_key_cmper_t::operator()(const op_dispatch_key_t &key0, - const op_dispatch_key_t &key1) const { - if (key0.impl_ != key1.impl_) { - return key0.impl_ < key1.impl_; - } - assert(key0.in_out_formats_.size() == key1.in_out_formats_.size()); - for (size_t i = 0; i < key0.in_out_formats_.size(); i++) { - if (key0.in_out_formats_[i].format_code_ != - key1.in_out_formats_[i].format_code_) { - return key0.in_out_formats_[i].format_code_ < - key1.in_out_formats_[i].format_code_; - } - if (key0.in_out_formats_[i].blocks_ != key1.in_out_formats_[i].blocks_) { - return key0.in_out_formats_[i].blocks_ < key1.in_out_formats_[i].blocks_; - } - } - assert(key0.var_block_.size() == key1.var_block_.size()); - for (size_t i = 0; i < key0.var_block_.size(); i++) { - assert(key0.var_block_[i].size() == key1.var_block_[i].size()); - for (size_t j = 0; j < key0.var_block_[i].size(); j++) { - if (key0.var_block_[i][j] != key1.var_block_[i][j]) { - return key0.var_block_[i][j] < key1.var_block_[i][j]; - } - } - } - // equal - return false; -} - -bool impl_dispatch_key_cmper_t::operator()( - const impl_op_dispatch_key_t &key0, - const impl_op_dispatch_key_t &key1) const { - if (key0.impl_ != key1.impl_) { - return key0.impl_ < key1.impl_; - } - // equal - return false; -} - -combined_dispatch_key_set_t::combined_dispatch_key_set_t( - const std::vector &inputs, const sc_op_ptr &modified_inp) { - if (!inputs.empty()) { - auto dispatch_sets = get_dispatch_set_vec_from_ops(inputs); - internal_construct(dispatch_sets, inputs, modified_inp); - } -} - -combined_dispatch_key_set_t::combined_dispatch_key_set_t( - const std::vector &dispatch_sets) { - if (!dispatch_sets.empty()) { - internal_construct(dispatch_sets); - } -} - -void recursive_construct(combined_dispatch_key_set_t::inner_set_t &set, - combined_op_dispatch_key_t &cur_combined_key, - op_layout_link_vec_t &op_link_relations, - const std::vector &dispatch_sets, - const std::vector &inputs, - size_t cur_op_idx, size_t len_key, - int linked_reorder_impl) { - if (cur_op_idx == len_key) { - if (cur_combined_key.size() == len_key) { - set.insert(cur_combined_key); - } - return; - } - auto &inner_set = dispatch_sets[cur_op_idx]->get_inner_set(); - for (auto it = inner_set.begin(); it != inner_set.end(); it++) { - bool is_valid = true; - auto cur_key = *it; - if (!op_link_relations.empty()) { - for (size_t j = 0; j < op_link_relations[cur_op_idx].size(); j++) { - auto &link_pair = op_link_relations[cur_op_idx][j]; - if (link_pair.first != no_link_idx && - !is_linked_layout(it->in_out_formats_[j], - cur_combined_key[link_pair.first] - .in_out_formats_[link_pair.second])) { - is_valid = false; - break; - } - } - } - // In order to reduce the number of combined dispatch key, all reorder - // inside subgraph follows same impl algorithm(normal/no_padding). - // no_padding is available when all reorders has no padding at runtime. - if (!inputs.empty() && inputs[cur_op_idx]->isa()) { - if (it->impl_ != linked_reorder_impl) { - // static reorder in dynamic pattern - if (!can_op_be_dispatched(inputs[cur_op_idx])) { - cur_key.impl_ = linked_reorder_impl; - } else { - is_valid = false; - } - } - } - if (!is_valid) { - continue; - } - cur_combined_key.keys_.emplace_back(cur_key); - recursive_construct(set, cur_combined_key, op_link_relations, dispatch_sets, - inputs, cur_op_idx + 1, len_key, linked_reorder_impl); - cur_combined_key.keys_.pop_back(); - } -} - -static bool -validate_dispatch_key_sets(const std::vector &dispatch_sets, - const std::vector &inputs) { - // todo: add tunable dispatch check - // reorder impl kind check. - int cur_impl_num = -1; - for (size_t i = 0; i < inputs.size(); i++) { - if (inputs[i]->isa()) { - if (dispatch_sets[i]->size() == 1) { - continue; - } - auto get_impl_kind_num = [](const dispatch_set_ptr &dispatch_set) { - auto &set = dispatch_set->get_inner_set(); - bool has_normal = false, has_padding = false; - for (auto &it : set) { - if (it.impl_ == impl_kind_t::normal) { - has_normal = true; - } else if (it.impl_ == impl_kind_t::no_padding) { - has_padding = true; - } else { - throw std::runtime_error("Wrong impl kind for reorder."); - } - if (has_normal && has_padding) { - return 2; - } - } - if (has_normal || has_padding) { - return 1; - } - return 0; - }; - if (cur_impl_num == -1) { - cur_impl_num = get_impl_kind_num(dispatch_sets[i]); - } else if (cur_impl_num != get_impl_kind_num(dispatch_sets[i])) { - return false; - } - } - } - return true; -} - -void combined_dispatch_key_set_t::internal_construct( - const std::vector &dispatch_sets, - const std::vector &inputs, const sc_op_ptr &modified_inp) { - op_layout_link_vec_t op_link_relations; - auto len_key = dispatch_sets.size(); - auto num_keys = UINT64_C(1); - for (size_t i = len_key; i > 0; i--) { - num_keys = num_keys * dispatch_sets[i - 1]->size(); - } - // no need to dispatch - if (num_keys == 0) { - return; - } - bool is_specific = false; - if (!inputs.empty()) { - op_link_relations = - get_op_layout_link_relationships(inputs, dispatch_sets, modified_inp); - is_specific = is_dyn_specific_graph(inputs[0]->get_owner_graph()); - } - COMPILE_ASSERT(validate_dispatch_key_sets(dispatch_sets, inputs), - "Wrong dispatch key sets, could not construct combined " - "dispatch key."); - combined_op_dispatch_key_t cur_combined_key; - cur_combined_key.keys_.reserve(len_key); - auto reorder_impl_candidates = get_default_impl_dispatch_candidates(); - for (auto &linked_reorder_impl : reorder_impl_candidates) { - recursive_construct(set_, cur_combined_key, op_link_relations, - dispatch_sets, inputs, 0, len_key, linked_reorder_impl); - } - COMPILE_ASSERT(!set_.empty(), "Empty linked combined dispatch key set!"); - if (!is_specific && set_.size() > DISPATCH_KEY_MAX_THRESHOLD) { - SC_MODULE_WARN << "Number of dispatch key set " << set_.size() - << " has exceeded threshold " << DISPATCH_KEY_MAX_THRESHOLD; - } -} - -bool combined_dispatch_key_cmper_t::operator()( - const combined_op_dispatch_key_t &key0, - const combined_op_dispatch_key_t &key1) const { - assert(key0.size() == key1.size()); - dispatch_key_cmper_t cmper; - for (size_t i = 0; i < key0.size(); i++) { - if (key0[i] != key1[i]) { - return cmper(key0[i], key1[i]); - } - } - // equal - return false; -} - -void dispatch_key_set_t::for_each_key_process( - const std::function &callback) { - for (auto &key : set_) { - callback(&key); - } -} - -std::set & -dispatch_key_set_t::get_inner_set() { - return set_; -} - -dispatch_set_ptr dispatch_key_set_t::copy() const { - return std::make_shared(set_); -} - -void impl_dispatch_key_set_t::for_each_key_process( - const std::function &callback) { - for (auto &key : set_) { - callback(&key); - } -} - -std::set & -impl_dispatch_key_set_t::get_inner_set() { - throw std::runtime_error("Impl disaptch key set can not get its inner set."); -} - -dispatch_set_ptr impl_dispatch_key_set_t::copy() const { - return std::make_shared(set_); -} - -void combined_dispatch_key_set_t::for_each_key_process( - const std::function &callback) { - for (auto &key : set_) { - callback(&key); - } -} - -std::set & -combined_dispatch_key_set_t::get_inner_set() { - throw std::runtime_error( - "Combined dispatch key set can not get its inner set"); -} - -dispatch_set_ptr combined_dispatch_key_set_t::copy() const { - return std::make_shared(set_); -} - -std::vector get_default_impl_dispatch_candidates() { - static std::vector default_impl_candidates = {impl_kind_t::normal, - impl_kind_t::no_padding}; - return default_impl_candidates; -} - -std::vector get_dynamic_impl_dispatch_candidates(tunable_op_t *op, - const context_ptr &ctx) { - auto configs = op->get_dynamic_config_candidates(ctx); - std::vector ret; - ret.reserve(configs.size()); - for (int i = 0; i < static_cast(configs.size()); i++) { - ret.push_back(i); - } - return ret; -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/dynamic_dispatch_key.hpp b/legacy/core/src/compiler/ir/graph/dynamic_dispatch_key.hpp deleted file mode 100644 index 10f3a4fc9..000000000 --- a/legacy/core/src/compiler/ir/graph/dynamic_dispatch_key.hpp +++ /dev/null @@ -1,203 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_DYNAMIC_DISPATCH_KEY_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_DYNAMIC_DISPATCH_KEY_HPP -#include -#include -#include -#include -#include -#include -#include -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace runtime { -union dispatch_key; -} -class tunable_op_t; -struct context_t; -class sc_op; -// the common base class of op_dispatch_key_t and combind_op_dispatch_key_t. -struct op_dispatch_key_base_t { - virtual ~op_dispatch_key_base_t() {} - // set dispatch key for op. - virtual void - set_op_dispatch_key(const std::shared_ptr &node, - const std::shared_ptr &ctx) const = 0; - virtual std::vector - convert_to_runtime_format_vec() const = 0; -}; - -// the dispatch key type for lowering. Will be used in a map in lowering. The -// key is this struct and the value is kernel. -struct op_dispatch_key_t : public op_dispatch_key_base_t { - // Currently only need for tunable op. Size is same as in_out_formats, and - // illustrate the config of input/outputs. E.g matmul_core config (M, N, - // K)[32, 16, 64], we got {{32, 64}, {64, 16}, {32, 16}}. - std::vector> var_block_; - // a vector of input/output formats, order is input 0,1,..., output 0,1,... - std::vector in_out_formats_; - // the op can be dispatched as padding or not. - // todo(cuij): find if the field could be deleted if we complete query by - // juxtaposed. If we don't need query of combination any more. - int impl_ = impl_kind_t::normal; - op_dispatch_key_t() = default; - virtual ~op_dispatch_key_t() {} - op_dispatch_key_t(const std::vector &formats, - int impl = impl_kind_t::normal) - : in_out_formats_(formats), impl_(impl) {} - op_dispatch_key_t(const std::vector> &var_block, - const std::vector &formats, - bool impl = false) - : var_block_(var_block), in_out_formats_(formats), impl_(impl) {} - bool operator==(const op_dispatch_key_t &other) const; - bool operator!=(const op_dispatch_key_t &other) const; - void - set_op_dispatch_key(const std::shared_ptr &node, - const std::shared_ptr &ctx) const override; - std::vector - convert_to_runtime_format_vec() const override; -}; - -struct impl_op_dispatch_key_t : public op_dispatch_key_base_t { - int impl_ = impl_kind_t::normal; - int repeat_; - virtual ~impl_op_dispatch_key_t() {} - // impl kernel dispatch uses same table of outer kernel dispatch, so set - // repeat here to match the number of outer kernel keys. - impl_op_dispatch_key_t(int impl, int repeat) : impl_(impl), repeat_(repeat) {} - void - set_op_dispatch_key(const std::shared_ptr &node, - const std::shared_ptr &ctx) const override; - std::vector - convert_to_runtime_format_vec() const override; -}; - -struct combined_op_dispatch_key_t : public op_dispatch_key_base_t { - combined_op_dispatch_key_t() = default; - combined_op_dispatch_key_t(std::initializer_list keys) - : keys_(std::move(keys)) {} - combined_op_dispatch_key_t(std::vector &&keys) - : keys_(std::move(keys)) {} - bool operator==(const combined_op_dispatch_key_t &other) const; - bool operator!=(const combined_op_dispatch_key_t &other) const; - op_dispatch_key_t &operator[](size_t i) { return keys_[i]; } - const op_dispatch_key_t &operator[](size_t i) const { return keys_[i]; } - - size_t size() const { return keys_.size(); } - void - set_op_dispatch_key(const std::shared_ptr &node, - const std::shared_ptr &ctx) const override; - std::vector - convert_to_runtime_format_vec() const override; - // explict op_dispatch_key_t here, not impl_op_dispatch_key_t because it - // could not be combined. - std::vector keys_; -}; - -struct dispatch_key_cmper_t { - bool operator()(const op_dispatch_key_t &key0, - const op_dispatch_key_t &key1) const; -}; - -struct impl_dispatch_key_cmper_t { - bool operator()(const impl_op_dispatch_key_t &key0, - const impl_op_dispatch_key_t &key1) const; -}; - -struct combined_dispatch_key_cmper_t { - bool operator()(const combined_op_dispatch_key_t &key0, - const combined_op_dispatch_key_t &key1) const; -}; - -// common base struct for dispatch_key_set_t and combined_dispatch_key_set_t. -struct disaptch_key_set_t; -struct dispatch_key_set_base_t { - virtual ~dispatch_key_set_base_t() {} - virtual size_t size() const = 0; - virtual void for_each_key_process( - const std::function &callback) = 0; - virtual std::set & - get_inner_set() = 0; - virtual std::shared_ptr copy() const = 0; -}; - -struct dispatch_key_set_t : public dispatch_key_set_base_t { - using inner_set_t = std::set; - dispatch_key_set_t() = default; - dispatch_key_set_t(const inner_set_t &set) : set_(set) {} - size_t size() const override { return set_.size(); } - void for_each_key_process( - const std::function &callback) - override; - inner_set_t &get_inner_set() override; - std::shared_ptr copy() const override; - inner_set_t set_; -}; - -struct impl_dispatch_key_set_t : public dispatch_key_set_base_t { - using inner_set_t = - std::set; - impl_dispatch_key_set_t() = default; - impl_dispatch_key_set_t(const inner_set_t &set) : set_(set) {} - size_t size() const override { return set_.size(); } - void for_each_key_process( - const std::function &callback) - override; - std::set &get_inner_set() override; - std::shared_ptr copy() const override; - inner_set_t set_; -}; - -struct combined_dispatch_key_set_t : public dispatch_key_set_base_t { - using inner_set_t = - std::set; - // modified inp is for compatibility of fused op as its internal tuanble op - // and fusible op are in seperate graphs. - combined_dispatch_key_set_t( - const std::vector> &inputs, - const std::shared_ptr &modified_inp = nullptr); - combined_dispatch_key_set_t( - const std::vector> - &dispatch_sets); - combined_dispatch_key_set_t(const inner_set_t &set) : set_(set) {} - void - internal_construct(const std::vector> - &dispatch_sets, - const std::vector> &inputs = - std::vector>(), - const std::shared_ptr &modified_inp = nullptr); - size_t size() const override { return set_.size(); } - void for_each_key_process( - const std::function &callback) - override; - std::set &get_inner_set() override; - std::shared_ptr copy() const override; - inner_set_t set_; -}; - -std::vector get_default_impl_dispatch_candidates(); -std::vector -get_dynamic_impl_dispatch_candidates(tunable_op_t *op, - const std::shared_ptr &ctx); -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/graph/dynamic_internal_info.hpp b/legacy/core/src/compiler/ir/graph/dynamic_internal_info.hpp deleted file mode 100644 index 04a1eae05..000000000 --- a/legacy/core/src/compiler/ir/graph/dynamic_internal_info.hpp +++ /dev/null @@ -1,40 +0,0 @@ -/******************************************************************************* - * Copyright 2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_DYNAMIC_INTERNAL_INFO_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_DYNAMIC_INTERNAL_INFO_HPP -#include -#include -#include -#include -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -struct dyn_internal_info_t { - // Internal function could be generated from a fused partition. Here we - // record final arguments for function decl and call. - std::vector parti_in_ltsrs_; - std::vector parti_out_ltsrs_; - // Inner(fused op) dispatch table name in fused op. - std::string dispatch_table_name_; -}; -using dyn_internal_info_ptr = std::shared_ptr; -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/graph/dynamic_lower_info.hpp b/legacy/core/src/compiler/ir/graph/dynamic_lower_info.hpp deleted file mode 100644 index 1c58f2f96..000000000 --- a/legacy/core/src/compiler/ir/graph/dynamic_lower_info.hpp +++ /dev/null @@ -1,59 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_DYNAMIC_LOWER_INFO_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_DYNAMIC_LOWER_INFO_HPP -#include -#include -#include -#include -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -struct dynamic_lower_info_t { - static constexpr sc_dim init_placeholder = dimensions::dynamic_any - 1; - // decreasing placeholder to represent dynamic dim in graph. - // range is [-2, min of int64_t] - sc_dim cur_dynamic_placeholder_ = init_placeholder; - // dynamic sc_dim => expr(uint64_t) map, we maitain one map for one graph. - std::unordered_map dim2expr_map_; - // Is specific sub graph for parallel compilation. - bool is_specific_ = false; - - const sc_dim &get_cur_dynamic_placeholder() const { - return cur_dynamic_placeholder_; - } - const std::unordered_map &get_dim2expr_map() const { - return dim2expr_map_; - } - sc_dim &get_cur_dynamic_placeholder() { return cur_dynamic_placeholder_; } - std::unordered_map &get_dim2expr_map() { return dim2expr_map_; } - void set_cur_dynamic_placeholder(const sc_dim &cur_dynamic_placeholder) { - cur_dynamic_placeholder_ = cur_dynamic_placeholder; - } - void set_dim2expr_map(const std::unordered_map &dim2expr_map) { - dim2expr_map_ = dim2expr_map; - } -}; -using dyn_lower_info_ptr = std::shared_ptr; -class sc_graph_t; -bool is_dyn_specific_graph(sc_graph_t &); -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/graph/dynamic_utils.cpp b/legacy/core/src/compiler/ir/graph/dynamic_utils.cpp deleted file mode 100644 index 30981e493..000000000 --- a/legacy/core/src/compiler/ir/graph/dynamic_utils.cpp +++ /dev/null @@ -1,654 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#include "dynamic_utils.hpp" -#include "dynamic_dispatch_key.hpp" -#include "dynamic_lower_info.hpp" -#include "utils.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -void initialize_format_table_with_op(const sc_op_ptr &op, - op_dispatch_tables_ptr &tb) { - uint32_t inp_size = op->get_inputs().size(); - auto dispatch_keys = op->get_dispatch_key_set(); - // allow empty format table query. It means format has been set with initial - // values during graph lowering. - if (dispatch_keys->size() <= 1) { - return; - } - if (op->isa()) { - auto set_format_by_keys = [&](const op_dispatch_key_base_t *dispatch_key) { - std::vector all_formats = - dispatch_key->convert_to_runtime_format_vec(); - std::vector keys(all_formats.begin(), - all_formats.end() - 1); - runtime::dispatch_key value = all_formats[all_formats.size() - 1]; - tb->format_table_[keys] = {value}; - }; - dispatch_keys->for_each_key_process(set_format_by_keys); - } else { - uint64_t unknown_fmt = 0; - std::vector keys(inp_size, unknown_fmt); - auto set_format_by_key = [&](const op_dispatch_key_base_t *dispatch_key) { - std::vector values = - dispatch_key->convert_to_runtime_format_vec(); - // only one input format known. - for (uint32_t i = 0; i < inp_size; i++) { - keys.clear(); - keys.resize(inp_size, unknown_fmt); - keys[i] = values[i]; - tb->format_table_[keys] = values; - } - // all input format known - for (uint32_t i = 0; i < inp_size; i++) { - keys[i] = values[i]; - } - tb->format_table_[keys] = values; - }; - dispatch_keys->for_each_key_process(set_format_by_key); - } -} - -std::shared_ptr op_dispatch_tables_t::op_func_info::lower() { - ir_module_ptr mod = - internal_ ? op_->get_internal_func(ctx_) : op_->get_func(ctx_); - auto func = mod->get_entry_func(); - func->attr()[function_attrs::top_level] = false; - func->name_ += name_or_postfix_; - func->decl_->name_ = func->name_; - - mod->attr_[ir_module_t::attr_key_t::MANAGED_THREAD_POOL] = *use_managed_tp_; - mod->attr_[ir_module_t::attr_key_t::STATIC_GLOBALS] = true; - return mod; -} - -void initialize_impl_kind_table_with_op(const context_ptr &ctx, - const sc_op_ptr &op, - op_dispatch_tables_ptr &tb) { - COMPILE_ASSERT(op->isa(), - "impl_kind table currently is only used for tunable op."); - auto tun_op = op->dyn_cast(); - tb->impl_kind_table_ = tun_op->convert_config_candidates_to_impl_map( - tun_op->get_dynamic_config_candidates(ctx)); -} - -void initialize_op_info_with_op(const sc_op_ptr &op, - op_dispatch_tables_ptr &tb) { - tb->op_info_ = op->get_dynamic_runtime_info(); -} - -void initialize_dispatch_table_with_op(const context_ptr &ctx, - const sc_op_ptr &op, - op_dispatch_tables_ptr &tb) { - initialize_format_table_with_op(op, tb); - if (op->isa()) { - initialize_impl_kind_table_with_op(ctx, op, tb); - } - initialize_op_info_with_op(op, tb); -} - -void add_dispatch_symbol_to_kernel_table( - op_dispatch_tables_ptr &tb, const op_dispatch_key_base_t *key, - op_dispatch_tables_t::op_func_info &&value) { - std::vector runtime_keys = - key->convert_to_runtime_format_vec(); - tb->kernel_table_.insert(std::make_pair(runtime_keys, std::move(value))); -} - -bool is_internal_op(const sc_op_ptr &op) { - return op->op_name_ != "input" && op->op_name_ != "output" && - op->op_name_ != "constant"; -} - -bool can_op_be_dispatched(const sc_op_ptr &op) { - return is_internal_op(op) && op->get_owner_graph().is_dynamic() && - op->get_dispatch_key_set()->size() > 1; -} - -bool can_op_query_output(const sc_op_ptr &op) { - if (!is_internal_op(op)) { - return false; - } - if (op->op_name_ == "dynamic_reshape") { - return false; - } - std::unordered_set set; - for (auto &in : op->get_inputs()) { - for (auto &d : in->details_.get_plain_dims()) { - if (is_dynamic_dim(d)) { - set.insert(d); - } - } - } - for (auto &out : op->get_outputs()) { - for (auto &d : out->details_.get_plain_dims()) { - if (is_dynamic_dim(d) && set.find(d) == set.end()) { - return true; - } - } - } - return false; -} - -bool can_op_be_queried(const sc_op_ptr &op) { - return can_op_be_dispatched(op) || can_op_query_output(op); -} - -std::vector -get_dispatch_set_vec_from_ops(const std::vector &ops) { - std::vector ret; - ret.reserve(ops.size()); - for (auto &op : ops) { - ret.emplace_back(op->get_dispatch_key_set()); - } - return ret; -} - -sc_op_ptr find_parent_dispatch_node(const graph_tensor_ptr &in) { - auto cur_op = in->producer_owner_; - while (!(cur_op->isa() || cur_op->isa() || - cur_op->isa())) { - int layout_input_idx = - cur_op->attrs_.get_or_else(op_attr_key::layout_input_index, 0); - cur_op = cur_op->get_inputs()[layout_input_idx]->producer_owner_; - }; - return cur_op->shared_from_this(); -} - -sc_op_ptr find_output_linked_tunable_op(const graph_tensor_ptr &in) { - for (auto &use : in->uses_) { - auto op = use.second.lock(); - if (op->isa() || op->isa() || - op->isa()) { - continue; - } - if (op->isa() || op->isa()) { - for (size_t i = 0; i < op->get_inputs().size(); i++) { - auto parent_node = find_parent_dispatch_node(op->get_inputs()[i]); - if (parent_node->isa()) { - return parent_node; - } - } - } - auto ret = find_output_linked_tunable_op(op->get_outputs()[0]); - if (ret) { - return ret; - } - } - return nullptr; -} - -static std::pair -make_linked_op_output_pair(const std::vector &dispatch_keys, - int op_idx) { - assert(op_idx >= 0 && op_idx < static_cast(dispatch_keys.size())); - return std::make_pair(op_idx, static_cast(dispatch_keys[op_idx] - ->get_inner_set() - .begin() - ->in_out_formats_.size()) - - 1); -} - -op_layout_link_vec_t get_op_layout_link_relationships( - const std::vector> &ops, - const std::vector &dispatch_keys, - const sc_op_ptr &modified_inp) { - op_layout_link_vec_t ret; - ret.resize(ops.size()); - for (size_t i = 0; i < ops.size(); i++) { - assert(!dispatch_keys[i]->get_inner_set().begin()->in_out_formats_.empty()); - ret[i].resize( - dispatch_keys[i]->get_inner_set().begin()->in_out_formats_.size(), - std::make_pair(no_link_idx, no_link_idx)); - } - for (size_t i = 0; i < ops.size(); i++) { - auto &op = ops[i]; - if (op->isa()) { - auto parent_inp = find_parent_dispatch_node(op->get_inputs()[0]); - int linked_op_idx = no_link_idx; - int op_layout_idx = no_link_idx; - if (parent_inp->isa() || parent_inp == modified_inp) { - // input linked with tuanble op output - if (parent_inp == modified_inp) { - linked_op_idx = 0; - } else { - auto it = std::find(ops.begin(), ops.end(), parent_inp); - COMPILE_ASSERT(it != ops.end(), "Wrong graph pattern!"); - linked_op_idx = std::distance(ops.begin(), it); - } - op_layout_idx = 0; - } else { - // output may be linked with tunable op output - auto tunable_op = find_output_linked_tunable_op(op->get_outputs()[0]); - op_layout_idx = 1; - if (tunable_op) { - auto it = std::find(ops.begin(), ops.end(), tunable_op); - COMPILE_ASSERT(it != ops.end(), "Wrong graph pattern!"); - linked_op_idx = std::distance(ops.begin(), it); - } - } - if (linked_op_idx != no_link_idx) { - auto linked_layout_idx = - static_cast(dispatch_keys[linked_op_idx] - ->get_inner_set() - .begin() - ->in_out_formats_.size()) - - 1; - if (linked_op_idx < static_cast(i)) { - ret[i][op_layout_idx] = - std::make_pair(linked_op_idx, linked_layout_idx); - } else { - ret[linked_op_idx][linked_layout_idx] = - std::make_pair(i, op_layout_idx); - } - } - } else { - // tunable_op link - assert(op->isa()); - for (size_t j = 0; j < op->get_inputs().size(); j++) { - auto parent_node = find_parent_dispatch_node(op->get_inputs()[j]); - if (parent_node->isa() || - parent_node->isa()) { - auto it = std::find(ops.begin(), ops.end(), parent_node); - COMPILE_ASSERT(it != ops.end(), "Wrong graph pattern!"); - int op_idx = std::distance(ops.begin(), it); - auto linked_layout_idx = - static_cast(dispatch_keys[op_idx] - ->get_inner_set() - .begin() - ->in_out_formats_.size()) - - 1; - if (op_idx < static_cast(i)) { - ret[i][j] = std::make_pair(op_idx, linked_layout_idx); - } else { - ret[op_idx][linked_layout_idx] = std::make_pair(i, j); - } - } - } - } - } - return ret; -} - -bool is_linked_layout(const sc_data_format_t &layout1, - const sc_data_format_t &layout2) { - if (!(layout1.is_blocking() && layout2.is_blocking())) { - return layout1 == layout2; - } - // both blocking - auto block_num1 = layout1.get_blocks_size(); - auto block_num2 = layout2.get_blocks_size(); - auto offset1 = block_num1 < block_num2 ? 0 : block_num1 - block_num2; - auto offset2 = block_num1 < block_num2 ? block_num2 - block_num1 : 0; - auto block_num = std::min(block_num1, block_num2); - for (int i = 0; i < block_num; i++) { - if (!(layout1.blocks_[offset1 + i] == layout2.blocks_[offset2 + i] || - layout1.blocks_[offset1 + i] == 1 || - layout2.blocks_[offset2 + i] == 1)) { - return false; - } - } - return true; -} - -runtime::dynamic_tensor_t -convert_graph_tensor_to_dynamic_tensor(const graph_tensor_ptr &in, - void *data_ptr, sc_dim *shape_ptr) { - runtime::dynamic_tensor_t ret; - auto &plain_dims = in->details_.get_plain_dims(); - ret.data_ = data_ptr; - ret.dims_ = shape_ptr; - ret.ndims_ = static_cast(plain_dims.size()); - ret.dtype_ = static_cast(in->details_.dtype_.type_code_); - ret.dyn_mask_ = 0; - for (int i = 0; i < static_cast(plain_dims.size()); i++) { - if (is_dynamic_dim(plain_dims[i])) { - ret.dyn_mask_ |= (1 << i); - } - } - return ret; -} - -static bool need_query_next_first(const sc_op_ptr &node) { - bool has_tail_reorder = node->isa(); - std::vector out_ops; - if (node->isa()) { - out_ops = node->stc_cast()->sub_graph_.get_output_ops(); - } - for (auto &op : out_ops) { - if (op->get_inputs()[0]->producer_owner_->isa()) { - has_tail_reorder = true; - break; - } - } - return has_tail_reorder && - can_op_be_dispatched(node->get_outputs()[0]->uses_[0].second.lock()); -} - -void lower_query_function( - std::vector &visited, const sc_op_ptr &node, - const std::function &callback) { - if (visited[node->logical_op_id_]) { - return; - } - if (node->attrs_.get_or_else("constant", const_kind::not_const) == - const_kind::not_const && - need_query_next_first(node)) { - for (size_t i = 0; i < node->get_outputs().size(); i++) { - auto query_node = node->get_outputs()[i]->uses_[0].second.lock(); - lower_query_function(visited, query_node, callback); - } - } - callback(node); - visited[node->logical_op_id_] = true; -} - -void visit_fused_graph_by_query_order( - sc_graph_t &graph, const std::function &callback) { - for (auto &op : graph.ops_) { - if (!op->isa()) { - callback(op); - } - } - for (auto &op : graph.ops_) { - if (op->isa()) { - callback(op); - } - } -} -std::vector get_graph_inner_dispatch_ops(sc_graph_t &graph, - int *total_key_num) { - std::vector ret; - // first add tunable_op dispatch key then reorder op - auto get_inner_ops = [&ret, &total_key_num](const sc_op_ptr &op) { - if (op->isa() || op->isa()) { - ret.emplace_back(op); - if (total_key_num) { - *total_key_num += static_cast(op->get_inputs().size() + - op->get_outputs().size()); - } - } - }; - visit_fused_graph_by_query_order(graph, get_inner_ops); - return ret; -} - -static int find_padding_impl(const combined_op_dispatch_key_t &key) { - for (auto &cur_key : key.keys_) { - // reorder - if (cur_key.in_out_formats_.size() == 2) { - return cur_key.impl_; - } - } - return impl_kind_t::normal; -} - -void update_graph_format_by_key(const context_ptr &ctx, - const sc_op_ptr &fused_op, sc_graph_t &graph, - const combined_op_dispatch_key_t &key, - int &key_idx, size_t node_input_offset, - size_t graph_input_offset, - const sc_op_ptr &modified_inp) { - auto &node_inputs = fused_op->get_inputs(); - std::vector visited(graph.ops_.size(), false); - int padding_impl = find_padding_impl(key); - // currently we store dispatch key of tunable op and reorder only in - // fused op. - auto update_format = [&](const sc_op_ptr &node) { - if (node->isa() || node->isa()) { - auto &cur_key = key[key_idx++]; - assert(cur_key.in_out_formats_.size() == 2 || - cur_key.in_out_formats_.size() == 3); - // update format - for (size_t i = 0; i < cur_key.in_out_formats_.size() - 1; i++) { - node->get_inputs()[i]->details_.set_format(cur_key.in_out_formats_[i]); - } - if (node->isa()) { - node->stc_cast()->set_config_by_key(cur_key, ctx); - } - if (node->isa() && - cur_key.in_out_formats_[0].is_blocking()) { - auto inp_parent = find_parent_dispatch_node(node->get_inputs()[0]); - if (inp_parent->isa() && inp_parent != modified_inp) { - inp_parent->get_outputs()[0]->details_.set_format( - cur_key.in_out_formats_[0]); - } - } - node->get_outputs()[0]->details_.set_format( - cur_key.in_out_formats_[cur_key.in_out_formats_.size() - 1]); - // update impl alg - node->info_.cur_impl_ = cur_key.impl_; - } else if (node->isa() || - node->isa()) { - node->info_.cur_impl_ = padding_impl; - } - }; - visit_fused_graph_by_query_order(graph, update_format); - assert(key_idx == static_cast(key.size())); - graph.attrs_.set("insert_reorder", false); - graph.attrs_.set("is_output_plain", false); - layout_propagation(graph); - - // sync fused op's input/output format with inner graph - auto graph_inputs = graph.get_input_ops(); - assert(node_inputs.size() + graph_input_offset == - graph_inputs.size() + node_input_offset); - for (size_t i = 0; i + graph_input_offset < graph_inputs.size(); i++) { - node_inputs[i + node_input_offset]->details_.set_format( - graph_inputs[i + graph_input_offset] - ->get_outputs()[0] - ->details_.get_format()); - } - // update fused op output format - auto &node_outputs = fused_op->get_outputs(); - auto graph_outputs = graph.get_output_ops(); - assert(graph_outputs.size() == node_outputs.size()); - for (size_t i = 0; i < graph_outputs.size(); i++) { - node_outputs[i]->details_.set_format( - graph_outputs[i]->get_inputs()[0]->details_.get_format()); - } -} - -expr call_op_dynamic_query_function(const sc_op_ptr &op, - const std::vector &args) { - if (op->isa()) { - assert(args.size() == 13 || args.size() == 14); - return builtin::call_matmul_core_query_format( - args[0], args[1], args[2], args[3], args[4], args[5], args[6], args[7], - args[8], args[9], args[10], args[11], args[12], - args.size() == 13 ? get_ir_null() : args[13]); - } else if (op->isa()) { - assert(args.size() == 13 || args.size() == 14); - return builtin::call_managed_matmul_core_query_format( - args[0], args[1], args[2], args[3], args[4], args[5], args[6], args[7], - args[8], args[9], args[10], args[11], args[12], - args.size() == 13 ? get_ir_null() : args[13]); - } else if (op->isa()) { - assert(args.size() == 13 || args.size() == 14); - return builtin::call_conv_fwd_core_query_format( - args[0], args[1], args[2], args[3], args[4], args[5], args[6], args[7], - args[8], args[9], args[10], args[11], args[12], - args.size() == 13 ? get_ir_null() : args[13]); - } else if (op->isa()) { - assert(args.size() == 7); - return builtin::call_unary_fusible_op_query_format( - args[0], args[1], args[2], args[3], args[4], args[5], args[6]); - } else if (op->isa()) { - assert(args.size() == 9); - return builtin::call_binary_fusible_op_query_format( - args[0], args[1], args[2], args[3], args[4], args[5], args[6], args[7], - args[8]); - } else if (op->isa()) { - assert(args.size() == 7 || args.size() == 8); - return builtin::call_reorder_op_query_format( - args[0], args[1], args[2], args[3], args[4], args[5], args[6], - args.size() == 7 ? get_ir_null() : args[7]); - } else if (op->isa()) { - assert(args.size() == 7); - return builtin::call_reduce_op_query_format( - args[0], args[1], args[2], args[3], args[4], args[5], args[6]); - } else if (op->isa()) { - assert(args.size() == 7); - return builtin::call_tensor_view_op_query_format( - args[0], args[1], args[2], args[3], args[4], args[5], args[6]); - } else if (op->isa()) { - assert(args.size() == 11); - return builtin::call_select_op_query_format( - args[0], args[1], args[2], args[3], args[4], args[5], args[6], args[7], - args[8], args[9], args[10]); - } else if (op->isa()) { - return builtin::call_padding_op_query_format( - args[0], args[1], args[2], args[3], args[4], args[5], args[6]); - } else if (op->isa()) { - return builtin::call_pooling_op_query_format( - args[0], args[1], args[2], args[3], args[4], args[5], args[6]); - } else { - COMPILE_ASSERT(false, "unsupported op query function: " << op->op_name_); - } - return expr(); -} - -void create_internal_dispatch_funcs_by_node( - const context_ptr &ctx, ir_module_ptr &ret_mod, - const std::string &table_name, const sc_op_ptr &node, - const std::shared_ptr &use_mtp) { - if (node->isa()) { - node->stc_cast()->create_internal_dispatch_funcs( - ctx, ret_mod, use_mtp); - } else { - auto internal_keys = node->get_internal_dispatch_key_set(ctx); - std::vector op_dispatch_kernel; - int dyn_idx = 0; - internal_keys->for_each_key_process( - std::bind(create_dispatch_funcs_by_keys, ctx, std::ref(ret_mod), - table_name, node, std::placeholders::_1, - std::ref(op_dispatch_kernel[node->logical_op_id_]), - std::ref(dyn_idx), use_mtp, - /*internal*/ true)); - } -} - -void create_dispatch_funcs_by_keys( - const context_ptr &ctx, ir_module_ptr &ret_mod, - const std::string &table_name, const sc_op_ptr &node, - const op_dispatch_key_base_t *key, expr &op_dispatch_kernel, int &dyn_idx, - const std::shared_ptr &use_mtp, bool internal) { - auto cur_table = ret_mod->get_op_table_map()[table_name]; - assert(cur_table); - bool should_compile_later = false; - if (!should_compile_later) { - // we compile the first format specialization in main module - key->set_op_dispatch_key(node, ctx); - auto mod = internal ? node->get_internal_func(ctx) : node->get_func(ctx); - auto func = mod->get_entry_func(); - if (internal) { - func->name_ += "_internal"; - } - func->name_ += "_" + std::to_string(dyn_idx); - func->decl_->name_ = func->name_; - if (!dyn_idx && !internal) { - // mark the first function as prototype. - op_dispatch_kernel->attr().set("prototype", func); - } - ret_mod->merge(*mod); - add_dispatch_symbol_to_kernel_table( - cur_table, key, op_dispatch_tables_t::op_func_info{func->name_}); - } - dyn_idx++; -} - -int get_num_of_internal_funcs(const sc_op_ptr &node) { - if (node->isa()) { - auto mixed_op = node->stc_cast(); - int ret = 0; - for (auto &op : mixed_op->sub_graph_.ops_) { - if (op->need_dynamic_internal_query()) { - ret++; - } - } - return ret; - } else if (node->need_dynamic_internal_query()) { - return 1; - } - return 0; -} - -int get_num_of_internal_funcs(const sc_graph_t &graph) { - int ret = 0; - for (auto &op : graph.ops_) { - if (op->need_dynamic_internal_query()) { - ret++; - } - } - return ret; -} - -int count_dynamic_dims(const sc_dims &in) { - int ret = 0; - for (auto &dim : in) { - if (is_dynamic_dim(dim)) { - ret++; - } - } - return ret; -} - -expr divide_and_ceil(const expr &v, const expr &d) { - return do_cast_and_fold((v + d - 1) / d); -} - -bool is_dyn_specific_graph(sc_graph_t &graph) { - if (graph.dyn_info_ && graph.dyn_info_->is_specific_) { - return true; - } - return false; -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/dynamic_utils.hpp b/legacy/core/src/compiler/ir/graph/dynamic_utils.hpp deleted file mode 100644 index 65c419d62..000000000 --- a/legacy/core/src/compiler/ir/graph/dynamic_utils.hpp +++ /dev/null @@ -1,192 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_DYNAMIC_UTILS_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_DYNAMIC_UTILS_HPP -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -class sc_op; -class sc_graph_t; -struct graph_tensor; -struct op_dispatch_key_base_t; -struct combined_op_dispatch_key_t; -struct dispatch_key_set_base_t; -struct sc_data_format_t; -class ir_module_t; -struct context_t; -// op tables used in ir_module -struct op_dispatch_tables_t { - // format table: - // for tunable op: input format keys => out formats value - // for reorder op: the field is meaningless. - // for other ops: input format keys with only one known format => all in/out - // formats value - std::unordered_map, - std::vector> - format_table_; - /** - * the struct to hold info of a partial specialization instance of a dynamic - * shaped op. - * If graph_ is null, the field `name_or_postfix_` is the only - * valid field of `op_func_info`. `name_or_postfix_` indicates the name of - * TIR function in the IR module for the partial specialization. - * If graph_ is not null, the partial specialization has not been compiled - * when `lower_graph()` is called. The field `op_` contains the dynamic - * shaped op, whose container is stored in `graph_`. The `name_or_postfix_` - * is the postfix for the TIR function name of the partial specialization - */ - struct op_func_info { - std::shared_ptr graph_; - std::shared_ptr op_; - std::string name_or_postfix_; - std::shared_ptr ctx_; - std::shared_ptr use_managed_tp_; - bool internal_; - // only valid when `!already_compiled()`. Lower the partial - // specialization to TIR - std::shared_ptr lower(); - /** - * op_func_info which indicates that the partial specialization is not - * compiled or lowered yet, internal means the internal functions of op. - */ - op_func_info( - const std::shared_ptr &graph, - const std::shared_ptr &op, const std::string &name_or_postfix, - const std::shared_ptr &ctx, - const std::shared_ptr &use_managed_tp, - bool internal = false) - : graph_(graph), op_(op), name_or_postfix_(name_or_postfix), ctx_(ctx), - use_managed_tp_(use_managed_tp), internal_(internal) {} - /** - * op_func_info which indicates that the partial specialization is - * already lowered to TIR in the current IR module - */ - op_func_info(const std::string &name_or_postfix) - : name_or_postfix_(name_or_postfix), internal_(false) {} - bool already_compiled() { return graph_ == nullptr; } - }; - // config table: configs of tunable_op => impl kind - std::unordered_map, int> impl_kind_table_; - // kernel table: in/out format keys => function symbol - std::unordered_map, op_func_info> - kernel_table_; - reflection::shared_general_object_t op_info_; -}; - -using op_dispatch_tables_ptr = std::shared_ptr; -using dispatch_table_map_t = - std::unordered_map; - -void initialize_format_table_with_op(const std::shared_ptr &op, - op_dispatch_tables_ptr &tb); -void initialize_impl_kind_table_with_op(const std::shared_ptr &ctx, - const std::shared_ptr &op, - op_dispatch_tables_ptr &tb); -void initialize_dispatch_table_with_op(const std::shared_ptr &ctx, - const std::shared_ptr &op, - op_dispatch_tables_ptr &tb); -void add_dispatch_symbol_to_kernel_table( - op_dispatch_tables_ptr &tb, const op_dispatch_key_base_t *keys, - op_dispatch_tables_t::op_func_info &&func_module); -bool is_internal_op(const std::shared_ptr &op); -bool can_op_be_dispatched(const std::shared_ptr &op); -bool can_op_query_output(const std::shared_ptr &op); -bool can_op_be_queried(const std::shared_ptr &op); -std::vector> -get_dispatch_set_vec_from_ops(const std::vector> &ops); - -// find the parent node who could be dispatched related to current op, parent -// node could be tuanble op/reorder op/input op. -std::shared_ptr -find_parent_dispatch_node(const std::shared_ptr &in); - -constexpr const int no_link_idx = -1; -using op_layout_link_vec_t = std::vector>>; -// The first dim is the op index who has effective dispatch keys inside fused -// op, the second dim is op's in/out format index, the value is linked op index -// and linked op in/out format index pair. -op_layout_link_vec_t get_op_layout_link_relationships( - const std::vector> &ops, - const std::vector> &dispatch_keys, - const std::shared_ptr &modified_inp); -// Query function order, if we meet reorder, query its next tunable op first. -void lower_query_function( - std::vector &visited, const std::shared_ptr &node, - const std::function &)> &callback); - -void visit_fused_graph_by_query_order( - sc_graph_t &graph, - const std::function &)> &callback); -// Judge whether the two input layout could be linked. The linked means the -// graph is in the valid status of layout. For example, pattern like "reorder + -// matmul", the output layout of reorder should be equal to the input layout of -// matmul. And blocking factor =1 is prepared for binary elementwise op with -// broadcast semantic. For example, MKmk(32, 16) and MKmk(1, 16) are linked, -// while MKmk(32,16) and MKmk(1, 32) not. -bool is_linked_layout(const sc_data_format_t &layout1, - const sc_data_format_t &layout2); -std::vector> -get_graph_inner_dispatch_ops(sc_graph_t &graph, int *total_num_key); -void update_graph_format_by_key( - const std::shared_ptr &ctx, - const std::shared_ptr &fused_op, sc_graph_t &graph, - const combined_op_dispatch_key_t &key, int &key_idx, - size_t node_input_offset, size_t graph_input_offset, - const std::shared_ptr &modified_inp = nullptr); -expr call_op_dynamic_query_function(const std::shared_ptr &op, - const std::vector &args); -void create_internal_dispatch_funcs_by_node( - const std::shared_ptr &ctx, - std::shared_ptr &ret_mod, const std::string &table_name, - const std::shared_ptr &node, - const std::shared_ptr &use_mtp); -void create_dispatch_funcs_by_keys( - const std::shared_ptr &ctx, - std::shared_ptr &ret_mod, const std::string &table_name, - const std::shared_ptr &node, const op_dispatch_key_base_t *key, - expr &op_dispatch_kernel, int &dyn_idx, - const std::shared_ptr &use_mtp, bool internal); -int get_num_of_internal_funcs(const std::shared_ptr &node); -int get_num_of_internal_funcs(const sc_graph_t &graph); -int count_dynamic_dims(const sc_dims &in); - -namespace runtime { -struct dynamic_tensor_t; -} -SC_API runtime::dynamic_tensor_t -convert_graph_tensor_to_dynamic_tensor(const std::shared_ptr &in, - void *data_ptr = nullptr, - sc_dim *shape_ptr = nullptr); -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl -#endif diff --git a/legacy/core/src/compiler/ir/graph/fused_op.cpp b/legacy/core/src/compiler/ir/graph/fused_op.cpp deleted file mode 100644 index f94f0edcf..000000000 --- a/legacy/core/src/compiler/ir/graph/fused_op.cpp +++ /dev/null @@ -1,1224 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "fused_op.hpp" -#include "anchor_loop_generator.hpp" -#include "fusible_op_utils.hpp" -#include "lowering.hpp" -#include "pass/pass.hpp" -#include "tunable_op.hpp" -#include "visitor.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -SC_MODULE(graph.fused_op) - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -fusion_partition_t *fusion_partition_t::get_root() const { - if (merged_to) { - assert(ops.empty()); - return merged_to->get_root(); - } - return const_cast(this); -} - -bool fusion_partition_t::is_ok_to_add(sc_op *op, - const op_dep_matrix_t &g) const { - if (merged_to) { - return get_root()->is_ok_to_add(op, g); - } - for (auto &in : op->get_inputs()) { - // if the input is in the partition, don't need to check - if (contains(in->producer_owner_)) { - continue; - } - // if "in" depends on an op in the partition - if (main_tunable_op && g.lookup(main_tunable_op->logical_op_id_, - in->producer_owner_->logical_op_id_) == 1) { - return false; - } - for (auto &op_in_set : ops) { - auto result = g.lookup(op_in_set->logical_op_id_, - in->producer_owner_->logical_op_id_); - // if "in" depends on an op in the partition - if (result == 1) { - return false; - } - } - } - return true; -} - -bool fusion_partition_t::contains(sc_op *op) const { - if (merged_to) { - return get_root()->contains(op); - } - return op == main_tunable_op.get() || - ops.find(op->shared_from_this()) != ops.end(); -} - -// merge the ops in "other" to "this" -void fusion_partition_t::merge(const ptr &other) const { - auto ths_root = get_root(); - auto other_root = other->get_root(); - if (ths_root == other_root) { - return; - } - ths_root->ops.insert(other_root->ops.begin(), other_root->ops.end()); - assert(other_root->main_tunable_op == nullptr); - other_root->ops.clear(); - other_root->merged_to = ths_root->shared_from_this(); -} - -namespace graph { -void mark_read_or_write_buffers(std::vector &args, bool is_read) { - const char *name = is_read ? "read_buffer" : "write_buffer"; - for (auto &tsr : args) { - tsr->attr()[name] = true; - } -} - -func_t create_func_decl_for_op(sc_op *op, std::vector &ins, - std::vector &outs) { - auto &graph = op->get_owner_graph(); - ins = ins.empty() ? graph::tensor_detail_to_ir_tensor(graph, "__ins_", - op->get_inputs()) - : ins; - outs = outs.empty() ? graph::tensor_detail_to_ir_tensor(graph, "__outs_", - op->get_outputs()) - : outs; - graph::mark_read_or_write_buffers(ins, true); - graph::mark_read_or_write_buffers(outs, false); - std::vector args = outs; - args.insert(args.end(), ins.begin(), ins.end()); - std::string func_name; - if (auto layer_name = - op->attrs_.get_or_null(op_attr_key::layer_name)) { - COMPILE_ASSERT(!layer_name->empty() && isalpha(layer_name->front()), - "Bad layername: " << *layer_name); - func_name = *layer_name; - } else { - func_name = op->op_name_; - } - func_name += "__"; - func_name += std::to_string(op->logical_op_id_); - auto func = builder::make_func(func_name, args, - make_stmt(std::vector()), - datatypes::boolean); - // func->attr()["Gflop"] = gen_ptr->get_gflop(); - return func; -} - -func_t create_query_func_decl_for_op( - sc_op *op, std::vector &ins, std::vector &ori_ins, - std::vector &outs, std::vector &in_fmts, - std::vector &ori_in_fmts, std::vector &out_fmts, - std::vector &out_sizes, expr &kernel) { - func_t func = create_func_decl_for_op(op, ins, outs); - in_fmts.resize(ins.size()); - out_fmts.resize(outs.size()); - out_sizes.resize(outs.size()); - assert(op->isa()); - if ((op->isa() && !op->stc_cast() - ->get_internal_tunable_input_indices() - .empty())) { - size_t ori_sz = op->stc_cast() - ->get_internal_tunable_input_indices() - .size(); - auto &graph = op->get_owner_graph(); - for (size_t i = 0; i < ori_sz; i++) { - auto ori_in = graph::tensor_detail_to_ir_tensor( - graph, std::string("__ori_ins_") + std::to_string(i), - op->get_inputs()[i]->details_); - ori_in->attr().set(attr_keys::always_trans, true); - ori_ins.emplace_back(ori_in); - ori_in_fmts.emplace_back( - builder::make_tensor(ori_in.static_as()->name_ + "_format", - {UINT64_C(1)}, datatypes::index)); - } - } - std::transform(ins.begin(), ins.end(), in_fmts.begin(), [](const expr &in) { - in->attr().set(attr_keys::always_trans, true); - return builder::make_tensor(in.static_as()->name_ + "_format", - {UINT64_C(1)}, datatypes::index); - }); - std::transform( - outs.begin(), outs.end(), out_fmts.begin(), [](const expr &in) { - in->attr().set(attr_keys::always_trans, true); - return builder::make_tensor(in.static_as()->name_ + "_format", - {UINT64_C(1)}, datatypes::index); - }); - std::transform( - outs.begin(), outs.end(), out_sizes.begin(), [](const expr &in) { - return builder::make_tensor(in.static_as()->name_ + "_size", - {UINT64_C(1)}, datatypes::index); - }); - // table should be get from module global data, here is only for the - // alignment to other functions. - auto table = builder::make_var(datatypes::pointer, "dummy_table"); - std::vector args = func->params_; - args.insert(args.begin(), table); - args.insert(args.end(), ori_ins.begin(), ori_ins.end()); - args.insert(args.end(), out_fmts.begin(), out_fmts.end()); - args.insert(args.end(), in_fmts.begin(), in_fmts.end()); - args.insert(args.end(), ori_in_fmts.begin(), ori_in_fmts.end()); - args.insert(args.end(), out_sizes.begin(), out_sizes.end()); - kernel = builder::make_tensor( - "func_kernel", {1 + get_num_of_internal_funcs(op->shared_from_this())}, - datatypes::index); - args.push_back(kernel); - func->params_ = args; - func->name_ = std::string("query_format_") + func->name_; - return func; -} -} // namespace graph -static std::atomic out_idx(0); - -struct fused_exprs_t { - expr dummy_size; - expr dummy_kernel; - expr combined_keys; - expr combined_algs; -}; - -struct general_fused_params_t { - builder::ir_builder_t &bld; - ir_module_ptr modu; - sc_graph_t &graph; - sc_op_ptr node; - std::unordered_map <sr_rtsr; - std::unordered_map &fmgr_2_orig; - std::unordered_map &visited; - int &inner_tsr_count; - int &cur_combined_op_idx; - int &cur_combined_key_idx; - int &cur_ori_inp_idx; - int &cur_internal_idx; - fused_exprs_t exprs; -}; - -tsr_info_t get_or_create_tsr_and_fmt(general_fused_params_t &gp, - const graph_tensor_ptr &in) { - auto it = gp.ltsr_rtsr.find(in); - if (it != gp.ltsr_rtsr.end()) { - return it->second; - } - auto &bld = gp.bld; - auto rtsr = - builder::make_tensor("tsr_" + std::to_string(gp.inner_tsr_count), - {sizeof(runtime::dynamic_tensor_t)}, datatypes::u8); - auto shape_tsr = builder::make_tensor( - "dyn_shape_tsr_" + std::to_string(gp.inner_tsr_count), - {in->details_.get_plain_dims().size()}, datatypes::index); - shape_tsr->attr().set(attr_keys::no_dead_write, true); - shape_tsr->attr().set(attr_keys::no_tensor2var, true); - bool fmt_init = in->details_.get_format_candidates().size() <= 1; - auto out_fmt = builder::make_tensor( - "format_" + std::to_string(gp.inner_tsr_count), {1UL}, datatypes::index); - bld.push_var_tensor_def(rtsr); - bld.push_var_tensor_def(shape_tsr); - bld.push_evaluate( - builder::make_write_struct(rtsr, shape_tsr, dyn_tsr_struct_t::name, - dyn_tsr_struct_t::fields::dim_ptr)); - bld.push_evaluate(builder::make_write_struct( - rtsr, - builder::make_constant({in->details_.get_plain_dims().size()}, - datatypes::s32), - dyn_tsr_struct_t::name, dyn_tsr_struct_t::fields::ndims)); - uint64_t etype = - in->details_.dtype_.is_etype_pointer() - ? in->details_.dtype_.get_pointer_element().as_etype_int() - : in->details_.dtype_.as_etype_int(); - bld.push_evaluate(builder::make_write_struct( - rtsr, builder::make_constant({etype}, datatypes::u32), - dyn_tsr_struct_t::name, dyn_tsr_struct_t::fields::dtype)); - auto plain_shapes = - gp.node->get_owner_graph().dims_to_expr(in->details_.get_plain_dims()); - uint64_t dyn_mask_int = 0; - for (size_t i = 0; i < plain_shapes.size(); i++) { - bld.push_assign(builder::make_indexing(shape_tsr, {i}), plain_shapes[i]); - dyn_mask_int |= (uint64_t(!plain_shapes[i].isa()) << i); - } - bld.push_evaluate(builder::make_write_struct( - rtsr, builder::make_constant({dyn_mask_int}, datatypes::u8), - dyn_tsr_struct_t::name, dyn_tsr_struct_t::fields::dyn_mask)); - uint64_t init_format = 0; - if (fmt_init) { - init_format = uint64_t(in->details_.get_format().to_runtime()); - } - bld.push_var_tensor_def(out_fmt); - bld.push_assign(builder::make_indexing(out_fmt, {0}), init_format); - - gp.inner_tsr_count++; - auto ret = tsr_info_t(rtsr, expr(), out_fmt, gp.exprs.dummy_size); - gp.ltsr_rtsr[in] = ret; - return ret; -} - -static bool need_inner_query(general_fused_params_t &gp, const sc_op_ptr &node, - int &main_idx) { - auto &inputs = node->get_inputs(); - auto &outputs = node->get_outputs(); - // check if the op is associated with const. - for (size_t i = 0; i < inputs.size(); i++) { - auto &in = inputs[i]; - // original ltensor is legal and is constant - if (!gp.visited[in]) { - return true; - } - auto it = gp.fmgr_2_orig.find(in); - if (it != gp.fmgr_2_orig.end() && !it->second->uses_.empty() && - it->second->producer_owner_ && - it->second->producer_owner_->attrs_.get_or_else( - "constant", const_kind::not_const) != const_kind::not_const) { - return true; - } - if (!in->uses_.empty() && in->producer_owner_ && - in->producer_owner_->isa()) { - return true; - } - } - if (node->isa()) { - int bc_idx = - node->stc_cast()->get_broadcast_input(); - main_idx = bc_idx == -1 ? 0 : 1 - bc_idx; - } - // check the op is linked to output, need to query output size. - for (size_t i = 0; i < outputs.size(); i++) { - auto &out = outputs[i]; - for (size_t j = 0; j < out->uses_.size(); j++) { - if (out->uses_[j].second->isa()) { - return true; - } - } - } - return false; -} - -void update_op_visited(general_fused_params_t &gp, const sc_op_ptr &node) { - for (auto &in : node->get_inputs()) { - gp.visited[in] = true; - } - for (auto &out : node->get_outputs()) { - gp.visited[out] = true; - } -} - -void add_global_table_var(general_fused_params_t &gp, - const std::string &table_name, - const op_dispatch_tables_ptr &table_ptr, - const expr &table_var) { - gp.modu->add_op_table(std::make_pair(table_name, table_ptr)); - auto table_def = builder::make_var_tensor_def_unattached( - table_var, linkage::private_global); - gp.modu->add_global_var(table_def.checked_as()); -} - -void declare_dummy_and_combined_tsrs(general_fused_params_t &gp, - int total_key_num, int dispatch_op_num) { - auto &bld = gp.bld; - bld.push_scope(); - // create dummy tensor/var for inner query. - gp.exprs.dummy_kernel = builder::make_var(datatypes::pointer, "dummy_kernel"); - bld.push_var_tensor_def(gp.exprs.dummy_kernel, linkage::local); - gp.exprs.dummy_size = - builder::make_tensor("dummy_size", {1}, datatypes::index); - bld.push_var_tensor_def(gp.exprs.dummy_size, linkage::local); - if (total_key_num) { - expr combined_keys = builder::make_tensor("combined_keys", {total_key_num}, - datatypes::pointer); - gp.exprs.combined_keys = combined_keys; - bld.push_var_tensor_def(combined_keys); - } - if (dispatch_op_num) { - expr combined_algs = builder::make_tensor( - "combined_algs", {dispatch_op_num}, datatypes::s32); - gp.exprs.combined_algs = combined_algs; - bld.push_var_tensor_def(combined_algs); - } -} - -void set_original_tensor_and_format_for_tunables( - general_fused_params_t &gp, sc_op *node_before, - const std::vector &ori_ins, const std::vector &ori_in_fmts, - expr &ori_tsr, expr &ori_fmt) { - tsr_info_t tsr_info; - if (node_before->isa()) { - tsr_info = get_or_create_tsr_and_fmt(gp, node_before->get_inputs()[0]); - } else { - auto ltsr = node_before->get_outputs()[0]; - auto it = gp.fmgr_2_orig.find(ltsr); - // if find in fmgr_2_orig, it is a input op, else is a internal ltsr. - if (it != gp.fmgr_2_orig.end()) { - assert(node_before->isa()); - ltsr = it->second; - } - tsr_info = get_or_create_tsr_and_fmt(gp, ltsr); - } - ori_tsr = tsr_info.tensor_; - ori_fmt = tsr_info.format_; -} - -void create_query_function_by_graph( - general_fused_params_t &gp, const expr &kernel, - const std::vector &ori_ins, const std::vector &ori_in_fmts, - std::vector &each_op_num_keys, int total_key_num, int dispatch_op_num, - const std::vector &tunable_inp_indices = std::vector()) { - auto &bld = gp.bld; - auto combined_keys = gp.exprs.combined_keys; - auto combined_algs = gp.exprs.combined_algs; - auto &cur_combined_key_idx = gp.cur_combined_key_idx; - auto &cur_combined_op_idx = gp.cur_combined_op_idx; - auto &cur_internal_idx = gp.cur_internal_idx; - std::vector table_names(gp.graph.ops_.size()); - auto create_internal_query_func = [&](const sc_op_ptr &op) { - // Can not use can_op_be_dispatched as tsr and format need - // pass through each op. - if (op->isa() || op->isa() || - op->isa()) { - return; - } - expr dummy_kernel = gp.exprs.dummy_kernel; - expr dummy_size = gp.exprs.dummy_size; - auto ctx = gp.modu->ctx_; - int main_idx = 0; - size_t in_size = op->get_inputs().size(); - size_t out_size = op->get_outputs().size(); - auto table_name = gp.node->op_name_ + "__" + - std::to_string(gp.node->logical_op_id_) + "_inner__" + - std::to_string(op->logical_op_id_) + "_table"; - if (op->info_.internal_info_) { - op->info_.internal_info_->dispatch_table_name_ = table_name; - } - auto table_var = builder::make_var(datatypes::pointer, table_name); - auto table_ptr = std::make_shared(); - std::vector op_outs(out_size), op_ins(in_size); - for (size_t i = 0; i < out_size; i++) { - op_outs[i] = get_or_create_tsr_and_fmt(gp, op->get_outputs()[i]); - } - for (size_t i = 0; i < in_size; i++) { - op_ins[i] = get_or_create_tsr_and_fmt(gp, op->get_inputs()[i]); - } - if (op->isa() || - op->isa() || - op->isa()) { - auto table_ptr = std::make_shared(); - // create origin tsr and dispatch key for tunable ops - expr ori_in0, ori_in1, ori_in_fmt0, ori_in_fmt1; - auto node_before_in0 = op->get_inputs()[0]->producer_owner_; - auto node_before_in1 = op->get_inputs()[1]->producer_owner_; - set_original_tensor_and_format_for_tunables( - gp, node_before_in0, ori_ins, ori_in_fmts, ori_in0, ori_in_fmt0); - set_original_tensor_and_format_for_tunables( - gp, node_before_in1, ori_ins, ori_in_fmts, ori_in1, ori_in_fmt1); - add_global_table_var(gp, table_name, table_ptr, table_var); - auto internal_kernel = - op->need_dynamic_internal_query() - ? builder::tensor_ptr(kernel, {cur_internal_idx}) - : make_expr(UINT64_C(0), datatypes::pointer); - std::vector args = { - table_var, - op_outs[0].tensor_, - op_ins[0].tensor_, - op_ins[1].tensor_, - ori_in0, - ori_in1, - op_outs[0].format_, - op_ins[0].format_, - op_ins[1].format_, - ori_in_fmt0, - ori_in_fmt1, - op_outs[0].size_, - internal_kernel, - builder::tensor_ptr(combined_algs, {cur_combined_op_idx})}; - bld.push_evaluate(call_op_dynamic_query_function(op, args)); - initialize_dispatch_table_with_op(ctx, op, table_ptr); - // set combined tensor - bld.push_assign( - builder::make_indexing(combined_keys, {cur_combined_key_idx++}), - op_ins[0].format_); - bld.push_assign( - builder::make_indexing(combined_keys, {cur_combined_key_idx++}), - op_ins[1].format_); - bld.push_assign( - builder::make_indexing(combined_keys, {cur_combined_key_idx++}), - op_outs[0].format_); - each_op_num_keys[cur_combined_op_idx] = 3; - cur_combined_op_idx++; - if (op->isa()) { - cur_internal_idx++; - } - } else if (op->isa()) { - if (need_inner_query(gp, op, main_idx)) { - add_global_table_var(gp, table_name, table_ptr, table_var); - initialize_dispatch_table_with_op(ctx, op, table_ptr); - std::vector args = {table_var, op_outs[0].tensor_, - op_ins[0].tensor_, op_outs[0].format_, - op_ins[0].format_, op_outs[0].size_, - dummy_kernel}; - bld.push_evaluate(call_op_dynamic_query_function(op, args)); - } else { - auto &out = op->get_outputs()[0]; - gp.ltsr_rtsr[out] = op_ins[main_idx]; - } - } else if (op->isa()) { - add_global_table_var(gp, table_name, table_ptr, table_var); - initialize_dispatch_table_with_op(ctx, op, table_ptr); - std::vector args = {table_var, op_outs[0].tensor_, - op_ins[0].tensor_, op_outs[0].format_, - op_ins[0].format_, op_outs[0].size_, - dummy_kernel}; - bld.push_evaluate(call_op_dynamic_query_function(op, args)); - } else if (op->isa()) { - add_global_table_var(gp, table_name, table_ptr, table_var); - initialize_dispatch_table_with_op(ctx, op, table_ptr); - std::vector args = {table_var, op_outs[0].tensor_, - op_ins[0].tensor_, op_outs[0].format_, - op_ins[0].format_, op_outs[0].size_, - dummy_kernel}; - bld.push_evaluate(call_op_dynamic_query_function(op, args)); - } else if (op->isa()) { - if (need_inner_query(gp, op, main_idx)) { - add_global_table_var(gp, table_name, table_ptr, table_var); - initialize_dispatch_table_with_op(ctx, op, table_ptr); - std::vector args = { - table_var, op_outs[0].tensor_, op_ins[0].tensor_, - op_ins[1].tensor_, op_outs[0].format_, op_ins[0].format_, - op_ins[1].format_, op_outs[0].size_, dummy_kernel}; - bld.push_evaluate(call_op_dynamic_query_function(op, args)); - } else { - auto &out = op->get_outputs()[0]; - gp.ltsr_rtsr[out] = op_ins[main_idx]; - } - } else if (op->isa()) { - // Currently reorder is the last op of fusion pattern, so always - // query. - add_global_table_var(gp, table_name, table_ptr, table_var); - std::vector args = { - table_var, - op_outs[0].tensor_, - op_ins[0].tensor_, - op_outs[0].format_, - op_ins[0].format_, - op_outs[0].size_, - dummy_kernel, - builder::tensor_ptr(combined_algs, {cur_combined_op_idx})}; - bld.push_evaluate(call_op_dynamic_query_function(op, args)); - // set combined key tensor - bld.push_assign( - builder::make_indexing(combined_keys, {cur_combined_key_idx++}), - op_ins[0].format_); - bld.push_assign( - builder::make_indexing(combined_keys, {cur_combined_key_idx++}), - op_outs[0].format_); - each_op_num_keys[cur_combined_op_idx] = 2; - cur_combined_op_idx++; - } else if (op->isa()) { - // always query - add_global_table_var(gp, table_name, table_ptr, table_var); - initialize_dispatch_table_with_op(ctx, op, table_ptr); - std::vector args = {table_var, op_outs[0].tensor_, - op_ins[0].tensor_, op_outs[0].format_, - op_ins[0].format_, op_outs[0].size_, - dummy_kernel}; - bld.push_evaluate(call_op_dynamic_query_function(op, args)); - } else if (op->isa()) { - // always query - add_global_table_var(gp, table_name, table_ptr, table_var); - initialize_dispatch_table_with_op(ctx, op, table_ptr); - std::vector args = {table_var, op_outs[0].tensor_, - op_ins[0].tensor_, op_outs[0].format_, - op_ins[0].format_, op_outs[0].size_, - dummy_kernel}; - bld.push_evaluate(call_op_dynamic_query_function(op, args)); - } else if (op->isa()) { - add_global_table_var(gp, table_name, table_ptr, table_var); - initialize_dispatch_table_with_op(ctx, op, table_ptr); - std::vector args = { - table_var, op_outs[0].tensor_, op_ins[0].tensor_, - op_ins[1].tensor_, op_ins[2].tensor_, op_outs[0].format_, - op_ins[0].format_, op_ins[1].format_, op_ins[2].format_, - op_outs[0].size_, dummy_kernel}; - bld.push_evaluate(call_op_dynamic_query_function(op, args)); - } else if (op->isa()) { - // do nothing - } else { - COMPILE_ASSERT(false, "Currently dynamic fusbile op only support " - "unary/binary."); - } - update_op_visited(gp, op); - }; - visit_fused_graph_by_query_order(gp.graph, create_internal_query_func); - - // final query the fused op kernel. - assert(gp.cur_combined_key_idx == total_key_num && - gp.cur_combined_op_idx == dispatch_op_num); - assert(gp.cur_internal_idx - 1 == get_num_of_internal_funcs(gp.node)); - auto main_table_name = gp.node->op_name_ + "__" + - std::to_string(gp.node->logical_op_id_) + "_ptr_table"; - auto main_table_var = builder::make_var(datatypes::pointer, main_table_name); - auto main_table_ptr = std::make_shared(); - add_global_table_var(gp, main_table_name, main_table_ptr, main_table_var); - expr each_op_num_keys_tsr = - builder::make_tensor("each_op_num_keys", {dispatch_op_num}, - datatypes::s32, address_space::automatic, - std::make_shared(each_op_num_keys)); - gp.modu->add_global_var(builder::make_var_tensor_def_unattached( - each_op_num_keys_tsr, linkage::private_global) - .static_as()); - bld.push_evaluate(builtin::call_fused_op_query_combined( - main_table_var, combined_keys.defined() ? combined_keys : get_ir_null(), - combined_algs.defined() ? combined_algs : get_ir_null(), - each_op_num_keys_tsr, dispatch_op_num, kernel)); -} - -mixed_fuse_op_t::mixed_fuse_op_t( - const std::string &name, const std::vector &parti_list, - const ir_module_ptr &mod, const sc_graph_t &graph, - const std::vector &ins, - const std::vector &outs, const any_map_t &attrs) { - info_.inputs_ = ins; - info_.outputs_ = outs; - parti_list_ = parti_list; - mod_ = mod; - sub_graph_ = copy_graph(graph); - op_name_ = name; - attrs_ = attrs; -} - -bool mixed_fuse_op_t::need_dynamic_internal_query_impl() const { - return !std::all_of( - sub_graph_.ops_.begin(), sub_graph_.ops_.end(), - [](const sc_op_ptr &op) { return !op->need_dynamic_internal_query(); }); -} - -ir_module_ptr mixed_fuse_op_t::get_func(context_ptr ctx) { - func_t func; - bool use_cache = - get_owner_graph().attrs_.get_or_else("temp.force_static", false); - ir_module_ptr modu; - if (!use_cache && can_op_be_dispatched(shared_from_this())) { - modu = std::make_shared(ctx); - // max fusion policy don't need any conditions. - expr max_loop_parallelism_cond; - func_t max_fusion_func, max_loop_parallel_func; - ir_module_ptr max_fusion_modu, max_loop_parallel_modu; - std::vector ins, outs; - func = graph::create_func_decl_for_op(this, ins, outs); - outs.insert(outs.end(), ins.begin(), ins.end()); - std::for_each(outs.begin(), outs.end(), [](const expr &arg) { - arg->attr().set(attr_keys::always_trans, true); - }); - if (need_dynamic_internal_query()) { - auto internal_func_arg = builder::make_tensor( - "extra_internal_funcs", {get_num_of_internal_funcs(sub_graph_)}, - datatypes::index); - func->params_.emplace_back(internal_func_arg); - func->decl_->params_.emplace_back(internal_func_arg); - outs.emplace_back(internal_func_arg); - } - func->name_ = op_name_; - func->decl_->name_ = op_name_; - func->name_ += "_" + std::to_string(logical_op_id_); - func->decl_->name_ += "_" + std::to_string(logical_op_id_); - auto return_stmt = builder::make_returns_unattached(true); - stmt policy_dispatch; - { - // max_loop_parallelism policy - auto cpy_graph = copy_graph(sub_graph_); - cpy_graph.attrs_.set("temp.dynamic_fusion_policy", - dynamic_fusion_policy_t::max_loop_parallelism); - mixed_partition(cpy_graph, ctx); - std::vector lower_args(cpy_graph.get_output_ops()); - auto input_ops = cpy_graph.get_input_ops(); - lower_args.insert(lower_args.end(), input_ops.begin(), input_ops.end()); - cpy_graph.attrs_.set("temp.force_static", true); - max_loop_parallel_modu = lower_graph(ctx, cpy_graph, lower_args, false); - max_loop_parallel_func = max_loop_parallel_modu->get_entry_func(); - max_loop_parallel_func->name_ = op_name_; - max_loop_parallel_func->decl_->name_ = op_name_; - max_loop_parallel_func->name_ += - "_max_loop_parallism_" + std::to_string(logical_op_id_); - max_loop_parallel_func->decl_->name_ += - "_max_loop_parallism_" + std::to_string(logical_op_id_); - max_loop_parallelism_cond = - cpy_graph.attrs_.get("temp.fusion_policy_condition"); - schedule_loops(max_loop_parallel_func->body_); - } - // if condition is true or false after simplify, keep only one module - // for less functions. - max_loop_parallelism_cond = do_cast_and_fold(max_loop_parallelism_cond); - if (max_loop_parallelism_cond->equals(expr(true))) { - modu->merge(*max_loop_parallel_modu); - policy_dispatch = builder::make_evaluate_unattached( - builder::make_call(max_loop_parallel_func->decl_, outs)); - } else { - // max_fusion policy - auto cpy_graph = copy_graph(sub_graph_); - cpy_graph.attrs_.set("temp.dynamic_fusion_policy", - dynamic_fusion_policy_t::max_fusion); - mixed_partition(cpy_graph, ctx); - std::vector lower_args(cpy_graph.get_output_ops()); - auto input_ops = cpy_graph.get_input_ops(); - lower_args.insert(lower_args.end(), input_ops.begin(), input_ops.end()); - cpy_graph.attrs_.set("temp.force_static", true); - max_fusion_modu = lower_graph(ctx, cpy_graph, lower_args, false); - max_fusion_func = max_fusion_modu->get_entry_func(); - max_fusion_func->name_ = op_name_; - max_fusion_func->name_ += "_max_fusion_" + std::to_string(logical_op_id_); - max_fusion_func->decl_->name_ += - "_max_fusion_" + std::to_string(logical_op_id_); - schedule_loops(max_fusion_func->body_); - modu->merge(*max_fusion_modu); - if (max_loop_parallelism_cond->equals(expr(false))) { - policy_dispatch = builder::make_evaluate_unattached( - builder::make_call(max_fusion_func->decl_, outs)); - } else { - modu->merge(*max_loop_parallel_modu); - policy_dispatch = builder::make_if_else_unattached( - max_loop_parallelism_cond, - builder::make_evaluate_unattached( - builder::make_call(max_loop_parallel_func->decl_, outs)), - builder::make_evaluate_unattached( - builder::make_call(max_fusion_func->decl_, outs))); - } - } - func->body_ = - builder::make_stmts_unattached({policy_dispatch, return_stmt}); - modu->add_func({func}); - modu->set_entry_func_idx(modu->get_contents().size() - 1); - } else { - // if mod_ is not empty, usually when redo occurs in partition stage. - if (mod_) - return mod_; - COMPILE_ASSERT(parti_list_.size() == 1, - "partition size is expected for 1, but got " - << parti_list_.size()) - func = parti_list_[0]->func_; - func->name_ = op_name_; - func->decl_->name_ = op_name_; - func->name_ += "_" + std::to_string(logical_op_id_); - func->decl_->name_ += "_" + std::to_string(logical_op_id_); - schedule_loops(func->body_); - modu = std::make_shared(ctx); - modu->add_func({func}); - modu->set_entry_func_idx(0); - if (need_dynamic_internal_query()) { - for (auto &parti : parti_list_) { - assert(parti->dyn_inter_); - modu->merge(*parti->dyn_inter_->mod_); - } - } - } - - return modu; -} - -void schedule_loop_body(const stmt &body, node_ptr_map *node_remap) { - stmt target_loop = body; - if (target_loop.isa()) { - auto ss = target_loop.static_as(); - COMPILE_ASSERT(ss->seq_.size() == 1 && ss->seq_[0].isa(), - "for loop node is expected"); - target_loop = ss->seq_[0]; - } - COMPILE_ASSERT(target_loop.isa(), "for loop node is expected"); - for_loop outer_most_loop = target_loop.checked_as(); - outer_most_loop->kind_ = for_type::PARALLEL; - assert(outer_most_loop.defined()); - const int run_threads = runtime_config_t::get().get_num_threads(); - for_loop cur_loop = outer_most_loop; - std::vector loops; - auto fused_number = 1; - while (true) { - if (cur_loop->iter_end_.isa() && - cur_loop->iter_begin_.isa()) { - fused_number *= (get_expr_as_int(cur_loop->iter_end_) - - get_expr_as_int(cur_loop->iter_begin_)); - } - if (fused_number / run_threads > 12 || - (fused_number >= run_threads && (fused_number % run_threads) == 0)) - break; - auto inner_loop = get_inner_for_loop(cur_loop.get()); - if (inner_loop.defined() && !inner_loop->num_threads_ && - (inner_loop->step_.isa() && - get_expr_as_int(inner_loop->step_) == 1) && - !inner_loop->attr().get_or_else(stmt_attr_key::no_loop_fuse, false)) { - node_ptr_map cur_remap; - outer_most_loop->fuse(inner_loop, node_remap ? &cur_remap : nullptr); - cur_loop = inner_loop; - if (node_remap) { - for (auto &exp_m : (*node_remap)) { - auto iter = cur_remap.find(exp_m.second); - if (iter != cur_remap.end()) { - exp_m.second = iter->second; - cur_remap.erase(iter); - } - } - node_remap->insert(cur_remap.begin(), cur_remap.end()); - } - } else { - break; - } - } -} - -static op_traits::may_prefetch_t *find_prefetch_op(const sc_graph_t &graph) { - // fix-me (yijie): should provide a way to find the main op of the partition - // find the first tunable & prefetchable op - op_traits::may_prefetch_t *found_op = nullptr; - op_visitor_t::dfs_topology_sort(graph.ops_.size()) - .visit_graph(graph, [&found_op](op_visitor_t *vis, const sc_op_ptr &op) { - if (found_op) { - return; - } - if (op->isa() && op->isa() && - op->attrs_.get_or_else(mixed_partition_hint::first_prefetch_op, - false)) { - for (auto &ins : op->get_inputs()) { - if (ins->producer_owner_->isa()) { - found_op = op->dyn_cast(); - return; - } - } - } - }); - return found_op; -} - -std::vector -mixed_fuse_op_t::query_prefetch(const context_ptr &ctx, bool is_global, - const std::vector &ins) { - if (auto found_op = find_prefetch_op(sub_graph_)) { - auto ret = found_op->query_prefetch(ctx, is_global, ins); - auto &ins = dynamic_cast(found_op)->get_inputs(); - // check that the inputs to prefetch are from the sub-graph inputs - std::vector indices; - auto graph_inputs = sub_graph_.get_input_ops(); - for (auto in_of_op : ret) { - if (auto the_in_op = - ins.at(in_of_op)->producer_owner_->dyn_cast()) { - // find the index of the input op in graph inputs - int idx = std::find(graph_inputs.begin(), graph_inputs.end(), - the_in_op->shared_from_this()) - - graph_inputs.begin(); - // if idx is not in indices, push - if (std::find(indices.begin(), indices.end(), idx) == indices.end()) { - indices.emplace_back(idx); - } - } - } - return indices; - } - return {}; -} - -void mixed_fuse_op_t::generate_prefetcher_body_for_tensor( - const context_ptr &ctx, const std::vector &func_args, - const std::vector &ins, const std::vector &indices) { - if (auto found_op = find_prefetch_op(sub_graph_)) { - auto graph_inputs = sub_graph_.get_input_ops(); - auto &op_ins = dynamic_cast(found_op)->get_inputs(); - std::vector indices_in_op; - for (auto idx : indices) { - auto input_op_ptr = graph_inputs.at(idx).get(); - for (size_t i = 0; i < op_ins.size(); i++) { - if (op_ins[i]->producer_owner_ == input_op_ptr) { - indices_in_op.emplace_back(i); - } - } - } - found_op->generate_prefetcher_body_for_tensor(ctx, func_args, ins, - indices_in_op); - } -} - -void mixed_fuse_op_t::schedule_loops(const stmt &body) { - if (body.isa()) - schedule_loop_body(body); - else if (body.isa()) { - for (auto &st : body.checked_as()->seq_) { - if (st.isa()) { - schedule_loop_body(st); - } else { - // recursively call - schedule_loops(st); - } - } - } -} - -std::vector mixed_fuse_op_t::get_internal_tunable_input_indices() { - auto graph_input_ops = sub_graph_.get_input_ops(); - std::vector ret; - for (size_t i = 0; i < graph_input_ops.size(); i++) { - auto &uses = graph_input_ops[i]->get_outputs()[0]->uses_; - for (auto &use : uses) { - if (use.second.lock() && use.second->isa()) { - ret.emplace_back(i); - break; - } - } - } - return ret; -} - -dispatch_set_ptr &mixed_fuse_op_t::get_dispatch_key_set() { - if (!info_.dispatch_key_set_) { - int dummy_num; - info_.dispatch_key_set_ = std::make_shared( - get_inner_dispatch_ops(&dummy_num)); - } - return info_.dispatch_key_set_; -} - -std::vector -mixed_fuse_op_t::get_inner_dispatch_ops(int *total_key_num) { - std::vector ret; - if (total_key_num) { - *total_key_num = 0; - } - ret = get_graph_inner_dispatch_ops(sub_graph_, total_key_num); - return ret; -} - -void mixed_fuse_op_t::update_internal_graph_format( - const combined_op_dispatch_key_t &key, const context_ptr &ctx) { - int key_idx = 0; - update_graph_format_by_key(ctx, shared_from_this(), sub_graph_, key, key_idx, - 0, 0); - assert(key_idx == static_cast(key.size())); -} - -ir_module_ptr mixed_fuse_op_t::get_dynamic_query_func(const context_ptr &ctx) { - auto modu = std::make_shared(ctx); - sub_graph_.sync_dynamic_info_with_graph(get_owner_graph()); - std::unordered_map fmgr_2_orig; - auto &node_inputs = get_inputs(); - const auto &graph_input_ops = sub_graph_.get_input_ops(); - const auto &graph_output_ops = sub_graph_.get_output_ops(); - assert(node_inputs.size() == graph_input_ops.size()); - for (size_t i = 0; i < node_inputs.size(); i++) { - fmgr_2_orig[graph_input_ops[i]->get_outputs()[0]] = node_inputs[i]; - } - // inner graph logical tensor visit states, for query pruning. - std::unordered_map visited; - std::vector ins, outs, in_fmts, out_fmts, ori_ins, ori_in_fmts, - out_sizes; - expr main_table_var, kernel; - size_t inp_idx = 0, out_idx = 0; - auto func = graph::create_query_func_decl_for_op(this, ins, ori_ins, outs, - in_fmts, ori_in_fmts, - out_fmts, out_sizes, kernel); - // inner logical tensor => real tensor, out_size and format. - std::unordered_map ltsr_rtsr; - builder::ir_builder_t bld; - int total_key_num = 0; - int inner_tsr_count = 0; - int dispatch_op_num = - static_cast(get_inner_dispatch_ops(&total_key_num).size()); - int cur_combined_op_idx = 0, cur_combined_key_idx = 0, cur_ori_inp_idx = 0, - cur_internal_idx = 1; - // create general params. - general_fused_params_t gp{bld, - modu, - sub_graph_, - shared_from_this(), - ltsr_rtsr, - fmgr_2_orig, - visited, - inner_tsr_count, - cur_combined_op_idx, - cur_combined_key_idx, - cur_ori_inp_idx, - cur_internal_idx, - fused_exprs_t()}; - // construct combined tensors for final query. - std::vector each_op_num_keys(dispatch_op_num, 0); - // build query function body - // declare dummy kernel and dummy size, combined tsrs - declare_dummy_and_combined_tsrs(gp, total_key_num, dispatch_op_num); - for (auto &inp : graph_input_ops) { - auto <sr = inp->get_outputs()[0]; - ltsr_rtsr[ltsr] = - tsr_info_t(ins[inp_idx], expr(), in_fmts[inp_idx], expr()); - inp_idx++; - } - for (auto &out : graph_output_ops) { - auto <sr = out->get_inputs()[0]; - ltsr_rtsr[ltsr] = tsr_info_t(outs[out_idx], expr(), out_fmts[out_idx], - out_sizes[out_idx]); - out_idx++; - } - auto query_idx = get_internal_tunable_input_indices(); - for (size_t i = 0; i < query_idx.size(); i++) { - auto &ori_inp_idx = query_idx[i]; - auto <sr = node_inputs[ori_inp_idx]; - ltsr_rtsr[ltsr] = tsr_info_t(ori_ins[i], expr(), ori_in_fmts[i], expr()); - } - // create query functions of valid ops inside graph and final query - // function. - create_query_function_by_graph(gp, kernel, ori_ins, ori_in_fmts, - each_op_num_keys, total_key_num, - dispatch_op_num); - bld.push_returns(true); - auto body = bld.pop_scope(); - func->body_ = std::move(body); - modu->add_func({func}); - modu->set_entry_func_idx(0); - return modu; -} - -void mixed_fuse_op_t::create_internal_dispatch_funcs( - const context_ptr &ctx, ir_module_ptr &mod, - const std::shared_ptr &use_mtp) { - // todo: currently we only support one op with internal func query. - for (auto &op : sub_graph_.ops_) { - if (op->need_dynamic_internal_query()) { - COMPILE_ASSERT( - op->info_.internal_info_ && - !op->info_.internal_info_->dispatch_table_name_.empty(), - "Not set the dispatch table in mixed op."); - auto &table_name = op->info_.internal_info_->dispatch_table_name_; - int dyn_idx = 0; - op->info_.internal_info_->parti_in_ltsrs_ = - info_.internal_info_->parti_in_ltsrs_; - op->info_.internal_info_->parti_out_ltsrs_ = - info_.internal_info_->parti_out_ltsrs_; - op->get_internal_dispatch_key_set(ctx)->for_each_key_process(std::bind( - create_dispatch_funcs_by_keys, ctx, std::ref(mod), table_name, op, - std::placeholders::_1, expr(), std::ref(dyn_idx), use_mtp, - /*internal*/ true)); - } - } -} - -void mixed_fuse_op_t::get_graph_impl(std::shared_ptr &graph) { - throw std::runtime_error("mixed_fuse_op_t::get_graph Not implemented"); -} - -struct inplace_recursion_context_t { - int depth_ = 0; - // UNDEF means a tensor is not directly or indirectly connected to an input - enum kind_t { UNDEF = 0, NO_INPLACE, ZERO_OFFSET_INPLACE, FREE_INPLACE }; - - // the graph input tensor -> its index in std::vector - const std::unordered_map &tsr_2_in_index_; - // the map of all graph tensors -> a vector of graph inputs. Each element of - // the vector represents the in-place status of a graph input - std::unordered_map> result_; - - inplace_recursion_context_t( - const std::unordered_map &tsr_2_in_index) - : tsr_2_in_index_(tsr_2_in_index) {} - - // merges the in-place results. Used when an Op depends on multiple graph - // inputs and we need to merge the status of the same input - static kind_t merge_result(kind_t a, kind_t b, bool good_op) { - if (good_op) { - if (a == NO_INPLACE || b == NO_INPLACE) { - return NO_INPLACE; - } - if (a == UNDEF) { - return b; - } - if (b == UNDEF) { - return a; - } - if (a == ZERO_OFFSET_INPLACE || b == ZERO_OFFSET_INPLACE) { - return ZERO_OFFSET_INPLACE; - } - return FREE_INPLACE; - } else { - // if the current op is not a "good" op, we need to mark all inputs - // it depends on with NO_INPLACE - if (a != UNDEF || b != UNDEF) { - return NO_INPLACE; - } - return UNDEF; - } - } - - // the main recursion function to recursively find the in-place status - const std::vector *call(const graph_tensor_ptr &tsr) { - depth_++; - if (depth_ > 500) { - return nullptr; - } - auto itr = result_.find(tsr); - if (itr != result_.end()) { - return &itr->second; - } - auto &ret = (result_[tsr] = std::vector(tsr_2_in_index_.size())); - // if the tensor is used more than once, we simply skip it for the sake - // of correctness. We can obviously do better than this. - auto producer = tsr->producer_owner_; - bool good_op = tsr->uses_.size() <= 1UL; - if (producer->isa()) { - // we define that input tensor can in-place reuse itself. - ret.at(tsr_2_in_index_.find(tsr)->second) = - good_op ? FREE_INPLACE : NO_INPLACE; - depth_--; - return &ret; - } - bool is_binary = producer->isa(); - // if it is an broadcast op, we cannot in-place reuse the broadcast - // input - int bcast_idx = -1; - if (is_binary) { - bcast_idx = - producer->stc_cast()->get_broadcast_input(); - } - bool must_zero_offset = producer->isa() || is_binary || - producer->isa(); - bool can_be_free = producer->isa(); - good_op = good_op && (must_zero_offset || can_be_free); - // we allow cast ops in the dependency chain, even if the size of dtype - // is not the same. Here we only check the logical dependency instead of - // memory position - auto &inputs = producer->get_inputs(); - for (size_t input_idx = 0; input_idx < inputs.size(); input_idx++) { - auto &intsr = inputs[input_idx]; - auto *sub_result = call(intsr); - if (!sub_result) { - return nullptr; - } - for (size_t i = 0; i < ret.size(); i++) { - auto result = (*sub_result)[i]; - // if the op's input is broadcast, all the graph input tensors - // it depends on should be NO_INPLACE - if ((int64_t)bcast_idx == (int64_t)input_idx && result != UNDEF) { - result = NO_INPLACE; - } - ret[i] = merge_result(ret[i], result, good_op); - if (must_zero_offset && ret[i] == FREE_INPLACE) { - ret[i] = ZERO_OFFSET_INPLACE; - } - } - } - depth_--; - return &ret; - } -}; - -float mixed_fuse_op_t::get_gflop() { return sub_graph_.get_gflop(); } - -sc_op_ptr mixed_fuse_op_t::copy( // NOLINT - const std::vector &ins, - const std::vector &outs, sc_graph_t &graph) { - auto ret = - graph.make(op_name_, parti_list_, nullptr, - copy_graph(sub_graph_), ins, outs, attrs_); - ret->sub_graph_.sync_dynamic_info_with_graph(graph); - return ret; -} - -std::vector>> -mixed_fuse_op_t::get_inplace_map() { - std::vector>> ret; - auto in_ops = sub_graph_.get_input_ops(); - // create a map from input tensors to its index - std::unordered_map tsr_2_index; - std::vector index_2_tsr; - for (auto &in : in_ops) { - for (auto &tsr : in->get_outputs()) { - auto idx = tsr_2_index.size(); - tsr_2_index[tsr] = idx; - index_2_tsr.emplace_back(tsr); - } - } - inplace_recursion_context_t ctx{tsr_2_index}; - - // for each output tensors... - auto out_ops = sub_graph_.get_output_ops(); - size_t out_idx = 0; - for (auto &out : out_ops) { - for (auto &outtsr : out->get_inputs()) { - std::vector can_inplace; - auto *rec_ret = ctx.call(outtsr); - if (!rec_ret) { - SC_MODULE_WARN << "Max recursion count reached for tensor " - "inplace optimization " - "for fused op"; - return {}; - } - for (size_t i = 0; i < rec_ret->size(); i++) { - if ((*rec_ret)[i] == inplace_recursion_context_t::ZERO_OFFSET_INPLACE) { - // zero offset means that the output->input dependency - // chain contains elementwise ops. We need to ensure - // that each memory position of the output strictly - // depend on the same memory position of the input - if (utils::get_sizeof_type(index_2_tsr.at(i)->details_.dtype_) == - utils::get_sizeof_type(outtsr->details_.dtype_)) { - can_inplace.emplace_back(tensor_inplace_info_t{ - static_cast(i), inplace_kind::ZERO_OFFSET}); - } - } else if ((*rec_ret)[i] == inplace_recursion_context_t::FREE_INPLACE) { - can_inplace.emplace_back( - tensor_inplace_info_t{static_cast(i), inplace_kind::FREE}); - } - } - if (!can_inplace.empty()) { - ret.emplace_back(out_idx, std::move(can_inplace)); - } - } - out_idx++; - } - return ret; -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/fused_op.hpp b/legacy/core/src/compiler/ir/graph/fused_op.hpp deleted file mode 100644 index 896874dd7..000000000 --- a/legacy/core/src/compiler/ir/graph/fused_op.hpp +++ /dev/null @@ -1,141 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_FUSED_OP_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_FUSED_OP_HPP - -#include "graph_op.hpp" -#include "visitor.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -struct mixed_parti_t; - -struct fusion_partition_t : std::enable_shared_from_this { - // the fusible ops in the partition. Not including base tunable op - std::unordered_set ops; - // valid only after phase 3 - sc_op_ptr main_tunable_op; - // valid only after phase 3, out tensors of old ops to be fused => new - // fused_op out tensors - std::unordered_map output_replace_map; - using ptr = std::shared_ptr; - ptr merged_to; - // if "this" is not merged, return this. If "this" is merged to another - // partition, return the partition - fusion_partition_t *get_root() const; - - virtual ~fusion_partition_t() = default; - - /** - * Checks if an `op` node can be added to the partition. We check that for - * each input op x in op.get_inputs(), if x is not in the partition, then x - * must not depend on any op in the current partition. - * We add this check to avoid potential cycles in the graph after fusion. - * e.g., think the following graph (something like the basic block of - * resnet) - * v0=conv(v1,v2) - * v3=relu(v0) - * v4=conv(v3,v6) - * v7=add(v3,v4) - * - * We first add relu_v3 to the partition. Note that the add_v7 depends on - * our partition by 2 paths: v3->v7 and v3->v4->v7. For add_v7's - * dependencies, relu_v3 is in the partition and conv_v4 is not. If we fuse - * conv_v0, relu_v3 and add_v7, there will be a cycle in the graph: - * conv_v4 depends on v3, which is the output of the fused op. But the fused - * op depends on v4, which is the output of conv_v4 - */ - bool is_ok_to_add(sc_op *op, const op_dep_matrix_t &g) const; - - bool contains(sc_op *op) const; - - // merge the ops in "other" to "this" - void merge(const ptr &other) const; -}; - -class mixed_fuse_op_t : public graph_op_t, - public op_traits::may_prefetch_t, - public op_traits::may_inplace_t, - public op_traits::copyable_t { -public: - mixed_fuse_op_t(const std::string &name, - const std::vector> &parti_list, - const ir_module_ptr &mod, const sc_graph_t &graph, - const std::vector &ins, - const std::vector &outs, - const any_map_t &attrs); - sc_graph_t sub_graph_; - std::vector> parti_list_; - ir_module_ptr mod_; - ir_module_ptr get_func(context_ptr ctx) override; - void get_graph_impl(std::shared_ptr &) override; - void schedule_loops(const stmt &body); - - // dynamic related - // need internal dispatch func. - bool need_dynamic_internal_query_impl() const override; - // create internal dispatch functions with internal dispatch keys. - void create_internal_dispatch_funcs( - const context_ptr &ctx, ir_module_ptr &ret_mod, - const std::shared_ptr &use_mtp); - // return the indices of tunable op inputs in sub graph. - std::vector get_internal_tunable_input_indices(); - virtual dispatch_set_ptr &get_dispatch_key_set() override; - // Return vector of inner op with dispatch key set like tunable op/reorder - // op(dispatch alg). total_key_num is the pointer to number of input/output - // dispatch key inside(option). The return value is mainly used for combined - // dispatch key construction. - virtual std::vector get_inner_dispatch_ops(int *total_key_num); - void update_internal_graph_format(const combined_op_dispatch_key_t &key, - const context_ptr &ctx); - ir_module_ptr get_dynamic_query_func(const context_ptr &ctx); - - std::vector - query_prefetch(const context_ptr &ctx, bool is_global, - const std::vector &ins) override; - - void generate_prefetcher_body_for_tensor( - const context_ptr &ctx, const std::vector &func_args, - const std::vector &ins, const std::vector &indices) override; - std::vector>> - get_inplace_map() override; - float get_gflop() override; - sc_op_ptr copy(const std::vector &ins, // NOLINT - const std::vector &outs, - sc_graph_t &mgr) override; -}; - -void schedule_loop_body(const stmt &body, node_ptr_map *node_remap = nullptr); -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/graph/fusible_op.cpp b/legacy/core/src/compiler/ir/graph/fusible_op.cpp deleted file mode 100644 index 0764803d5..000000000 --- a/legacy/core/src/compiler/ir/graph/fusible_op.cpp +++ /dev/null @@ -1,562 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include -#include -#include -#include - -#include "anchor_loop_generator.hpp" -#include "fused_op.hpp" -#include "fusible_op.hpp" -#include "utils.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -int binary_elementwise_op_t::get_broadcast_input() const { - auto non_bc_input_idx = get_non_broadcast_input_index(true); - return non_bc_input_idx.size() > 1 ? -1 : 1 - non_bc_input_idx[0]; -} - -static int get_base_input_idx(fusible_op_t *cur) { - int base_idx = 0; - if (auto binary_node = cur->dyn_cast()) { - // if bc side (smaller side) is the lhs, we need to set base_idx to 1 - if (!binary_node->get_broadcast_input()) { - base_idx = 1; - } - } - if (auto select_node = cur->dyn_cast()) { - // we need to set base_idx to the max input - base_idx = select_node->get_non_broadcast_input_index(true)[0]; - } - COMPILE_ASSERT(base_idx >= 0, "Bad base idx for fusible_op"); - return base_idx; -} - -ir_module_ptr fusible_op_t::get_func(context_ptr ctx) { - return fusible_op_get_func(this, ctx); -} - -void fusible_op_t::create_mixed_partition(mixed_parti_t *parti) { - parti->buf_alloc_.allocate_buffer(this); - std::vector ins, outs; - std::tie(ins, outs) = parti->buf_alloc_.get_buffer(this); - builder::ir_builder_t bld; - bld.push_scope(); - // select loop mode - bool use_output_mode = false; - if (auto reo_op = this->dyn_cast()) { - // for padding reorder, it maybe prefer to select input loop - if (reo_op->check_padding()) { - use_output_mode = reo_op->use_output_loop(); - } else { - // for most reorder w/o padding, if it supports output loop, try to - // force it to use output loop for post-op fusion - use_output_mode = !reo_op->attrs_.get_or_else("use_input_loop", false) && - reo_op->support_output_loop(); - if (use_output_mode) { - // set attr to force reorder use output mode - reo_op->attrs_.set(op_attr_key::break_pre_fuse, true); - } - } - } - // set base idx - auto base_gt = use_output_mode ? get_outputs()[0] - : get_inputs()[get_base_input_idx(this)]; - // create fusion anchor mgr - fusion_anchor_mgr_t fmgr(parti); - // create anchor loop gen - anchor_loop_generator_t gen(base_gt); - // create outer loop anchor - bool status = gen.create_outer_loop_anchor(&fmgr, parti->ctx_); - COMPILE_ASSERT(status, "generate outer loops failed for " - << op_name_ << "_" << logical_op_id_ - << ", please check"); - auto body = bld.pop_scope(); - parti->func_ = builder::make_func(std::string(""), std::vector{}, - std::move(body), datatypes::boolean); - // append to partition - search_anchor(parti); - append_mixed_partition(parti); - - // double check used anchor - auto used_anchor = parti->lookup_anchor_map(this); - // Due to anchor is sorted by ascending order, clear invalid IR including - // for_loop and anchor - if (used_anchor != parti->fanchors_.back()) { - // remove all fanchor less then used anchor - for (auto iter = parti->fanchors_.begin(); iter < parti->fanchors_.end();) { - if ((*iter) == used_anchor) - break; - parti->clear_fanchor(*iter); - iter = parti->fanchors_.erase(iter); - } - // check whether need to keep last anchor - if (!is_single_op_graph(get_owner_graph())) { - // remove last anchor - parti->clear_fanchor(parti->fanchors_.back()); - parti->fanchors_.pop_back(); - } - } else { - // if last fanchor is used - // clear all fanchor in void of post op fusion - parti->fanchors_.clear(); - // remove all field related to anchor - parti->op_anchor_map_.erase(this); - parti->buf_alloc_.tsr2anch_map_.clear(); - } - auto parent_scope = used_anchor->get_parent_scope(); - for (auto iter = parent_scope->seq_.begin(); - iter < parent_scope->seq_.end();) { - if ((*iter).ptr_same(used_anchor->anchor_position_)) { - ++iter; - continue; - } - iter = parent_scope->seq_.erase(iter); - } -} - -void fusible_op_t::append_mixed_partition(mixed_parti_t *parti) { - COMPILE_ASSERT(parti->ready_for_op(this), "No suitable anchor found for " - << op_name_ << "_" - << logical_op_id_); - - if (!parti->empty()) { - parti->buf_alloc_.allocate_buffer(this); - parti->buf_alloc_.update_input_buffer_info(this); - - if (!this->isa()) { - int base_idx = get_base_input_idx(this); - auto committed_anchor = parti->lookup_anchor_map(this); - auto &fsmap = committed_anchor->fsmap_; - auto &base_gt = get_inputs()[base_idx]; - if (fsmap.get(base_gt).size() == 1) { - builder::ir_builder_t bld; - bld.push_scope(); - anchor_loop_generator_t gen(base_gt); - // create fusion anchor mgr - fusion_anchor_mgr_t fmgr(parti); - // create_inner_anchor - gen.create_inner_loop_anchor(&fmgr, committed_anchor); - auto inner_ss = bld.pop_scope().checked_as(); - // search inner anchor again - search_anchor(parti); - if (committed_anchor != parti->lookup_anchor_map(this)) { - committed_anchor->commit_stmts(inner_ss); - } else { - auto inner_anchors = fmgr.get_fusion_anchor(); - // erase unused inner anchor - parti->fanchors_.erase(parti->fanchors_.end() - inner_anchors.size(), - parti->fanchors_.end()); - // clear invoid inner anchor content from parent anchor - for (auto iter = inner_anchors.rbegin(); iter != inner_anchors.rend(); - iter++) { - committed_anchor->clear_content((*iter).get()); - } - } - } - } - } - // update output buffer info after inner anchor created - parti->buf_alloc_.update_output_buffer_info(this); - - fusion_anchor_ptr committed_anchor = parti->lookup_anchor_map(this); - if (attrs_.get_or_else(mixed_partition_hint::inplace_optimized_op, false)) { - // commit content id to anchor - committed_anchor->append_content(static_cast(this)); - return; - } - - commit_into_anchor(committed_anchor.get()); - - // append op inner anchor into parti - if (attrs_.has_key(op_attr_key::fusible_inner_anchors)) { - auto op_inner_anchors = attrs_.get>( - op_attr_key::fusible_inner_anchors); - for (const auto &op_inner_anchor : op_inner_anchors) { - op_inner_anchor->attach_parent_anchor(committed_anchor); - parti->append_fusion_anchor(op_inner_anchor); - } - } -} - -void fusible_op_t::search_anchor(mixed_parti_t *parti) { - search_op_anchor_in_parti(this, parti); -} - -void fusible_op_t::commit_into_anchor(fusion_anchor_t *committed_anchor) { - auto parti = committed_anchor->get_binded_mxp(); - std::vector in_tsrs, out_tsrs; - std::tie(in_tsrs, out_tsrs) = parti->buf_alloc_.get_buffer(this); - std::vector> inputs(in_tsrs.size()), - outputs(out_tsrs.size()); - auto ths = this; - auto wrap_tsr2tsl_ = [&ths, &committed_anchor](const expr &tsr, - const graph_tensor_ptr >, - bool is_output = false) { - auto &range_list = committed_anchor->fsmap_.get(gt); - std::vector multi_tsl; - if (!range_list.empty()) { - if (ths->isa() && is_output) { - auto input_size = - committed_anchor->fsmap_.get(ths->get_inputs()[0]).size(); - // multi-slice reorder - if (input_size && range_list.size() > input_size) { - // align output size with input - for (size_t i = 0; i < input_size; i++) { - multi_tsl.emplace_back(tensor_slice(tsr)); - } - return multi_tsl; - } - } - for (auto &range : range_list) { - multi_tsl.emplace_back(tensor_slice(tsr, slice_range(range))); - } - } else { - COMPILE_ASSERT(ths->isa(), - "only reorder op support this case, but got " - << ths->op_name_) - if (is_output) { - ths->attrs_.set(op_attr_key::break_post_fuse, true); - } else if (committed_anchor->isa()) { - // pre-fuse reorder, align input size with output - auto output_size = - committed_anchor->fsmap_.get(ths->get_outputs()[0]).size(); - for (size_t i = 0; i < output_size; i++) { - multi_tsl.emplace_back(tensor_slice(tsr)); - } - return multi_tsl; - } - multi_tsl.emplace_back(tensor_slice(tsr)); - } - return multi_tsl; - }; - std::transform(get_inputs().begin(), get_inputs().end(), in_tsrs.begin(), - inputs.begin(), - [&wrap_tsr2tsl_](const graph_tensor_ptr >, const expr &tsr) { - return wrap_tsr2tsl_(tsr, gt); - }); - std::transform(get_outputs().begin(), get_outputs().end(), out_tsrs.begin(), - outputs.begin(), - [&wrap_tsr2tsl_](const graph_tensor_ptr >, const expr &tsr) { - return wrap_tsr2tsl_(tsr, gt, true); - }); - auto in_slice_size = inputs[0].size(); - COMPILE_ASSERT(in_slice_size, "No input slice found for " << op_name_) - COMPILE_ASSERT( - std::all_of(inputs.begin(), inputs.end(), - [&in_slice_size](const std::vector &input_tsl) { - return input_tsl.size() == in_slice_size; - }), - "All input slice size should be equal") - - // generate IR: unwrapper tensor slice, for compute_block, it just accpet - // single tensor_slice - for (size_t i = 0; i < in_slice_size; i++) { - std::vector new_inputs_ptr(inputs.size()); - std::vector new_outputs_ptr(outputs.size()); - std::transform(inputs.begin(), inputs.end(), new_inputs_ptr.begin(), - [&i](std::vector &ins) { return &ins[i]; }); - - std::transform(outputs.begin(), outputs.end(), new_outputs_ptr.begin(), - [&i](std::vector &out) { return &out[i]; }); - builder::ir_builder_t bld; - bld.push_scope(); - compute_block(parti->ctx_, new_outputs_ptr, new_inputs_ptr); - auto compute_core = bld.pop_scope().checked_as(); - committed_anchor->commit_stmts(compute_core); - } - // commit content id to anchor - committed_anchor->append_content(static_cast(this)); -} - -void fusible_op_t::query_format( - context_ptr ctx, - std::vector> &supported_ins, - std::vector> &supported_outs) { - std::vector> in_formats, out_formats; - if (this->isa()) { - out_formats.push_back({info_.outputs_[0]->details_.get_format()}); - } else { - out_formats.push_back({info_.inputs_[0]->details_.get_format()}); - } - format_to_dense_format_stride_pair(in_formats, out_formats, supported_ins, - supported_outs); -} - -size_t -fusible_op_t::compute_workload(const std::vector &ins, - const std::vector &outs) { - size_t wkld = 0UL; - auto accumulate_workload = [&wkld](size_t weight, const shape_dtype_pair &v) { - auto &dtype = v.second; - wkld += utils::get_sizeof_type(dtype) * weight; - }; - std::for_each(ins.begin(), ins.end(), - std::bind(accumulate_workload, - static_cast( - op_traits::workload_computable_t::read_weight), - std::placeholders::_1)); - std::for_each(outs.begin(), outs.end(), - std::bind(accumulate_workload, - static_cast( - op_traits::workload_computable_t::write_weight), - std::placeholders::_1)); - return wkld; -} - -size_t fusible_op_t::compute_fusible_workload( - const context_ptr &ctx, const std::vector &dst, - const std::vector &inputs) { - if (is_dynamic() || - std::any_of(inputs.begin(), inputs.end(), - [](const tensor_slice *inp) { return !inp->is_const(); })) { - return memory_access_threshold_per_thread; - } - std::vector wkld_ins, wkld_outs; - wkld_ins.resize(inputs.size()); - wkld_outs.resize(dst.size()); - auto get_shape_dtype_pair = [](const tensor_slice *v) { - return std::make_pair(get_expr_to_dims(v->shape_), v->get_base_dtype()); - }; - std::transform(inputs.begin(), inputs.end(), wkld_ins.begin(), - get_shape_dtype_pair); - std::transform(dst.begin(), dst.end(), wkld_outs.begin(), - get_shape_dtype_pair); - return compute_workload(wkld_ins, wkld_outs); -} - -input_op::input_op(const sc_dims &dims, sc_data_type_t dtype) { - op_name_ = "input"; - info_.outputs_.emplace_back( - std::make_shared(this, sc_data_format_t(), dims, dtype)); -} - -input_op::input_op(const logical_tensor_t <) { - op_name_ = "input"; - info_.outputs_.emplace_back(std::make_shared(this, lt)); -} - -input_op::input_op(const std::vector &outs) { - info_.outputs_ = outs; - for (auto &out : info_.outputs_) { - out->producer_owner_ = this; - } - op_name_ = "input"; -} - -void input_op::initialize_dynamic_placeholder() { - for (auto &out : info_.outputs_) { - auto plain_dims = out->details_.get_plain_dims(); - for (auto &it : plain_dims) { - if (it == dimensions::dynamic_any) { - it = get_owner_graph().get_next_dynamic_placeholder(); - } - } - out->details_.set_plain_dims(plain_dims); - } -} - -output_op::output_op(const graph_tensor_ptr &v) { - info_.inputs_.emplace_back(v); - op_name_ = "output"; -} - -output_op::output_op(const std::vector &in) { - info_.inputs_ = in; - op_name_ = "output"; -} - -static void extract_const_op_data( - const sc_op *ths, - const std::function &filter, - const std::shared_ptr *&out_vales, - const sc_data_type_t *&out_dtype, const sc_data_format_t *&out_fmt) { - if (!filter || filter(ths, "values")) { - out_vales = - ths->attrs_.get_or_null>("values"); - COMPILE_ASSERT(out_vales, "expecting values"); - } - if (!filter || filter(ths, "dtype")) { - out_dtype = ths->attrs_.get_or_null("dtype"); - COMPILE_ASSERT(out_dtype, "expecting dtype"); - } - if (!filter || filter(ths, "format")) { - out_fmt = ths->attrs_.get_or_null("format"); - } -} - -// special handling for union values -bool constant_op_t::compare_contents( - const sc_op *other, - const std::function &filter) - const { - if (other->op_name_ != op_name_) { - return false; - } - const std::shared_ptr *values = nullptr, - *other_values = nullptr; - const sc_data_type_t *dtype = nullptr, *other_dtype = nullptr; - const sc_data_format_t *fmt = nullptr, *other_fmt = nullptr; - extract_const_op_data(this, filter, values, dtype, fmt); - extract_const_op_data(other, filter, other_values, other_dtype, other_fmt); - if (dtype) { - if (!other_dtype || *dtype != *other_dtype) { - return false; - } - } - if (fmt) { - if (!other_fmt || *fmt != *other_fmt) { - return false; - } - } - if (!values) { - return other_values == nullptr; - } - // now values must be non-null - if (!other_values) { - return false; - } - auto &vals = *values; - auto &vals2 = *other_values; - if (vals->size_ != vals2->size_) { - return false; - } - - if (dtype) { - switch (get_type_category_nothrow(*dtype)) { - case CATE_FLOAT: - for (size_t i = 0; i < vals->size_ / 4; i++) { - if (static_cast(vals->data_)[i] != - static_cast(vals2->data_)[i]) { - return false; - } - } - break; - case CATE_INT: - case CATE_UINT: - for (size_t i = 0; i < vals->size_ / 4; i++) { - if (static_cast(vals->data_)[i] != - static_cast(vals2->data_)[i]) { - return false; - } - } - break; - default: - throw std::runtime_error("Met unexpected dtype for constant"); - break; - } - } - return true; -} - -size_t constant_op_t::hash_contents( - const std::function &filter) - const { - size_t seed = 0; - const std::shared_ptr *values = nullptr; - const sc_data_type_t *dtype = nullptr; - const sc_data_format_t *fmt = nullptr; - extract_const_op_data(this, filter, values, dtype, fmt); - if (fmt) { - hash_combine(seed, *fmt); - } - if (!values) { - return seed; - } - auto &vals = *values; - - for (size_t i = 0; i < vals->size_; i++) { - hash_combine(seed, static_cast(vals->data_)[i]); - } - - return seed; -} - -void constant_op_t::reset_const_values() { - if (attrs_.has_key("temp.var") && attrs_.has_key("temp.val/var")) { - int K = - static_cast(attrs_.get>("temp.var")->var_); - int base_val = attrs_.get("temp.val/var"); - // update private member - const_values_ = - std::make_shared(std::vector{base_val * K}); - // update attr - attrs_.set("values", const_values_); - } -} - -constant_op_t::constant_op_t(const std::vector &ins, - const std::vector &outs, - const any_map_t &attrs) { - COMPILE_ASSERT(ins.empty(), "No op input.\n"); - COMPILE_ASSERT(attrs.has_key("values") && attrs.has_key("dtype") && - attrs.has_key("plain_dims"), - "expecting values, format and dtype in attr"); - op_name_ = "constant"; - sc_data_format_t format = - attrs.get_or_else("format", sc_data_format_t(format_kinds::A)); - attrs_ = attrs; - const_values_ = attrs.get>("values"); - sc_data_type_t dtype = attrs.get("dtype"); - sc_dims plain_dims = attrs.get("plain_dims"); - - if (outs.empty()) { - info_.outputs_.emplace_back(std::make_shared( - this, logical_tensor_t{format, plain_dims, dtype})); - } else { - COMPILE_ASSERT(outs.size() == 1, "Wrong op output size.\n"); - info_.outputs_ = outs; - } -} - -// todo: support tensor expr -constant_op_t::constant_op_t(std::shared_ptr v, - sc_data_type_t dtype, const sc_dims &plain_dims, - const sc_data_format_t &format) { - const_values_ = std::move(v); - info_.outputs_.emplace_back( - std::make_shared(this, format, plain_dims, dtype)); - info_.outputs_[0]->details_.dtype_ = dtype; - info_.outputs_[0]->details_.set_plain_dims(plain_dims); - attrs_.set("dtype", dtype); - attrs_.set("values", const_values_); - attrs_.set("plain_dims", plain_dims); - attrs_.set("format", format); - op_name_ = "constant"; -} - -OP_REGISTER(constant_op_t, constant) - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/fusible_op.hpp b/legacy/core/src/compiler/ir/graph/fusible_op.hpp deleted file mode 100644 index aeed8fa58..000000000 --- a/legacy/core/src/compiler/ir/graph/fusible_op.hpp +++ /dev/null @@ -1,283 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_FUSIBLE_OP_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_FUSIBLE_OP_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -struct tensor_slice; - -/** - * A fuser will do actual code injection on the fusion point. It will be managed - * by the fusion manager. - * */ -class SC_INTERNAL_API fusible_op_t - : public sc_op, - public op_traits::workload_computable_t, - public op_traits::mixed_partition_acceptable { -public: - // when fusible_op_t is as a started op in the graph/subgraph, query_format - // return certain format. - void query_format( - context_ptr ctx, - std::vector> &supported_ins, - std::vector> &supported_outs) override; - ir_module_ptr get_func(context_ptr ctx) override; - - /** - * 'infer_slice_ranges' is used to infer slice ranges for all fusible op in - * fusion manger, espically helpful for input arg op, because it could not - * know the slice information, it need to inferred by its partner input in - * binary or trinary op, such as add/mul, etc. Additional, sometimes, the - * slice range may be changed, e.g. reduce_op_t or movement type ops. we - * need to address for this speciall condition. Actually, - * 'infer_slice_ranges' can be viewed as `post_slice_ranges` and very simple - * `pre_infer_slice_ranges` with only one previous op inferred. - * */ - virtual infer_status_code infer_slice_ranges(const context_ptr &ctx, - fslice_map &fsmap) = 0; - - /** - * 'pre_infer_slice_ranges' is used to infer slice ranges especially - * for pre-op fusion. As mentioned above `infer_slice_ranges` can infer - * slice for input arg op, however, if sometimes the input of one fusible op - * is a sub fusion graph, it is expected to infer input slice by given - * output slice. - * */ - virtual infer_status_code pre_infer_slice_ranges(const context_ptr &ctx, - fslice_map &fsmap) = 0; - - void search_anchor(mixed_parti_t *parti) override; - - /** - * Does the actual code injection to the current func on an input - * and output tensor slices. - * - * */ - virtual void - compute_block(context_ptr ctx, const std::vector &dst, - const std::vector &inputs) = 0; - /** - * Compute workload of a fusible op during its compute block, ins and outs - * are from tensor slice. - * */ - size_t compute_workload(const std::vector &ins, - const std::vector &outs) override; - /** - * A wrapper of `compute_workload`, get workload from tensor slice. - * */ - virtual size_t - compute_fusible_workload(const context_ptr &ctx, - const std::vector &dst, - const std::vector &inputs); - - void create_mixed_partition(mixed_parti_t *parti) override; - - void append_mixed_partition(mixed_parti_t *parti) override; - - void commit_into_anchor(fusion_anchor_t *committed_anchor) override; - - void infer_binding_axis(binding_axis_map &bdax_map) override {} - - void pre_infer_binding_axis(binding_axis_map &bdax_map) override {} - - ~fusible_op_t() override = default; - - int anchor_id_ = -1; -}; - -using fusion_op_ptr = std::shared_ptr; - -#define DECLARE_COMPUTE() \ - virtual void compute_block( \ - context_ptr ctx, const std::vector &dst, \ - const std::vector &inputs) override; - -#define DECLARE_QUERY_AND_COMPUTE() \ - virtual infer_status_code infer_slice_ranges(const context_ptr &ctx, \ - fslice_map &fsmap) override; \ - virtual infer_status_code pre_infer_slice_ranges( \ - const context_ptr &ctx, fslice_map &fsmap) override; \ - virtual void compute_block( \ - context_ptr ctx, const std::vector &dst, \ - const std::vector &inputs) override; - -#define DECLARE_QUERY_AND_DEFAULT_COMPUTE() \ - virtual infer_status_code infer_slice_ranges(const context_ptr &ctx, \ - fslice_map &fsmap) override { \ - return infer_status_code::OK; \ - } \ - virtual infer_status_code pre_infer_slice_ranges( \ - const context_ptr &ctx, fslice_map &fsmap) override { \ - return infer_status_code::OK; \ - } \ - virtual void compute_block( \ - context_ptr ctx, const std::vector &dst, \ - const std::vector &inputs) override {} - -/** - * The input argument Op - * Inputs: - * - None - * Outputs: - * - One or more tensors - * Attrs: - * - values: std::shared_ptr - (Optional) if the values of this - * input is known at compile time, the values - * - keep_plain: bool - default = false. It will only be used when - * graph.attrs_["is_input_plain"]=false. If keep_plain=true, we will keep the - * plain format for this input. Otherwise we may reset this input to blocking - * format - * */ -class input_op : public fusible_op_t { -public: - DECLARE_QUERY_AND_DEFAULT_COMPUTE(); - - input_op(const sc_dims &dims = {}, sc_data_type_t dtype = datatypes::f32); - input_op(const logical_tensor_t <); - input_op(const std::vector &outs); - - void initialize_dynamic_placeholder(); -}; - -/** - * The output argument Op - * Inputs: - * - One or more tensors - * Outputs: - * - None - * Attrs: - * - target_formats: std::vector - default: vector of plain - * format. It will be used only if the graph's `is_output_plain` attr is - * true. target_formats is the format of the output tensor. They should be - * either plain or simply permuted. If the target_formats is set, the - * output tensor will be reordered into the specified format. - * - target_strides: std::vector - default: vector of dense strides - * target_strides is the stride of the output tensor. - * */ -class output_op : public fusible_op_t { -public: - DECLARE_QUERY_AND_DEFAULT_COMPUTE(); - - output_op(const graph_tensor_ptr &v); - output_op(const std::vector &in); -}; - -/** - * The op that produces constant tensors. - * Inputs: - * - None - * Outputs: - * - One single output tensor - * Attrs: - * - values: std::shared_ptr - the data - * - dtype: sc_data_type_t - the datatype. (todo: remove this attr) - * - plain_dims: dims (todo: remove this attr) - * - format: sc_data_format_t (todo: remove this attr) - * */ -class constant_op_t : public fusible_op_t, public op_traits::auto_copyable_t { -public: - DECLARE_QUERY_AND_DEFAULT_COMPUTE(); - - constant_op_t(const std::vector &ins, - const std::vector &outs, - const any_map_t &attrs); - constant_op_t(const std::shared_ptr v, sc_data_type_t dtype, - const sc_dims &plain_dims, - const sc_data_format_t &format = sc_data_format_t()); - std::shared_ptr get_constant_values() const { - return const_values_; - } - sc_data_type_t get_constant_dtype() const { - return info_.outputs_[0]->details_.dtype_; - } - const sc_dims &get_constant_plain_dims() const { - return info_.outputs_[0]->details_.get_plain_dims(); - } - const sc_data_format_t &get_constant_format() { - return info_.outputs_[0]->details_.get_format(); - } - sc_dims get_constant_blocking_dims() { - return sc_data_format_t::get_blocking_shapes(get_constant_plain_dims(), - get_constant_format()); - } - bool - compare_contents(const sc_op *other, - const std::function - &filter) const override; - size_t hash_contents( - const std::function &filter) - const override; - - // if necessary, reset const_values according possible `var` from attrs - void reset_const_values(); - -private: - std::shared_ptr const_values_; -}; - -// this structure is used to store vectorized information, including axis and -// lanes; -struct vectorized_info_t { - // the last valid axis to vectorized, skip invalid dims whose length equals - // to 1, the default is last dim for most conditions - int axis = -1; - // vectorized lanes - uint32_t lanes = 0; -}; - -class binary_elementwise_op_t : public fusible_op_t, - public op_traits::may_inplace_t, - public op_traits::may_broadcast_t, - public op_traits::brgemm_fusion_acceptable_t, - public op_traits::auto_copyable_with_trait_t< - op_traits::brgemm_fusion_acceptable_t> { -public: - int get_broadcast_input() const; -}; - -class binary_backward_op_t : public fusible_op_t, - public op_traits::may_inplace_t, - public op_traits::auto_copyable_t {}; - -class unary_elementwise_op_t : public fusible_op_t, - public op_traits::may_inplace_t, - public op_traits::brgemm_fusion_acceptable_t, - public op_traits::auto_copyable_with_trait_t< - op_traits::brgemm_fusion_acceptable_t> {}; - -// used for classification -class movement_op_t : public fusible_op_t, public op_traits::may_quantize_t {}; - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl -#endif diff --git a/legacy/core/src/compiler/ir/graph/fusible_op_utils.cpp b/legacy/core/src/compiler/ir/graph/fusible_op_utils.cpp deleted file mode 100644 index 09c6c13a3..000000000 --- a/legacy/core/src/compiler/ir/graph/fusible_op_utils.cpp +++ /dev/null @@ -1,1221 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "anchor_loop_generator.hpp" -#include "fusible_op.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -// incrementor for loop -static std::atomic idx = {0}; -// incrementor for temp var -static std::atomic var_idx = {0}; -// helper function to get current var_idx -std::string fusion_create_var_idx() { - return std::string("_") + std::to_string(var_idx++); -} - -std::string fusion_create_idx() { - return std::string("_") + std::to_string(idx++); -} - -static std::vector convert_t(std::vector &src) { - std::vector dst(src.size()); - std::transform(src.begin(), src.end(), dst.begin(), - [](tensor_slice &t) { return &t; }); - return dst; -} - -static std::vector -convert_const_t(std::vector &src) { - std::vector dst(src.size()); - std::transform(src.begin(), src.end(), dst.begin(), - [](tensor_slice &t) { return &t; }); - return dst; -} - -static std::vector -make_tensor_slice(sc_graph_t &graph, const std::vector &data, - const std::string &tensor_name, - std::vector &flattened) { - std::vector expected; - for (size_t i = 0; i < data.size(); ++i) { - std::vector dims = data[i]->details_.get_blocking_dims_expr(graph); - std::vector strides = dims_to_dense_stride(dims); - expr aexpr = builder::make_stensor(tensor_name + std::to_string(i), dims, - strides, data[i]->details_.dtype_); - flattened.emplace_back(aexpr); - expected.emplace_back(tensor_slice(aexpr)); - } - return expected; -} - -ir_module_ptr fusible_op_get_func(fusible_op_t *op, const context_ptr &ctx) { - sc_graph_t g; - g.sync_dynamic_info_with_graph(op->get_owner_graph()); - if (op->get_owner_graph().attrs_.get_or_else("temp.force_static", false)) { - g.attrs_.set("temp.force_static", true); - } - if (op->get_owner_graph().is_dynamic()) { - g.attrs_.set("temp.parent_graph_dynamic", true); - } - std::vector ins; - std::vector outs; - for (auto &in : op->get_inputs()) { - ins.emplace_back(std::make_shared(nullptr, in->details_)); - } - for (auto &out : op->get_outputs()) { - outs.emplace_back(std::make_shared(nullptr, out->details_)); - } - g.make_input(ins); - auto copyable = op->dyn_cast(); - COMPILE_ASSERT(copyable, - "The fusible op should be copyable: " << op->op_name_); - auto copied = copyable->copy(ins, outs, g); - copied->info_.cur_impl_ = op->info_.cur_impl_; - g.make_output(outs); - g.attrs_.set(mixed_partition_hint::single_op_graph, true); - g.attrs_.set(mixed_partition_hint::optimized_sub_graph, - is_optimized_sub_graph(op->get_owner_graph())); - // create dummy parti - auto parti = std::make_shared( - ctx, std::const_pointer_cast(op->shared_from_this())); - // create graph-to-original ops maping - std::unordered_map graph2orig_ops = { - {copied, op->shared_from_this()}}; - // try optimize partition - if (!op->attrs_.get_or_else("temp.no_optimize_op", false) && - try_optimize_parti(parti.get(), g, graph2orig_ops)) { - // redo partition - std::vector op2parti(g.ops_.size()); - do_partition(ctx, g, op2parti, std::make_shared(g)); - // collect legal partition - auto res = collect_parti_set(op2parti, false); - // Expect only one partition found - COMPILE_ASSERT(res.size() == 1, - "Only sinlge partition is expected, but got " << res.size()); - // reset new partition - parti = res[0]; - // validate optimization - if (!parti->validate_optimization()) { - // redo without optimization - op->attrs_.set("temp.no_optimize_op", true); - auto ret = fusible_op_get_func(op, ctx); - // remove temp attr - op->attrs_.remove("temp.no_optimize_op"); - return ret; - } - } else { - parti = std::make_shared( - ctx, std::const_pointer_cast(copied->shared_from_this())); - } - auto mx_op = parti->transform_to_mixed_op(); - mx_op->set_owner_graph(&g); - // copy op name - mx_op->op_name_ = op->op_name_; - // if layer name set - if (auto layer_name = - op->attrs_.get_or_null(op_attr_key::layer_name)) { - COMPILE_ASSERT(!layer_name->empty() && isalpha(layer_name->front()), - "Bad layername: " << *layer_name); - mx_op->op_name_ = *layer_name; - } - // copy logigcal id - mx_op->logical_op_id_ = op->logical_op_id_; - return mx_op->get_func(ctx); -} - -sc_dims get_expr_to_dims(const std::vector &dim) { - sc_dims dim_int; - dim_int.reserve(dim.size()); - for (const expr &d : dim) { - auto cd = do_cast_and_fold(d); - COMPILE_ASSERT(cd.isa(), "non-constant value found."); - dim_int.emplace_back(get_const_as_int(cd.static_as())); - } - return dim_int; -} - -stmt mask_compute_func_t::operator()(const std::vector &in, - std::vector &out, - const expr &cur_idx, - const expr &upper_bound, - uint32_t lanes) const { - auto ret = impl_(in, out); - if (cur_idx.defined() && upper_bound.defined()) { - auto bld = builder::get_current_builder(); - bld->emit(ret); - const size_t len = out.size(); - std::vector cur_list; - cur_list.reserve(len); - for (size_t i = 0; i < len; i++) { - cur_list.emplace_back(builder::make_assign_unattached( - out[i], make_select_by_mask(out[i], cur_idx, upper_bound, lanes))); - } - return builder::make_stmts_unattached(cur_list); - } - return ret; -} - -expr make_select_by_mask(const expr &lhs_vec, const expr &cur_index, - const expr &upper_bound, uint32_t lanes) { - sc_data_type_t var_dtype; - auto bld = builder::get_current_builder(); - auto upper_bound_int = builder::make_cast(datatypes::s32, upper_bound); - auto cur_index_int = builder::make_cast(datatypes::s32, cur_index); - int step = static_cast(lanes); - // upper_bound - cur_index - auto cur_step = builder::make_min( - builder::make_max(builder::make_sub(upper_bound_int, cur_index_int), 0), - step); - stmt mask_def; - auto mask = generate_mask_var_by_step(mask_def, cur_step, step); - bld->emit(mask_def); - expr rhs_vec = make_expr( - std::vector(lanes, UINT64_C(0)), - sc_data_type_t(lhs_vec->dtype_.type_code_, lanes)); - return builder::make_select(mask, lhs_vec, rhs_vec); -} - -void choose_mask_vartype_init_value(sc_data_type_t &var_dtype, - uint64_t &init_value, int32_t step) { - switch (step) { - case 4: { - var_dtype = datatypes::u8; - init_value = 15; - break; - } - case 8: { - var_dtype = datatypes::u8; - init_value = std::numeric_limits::max(); - break; - } - case 16: { - var_dtype = datatypes::u16; - init_value = std::numeric_limits::max(); - break; - } - case 32: { - var_dtype = datatypes::u32; - init_value = std::numeric_limits::max(); - break; - } - case 64: { - var_dtype = datatypes::index; - init_value = std::numeric_limits::max(); - break; - } - default: - COMPILE_ASSERT(false, "invalid lanes: " << step); - } -} - -expr calculate_mask_cur_step(const expr &len, const expr &iter_var, - const int32_t lanes) { - auto last_axis_offset = cast_to_s32(len) - cast_to_s32(iter_var); - // mask = min(max(0, last_dim_len - - // last_dim_idx),real_step) To choose [0 ~ - // step] mask - return builder::make_min( - builder::make_max(builder::make_constant(0), last_axis_offset), lanes); -} - -// generate mask = var < floor ? 0b1111 : 0b00..111; -expr last_dim_generate_mask(const expr &iter_var, const expr &floor, - expr const &last_dim_len, int const &lanes, - bool just_tail_part) { - // just_tail_part means that the floor and tail parts of the for loop are - // calculated separately. Only the tail part needs to calculate the mask. - auto s32_var = cast_to_s32(iter_var); - auto s32_floor = cast_to_s32(floor); - auto s32_dim_len = cast_to_s32(last_dim_len); - expr condition = s32_var < s32_floor; - expr tail_len = lanes + s32_var - s32_dim_len; - sc_data_type_t var_dtype; - uint64_t init_value; - choose_mask_vartype_init_value(var_dtype, init_value, lanes); - auto full_mask = builder::make_constant({init_value}, var_dtype); - if (floor.isa()) { - int floor_int = get_expr_as_int(floor); - int dim_len = get_expr_as_int(last_dim_len); - int res_mask = init_value >> (lanes + floor_int - dim_len); - - return just_tail_part - ? builder::make_cast(var_dtype, res_mask) - : builder::make_select(condition, full_mask, - builder::make_cast(var_dtype, res_mask)); - } - return just_tail_part - ? (full_mask >> builder::make_cast(var_dtype, tail_len)) - : builder::make_select( - condition, full_mask, - (full_mask >> builder::make_cast(var_dtype, tail_len))); -} - -expr generate_mask_var_by_step(stmt &mask_def, const expr &cur_step, - int32_t step, const expr &sup_condition, - bool direct_sup_cond) { - // notice: cur_step must be s32 - sc_data_type_t var_dtype; - uint64_t init_value; - choose_mask_vartype_init_value(var_dtype, init_value, step); - auto mask_select = generate_mask_by_step_directly( - cur_step, step, sup_condition, direct_sup_cond); - auto mask = - builder::make_var(var_dtype, "__mask_" + std::to_string(var_idx++)); - mask_def = builder::make_var_tensor_def_unattached(mask, linkage::local, - mask_select); - return mask; -} - -expr generate_mask_by_step_directly(const expr &cur_step, int32_t step, - const expr &sup_condition, - bool direct_sup_cond) { - // notice: cur_step must be s32 - sc_data_type_t var_dtype; - uint64_t init_value; - choose_mask_vartype_init_value(var_dtype, init_value, step); - auto full_mask = builder::make_constant({init_value}, var_dtype); - auto empty_mask = builder::make_constant({UINT64_C(0)}, var_dtype); - expr empty_mask_condition; - empty_mask_condition = - (sup_condition.defined()) - ? (cur_step == 0 || - (direct_sup_cond ? sup_condition : !sup_condition)) - : (cur_step == 0); - return builder::make_select( - empty_mask_condition, empty_mask, - builder::make_select(cur_step == step, full_mask, - full_mask >> - builder::make_cast(var_dtype, step - cur_step))); -} - -/** Determine whether masks are needed during elementwise computation and - * generate conditional expressions for the mask - * @param graph the graph - * @param src input slice - * @param plain_dims plain shapes - * @param format input format - * @param iter_vars input loop vars - * @param lanes simd lanes - * @param condition key is related iter var, value is two exprs: first is - * current accumulated index, second is its plain shape upperbound. - * @param last_axis_mask mask count, how many elements should be computed in - * this time. -1 means all. - * */ -void compute_mask_and_generate_condition( - sc_graph_t &graph, const std::vector &src, - const sc_dims &plain_dims, sc_data_format_t format, - const std::vector &iter_vars, int lanes, - std::unordered_map> &conditions, - int &last_axis_mask) { - auto blocking_dims = - sc_data_format_t::get_blocking_shapes(plain_dims, format); - auto padded_dims = - sc_data_format_t::get_padded_plain_shapes(blocking_dims, format); - auto &format_code = format.format_code_; - if (plain_dims == padded_dims) { - return; - } - auto offset = src[0]->get_offset(); - auto shapes = src[0]->get_shape(); - size_t ndims = format_code.ndims(); - assert(offset.size() == ndims && shapes.size() == ndims && - iter_vars.size() == ndims); - auto plain2block = format_code.collect_p2b_mapping(); - for (size_t i = 0; i < plain2block.size(); i++) { - auto &orig_dim = i; - if (plain_dims[orig_dim] == padded_dims[orig_dim] || - plain2block[i].size() == 1) { - continue; - } - auto &block_dim = plain2block[i][plain2block[i].size() - 1]; - auto blocks = format_code.collect_blocking_index(orig_dim); - int padding_count = 0; - conditions[iter_vars[block_dim]].first = - iter_vars[block_dim] + offset[block_dim]; - for (int b = static_cast(blocks.size()) - 1; b >= 0; b--) { - if (b > 0 && blocks[b - 1] % blocks[b] != 0) { - padding_count++; - } - conditions[iter_vars[block_dim]].first = - conditions[iter_vars[block_dim]].first + - (iter_vars[plain2block[i][b]] + offset[plain2block[i][b]]) * - format.blocks_[blocks[b]]; - } - conditions[iter_vars[block_dim]].second = - graph.dim_to_expr(plain_dims[orig_dim]); - COMPILE_ASSERT(padding_count < 2, - "Currently we don't support multi-level padding mask."); - if (block_dim == format_code.ndims() - 1) { - assert(lanes > 1); - last_axis_mask = plain_dims[orig_dim] % lanes; - } - } -} - -/** Get indexing based on different conditions - * @param is_lastdim_meet_require whether the shape len >= threshold - * @param has_tail whether has tail - * @param input input slice - * @param input_idx input index - * @param lanes simd lanes - * @param res_idx the final indexing we want to get - * @param axis_len length of indexing axis - * @param iter_var var of indexing axis - * @param just_tail_part floor part and tail part is calculated separately - * */ -expr indexing_from_diff_cond(const bool is_lastdim_meet_require, - const bool has_tail, const tensor_slice &input, - std::vector &input_idx, const int32_t lanes, - expr &res_idx, const expr &axis_len, - const expr &iter_var, const expr &floor, - bool just_tail_part) { - if (is_lastdim_meet_require) { - res_idx = builder::make_indexing(input.tptr_, input_idx); - } else if (has_tail && utils::is_one_of(lanes, 4, 8, 16, 32, 64)) { - auto mask = last_dim_generate_mask(iter_var, floor, axis_len, lanes, - just_tail_part); - res_idx = builder::make_indexing(input.tptr_, input_idx, lanes, mask); - } else { - res_idx = builder::make_indexing(input.tptr_, input_idx, lanes); - } - return res_idx; -} - -// Whether all the input shapes are blocking formats. -bool is_op_input_blocking_shape(const sc_op_info_t &info) { - return std::all_of(info.inputs_.begin(), info.inputs_.end(), - [](const graph_tensor_ptr &in) { - return in->details_.get_format().is_blocking(); - }); -} - -void vec_backend_require(const context_ptr &ctx, bool &use_vectorized) { -// llvm and g++ will perform special optimization on the scalar version, -// resulting in the performance of our vectorized version not being as good -// as the scalar version. Currently, these two backends still maintain the -// scalar method of tail processing. The builtin will use our vectorized -// version. -#if SC_BUILTIN_JIT_ENABLED - if (ctx->flags_.jit_kind_ == jit_kind::xbyak) { - use_vectorized = true; - } else { - use_vectorized = false; - } -#else - use_vectorized = false; -#endif -} - -void compute_vectorized_op(const context_ptr &ctx, sc_graph_t &graph, - const std::vector &src, - const tensor_slice &dst, sc_op_info_t &info, - const vectorized_info_t &vx_info, - const mask_compute_func_t &compute_lanes, - const mask_compute_func_t &compute_scalar, - any_map_t &attrs, const graph_tensor_ptr &expand_gt, - size_t wkld, bool use_mask, - const tensor_slice *expand_loop_by, - bool unroll_inner_loop) { - if (!expand_loop_by) { - expand_loop_by = &dst; - } - bool use_vectorized = false; - vec_backend_require(ctx, use_vectorized); - // In order to support non-stride test, we add dense_stride flag. - // If it is non-stride shape, we just use step = 1 to do - // this. - int graph_input_size = info.inputs_.size(); - bool dense_stride = std::all_of( - info.inputs_.begin(), info.inputs_.end(), - [](const graph_tensor_ptr &in) { return in->details_.is_dense(); }); - bool is_blocking_shape = is_op_input_blocking_shape(info); - // nested loop vars - std::vector iter_vars; - // the indices for multiple inputs. First dim: the input, Second dim: - // the dimemsions in the tensor - std::vector> src_indices_floor(src.size()); - std::vector> src_indices_tail(src.size()); - // the indices for the output tensor - std::vector dst_idx_floor; - std::vector dst_idx_tail; - for (unsigned i = 0; i < expand_loop_by->nslice_dims(); i++) { - // make the loop var for the for-loop - iter_vars.emplace_back( - range_from_outer_loop(expand_loop_by->get_ranges()[i]) - ? expr(0) - : builder::make_var(datatypes::index, std::string("_fuseiter") + - std::to_string(idx++))); - // for each input tensor - for (size_t j = 0; j < src.size(); j++) { - auto &src_idx_floor = src_indices_floor.at(j); - auto &src_idx_tail = src_indices_tail.at(j); - // push an index - src_idx_floor.emplace_back(iter_vars.back()); - src_idx_tail.emplace_back(iter_vars.back()); - } - // push an index for output tensor - dst_idx_floor.emplace_back(iter_vars.back()); - dst_idx_tail.emplace_back(iter_vars.back()); - } - auto tail_var = builder::make_var( - datatypes::index, std::string("_fuseiter") + std::to_string(idx++)); - for (size_t j = 0; j < src.size(); j++) { - auto &src_idx_tail = src_indices_tail.at(j); - src_idx_tail[vx_info.axis] = tail_var; - } - dst_idx_tail[vx_info.axis] = tail_var; - expr indexed_target_floor; - - auto bld = builder::get_current_builder(); - COMPILE_ASSERT(bld, "No active builder is set"); - int lanes = static_cast(vx_info.lanes); - auto slice_len = expand_loop_by->get_shape().at(vx_info.axis); - auto floor = do_cast_and_fold(slice_len / lanes * lanes); - auto tail = do_cast_and_fold(slice_len % lanes); - int floor_int = 0; - int tail_int = 0; - if (floor.isa()) { - floor_int = get_expr_as_int(floor); - tail_int = get_expr_as_int(tail); - COMPILE_ASSERT((floor_int + tail_int), "Don't support shape len = 0."); - } - const int INVALID_AXIS_MASK = -64; - int last_axis_mask = INVALID_AXIS_MASK; - std::unordered_map> conditions; - if (use_mask) { - compute_mask_and_generate_condition( - graph, src, info.inputs_[0]->details_.get_plain_dims(), - info.inputs_[0]->details_.get_format(), iter_vars, vx_info.lanes, - conditions, last_axis_mask); - } - if (last_axis_mask != INVALID_AXIS_MASK && floor_int > 0) { - COMPILE_ASSERT(tail_int == 0, - "Currently we only support mask in vectorize compute not " - "tail."); - } - std::vector tcur; - stmt cur; - int loop_size = static_cast(expand_loop_by->get_shape().size()); - bool tail_threshold = tail.isa() && tail_int <= 1; - bool use_scalar = - !use_vectorized || tail_threshold || lanes == 1 || !dense_stride; - // recover schedule loop - for (int i = loop_size - 1; i >= 0; i--) { - stmt body; - // currently vx_axis should be last axis - if (loop_size == vx_info.axis + 1 && i == vx_info.axis) { - if (dense_stride && (!floor.isa() || floor_int)) { - bld->push_scope(); - // if the shape is less than lanes, we don't use mask to - // process. - - indexing_from_diff_cond(false, false, dst, dst_idx_floor, lanes, - indexed_target_floor, slice_len, - iter_vars.at(i), floor); - std::vector indexed_input_floor; - expr input_floor_idx; - for (unsigned j = 0; j < src.size(); j++) { - indexed_input_floor.emplace_back(indexing_from_diff_cond( - false, false, *src.at(j), src_indices_floor.at(j), lanes, - input_floor_idx, slice_len, iter_vars.at(i), floor)); - } - std::vector target_floor = { - expr::lvalue_proxy_t(indexed_target_floor, false)}; - auto cond_it = conditions.find(iter_vars[i]); - if (cond_it != conditions.end()) { - assert(last_axis_mask != INVALID_AXIS_MASK); - cur = compute_lanes(indexed_input_floor, target_floor, - cond_it->second.first, cond_it->second.second, - vx_info.lanes); - } else { - cur = compute_lanes(indexed_input_floor, target_floor); - } - cur->attr()[op_traits::workload_computable_t::workload_number] = wkld; - bld->emit(cur); - cur = bld->pop_scope(); - if (iter_vars.at(i).isa()) { - cur = make_stmt(iter_vars.at(i), expr(0), floor, - expr(lanes), cur, true, - for_type::NORMAL); - if (unroll_inner_loop) { - cur->attr()[stmt_attr_key::unroll_loop] = 0; - } - bind_loop_axis(expand_gt, cur, i, true); - } - tcur.emplace_back(cur); - } - if (((!tail.isa() && !is_blocking_shape) || tail_int) || - !dense_stride) { - bld->push_scope(); - - std::vector indexed_input_tail; - expr mask; - if (!use_scalar) { - mask = - last_dim_generate_mask(tail_var, floor, slice_len, lanes, true); - } - expr indexed_target_tail = builder::make_indexing( - dst.tptr_, dst_idx_tail, use_scalar ? 1 : lanes, mask); - for (unsigned j = 0; j < src.size(); j++) { - indexed_input_tail.emplace_back( - builder::make_indexing(src.at(j)->tptr_, src_indices_tail.at(j), - use_scalar ? 1 : lanes, mask)); - } - std::vector target_tail = { - expr::lvalue_proxy_t(indexed_target_tail, false)}; - if (use_scalar) { - cur = compute_scalar(indexed_input_tail, target_tail); - } else { - cur = compute_lanes(indexed_input_tail, target_tail); - } - cur->attr()[op_traits::workload_computable_t::workload_number] = wkld; - bld->emit(cur); - cur = make_stmt( - tail_var, !dense_stride ? expr(0) : floor, slice_len, - use_scalar ? expr(1) : expr(lanes), bld->pop_scope(), true, - for_type::NORMAL); - if (unroll_inner_loop) { - cur->attr()[stmt_attr_key::unroll_loop] = 0; - } - bind_loop_axis(expand_gt, cur, i, true); - tcur.emplace_back(cur); - } - } else if (iter_vars.at(i).isa()) { - // Do not generate those dummy loop - if (!tcur.empty() && tcur[0].defined()) { - body = make_stmt(std::move(tcur)); - tcur.clear(); - // address special condition, like temp_buffer is used - cur = make_stmt(std::move(iter_vars.at(i)), expr(0), - expand_loop_by->get_shape().at(i), - expr(1), std::move(body), true, - for_type::NORMAL); - } else if (cur.defined()) { - body = make_stmt(std::vector{std::move(cur)}); - // address special condition, like temp_buffer is used - cur = make_stmt(std::move(iter_vars.at(i)), expr(0), - expand_loop_by->get_shape().at(i), - expr(1), std::move(body), true, - for_type::NORMAL); - } else { - // if cur not defined, means last axis of tensor slice has - // range 1, e.g. tensor_slice{{i, 100},{0, 1}} - indexed_target_floor = builder::make_indexing(dst.tptr_, dst_idx_floor); - std::vector indexed_input_floor; - for (unsigned j = 0; j < src.size(); j++) { - indexed_input_floor.emplace_back(builder::make_indexing( - src.at(j)->tptr_, src_indices_floor.at(j))); - } - - std::vector target_floor = { - expr::lvalue_proxy_t(indexed_target_floor, false)}; - bld->push_scope(); - cur = compute_scalar(indexed_input_floor, target_floor); - cur->attr()[op_traits::workload_computable_t::workload_number] = wkld; - bld->emit(cur); - cur = make_stmt( - iter_vars.at(i), expr(0), expand_loop_by->get_shape().at(i), - expr(1), bld->pop_scope(), true, for_type::NORMAL); - if (unroll_inner_loop) { - cur->attr()[stmt_attr_key::unroll_loop] = 0; - } - } - bind_loop_axis(expand_gt, cur, i, true); - } - } - if (!tcur.empty() && tcur[0].defined()) { - // TODO(xxx): currenly we don't add merge_loop attribute for this - // special case, need stronger loop analysis. - for (auto &it : tcur) { - bld->emit(it); - } - // TODO(yifei): analyze whether this is safe enough - cur->attr()[stmt_attr_key::merge_loop] = true; - } else { - cur->attr()[stmt_attr_key::merge_loop] = true; - bld->emit(cur); - } -} - -size_t get_dims_product(const sc_dims &dims) { - sc_dim ret = 1; - // todo: find out how to use this function in dynamic cases. - for (unsigned i = 0; i < dims.size(); ++i) { - if (!is_dynamic_dim(dims[i])) { - ret *= dims[i]; - } - } - assert(ret >= 0 && "Overflow or non-constant shape detected"); - return ret; -} - -int get_number_of_squeeze_dims(const sc_dims &dims) { - int ret = 0; - for (auto &it : dims) { - if (it == 1) { - ret++; - } - } - return ret; -} - -bool loop_can_be_fused(const for_loop &loop) { - return get_expr_as_int(loop->step_) == INT64_C(1); -} - -slice_range_map search_known_input_slice(sc_op *cur, fslice_map &fsmap) { - slice_range_map known_ranges_map; - auto input_size = cur->get_inputs().size(); - for (size_t i = 0; i < input_size; i++) { - auto &input = cur->get_inputs()[i]; - if (!fsmap.get(input).empty()) { - known_ranges_map[i] = fsmap.get(input); - } - } - return known_ranges_map; -} - -void set_unknown_input_slice(fusible_op_t *cur, - const slice_range_map &known_ranges_map, - fslice_map &fsmap) { - // set other unknown ranges. - auto input_size = cur->get_inputs().size(); - for (size_t i = 0; i < input_size; i++) { - auto input = cur->get_inputs()[i]; - auto &inp_slice = fsmap.get(input); - if (inp_slice.empty()) { - inp_slice = *utils::find_map_value(known_ranges_map, i).get(); - } - } -} - -std::unordered_map -search_known_input_axis(sc_op *cur, binding_axis_map &bdax_map) { - std::unordered_map known_axis_map; - auto input_size = cur->get_inputs().size(); - for (size_t i = 0; i < input_size; i++) { - auto &input = cur->get_inputs()[i]; - if (!bdax_map.get(input).empty()) { - known_axis_map[i] = bdax_map.get(input); - } - } - COMPILE_ASSERT(!known_axis_map.empty(), "No binded input axis found for " - << cur->op_name_ - << cur->logical_op_id_) - return known_axis_map; -} - -void call_output_user_axis_binding(sc_op *cur, binding_axis_map &bdax_map) { - for (auto &out : cur->get_outputs()) { - for (auto &user : out->uses_) { - if (auto bd_op = - user.second->dyn_cast()) { - bd_op->infer_binding_axis(bdax_map); - } - } - } -} - -void set_unknown_binding_axis( - sc_op *cur, const std::unordered_map &known_axis_map, - binding_axis_map &bdax_map) { - // set other unknown axis. - auto input_size = cur->get_inputs().size(); - // prepare infer binding axis upwards - std::unordered_set pre_infer_list; - for (size_t i = 0; i < input_size; i++) { - auto input = cur->get_inputs()[i]; - auto &inp_axis = bdax_map.get(input); - if (inp_axis.empty()) { - inp_axis = *utils::find_map_value(known_axis_map, i).get(); - auto producer = input->producer_owner_; - if (producer->isa()) - continue; - pre_infer_list.insert(producer); - } - } - for (auto &op : pre_infer_list) { - if (auto preop = op->dyn_cast()) { - preop->pre_infer_binding_axis(bdax_map); - // in avoid of more than one users cases - call_output_user_axis_binding(op, bdax_map); - } - } - // call output - call_output_user_axis_binding(cur, bdax_map); -} - -std::vector -transform_axis_plain2blocking(const logical_tensor_t <, - const std::vector &plain_axis) { - auto fmt = lt.get_format(); - // If format is any, just return. - if (fmt.is_any()) { - return plain_axis; - } - std::vector real_axis; - auto p2bmp = fmt.format_code_.collect_p2b_mapping(); - for (auto &i : plain_axis) { - std::vector res; - res.resize(p2bmp[i].size()); - std::transform(p2bmp[i].begin(), p2bmp[i].end(), res.begin(), - [](const int &v) { return v; }); - real_axis.insert(real_axis.end(), res.begin(), res.end()); - } - std::sort(real_axis.begin(), real_axis.end()); - return real_axis; -} - -std::vector -transform_axis_plain2blocking(const graph_tensor_ptr >, - const std::vector &plain_axis) { - return transform_axis_plain2blocking(gt->details_, plain_axis); -} - -std::vector -transform_axis_blocking2plain(const logical_tensor_t <, - const std::vector &blocking_axis) { - auto fmt = lt.get_format(); - // If format is any, just return. - if (fmt.is_any()) { - return blocking_axis; - } - std::vector plain_axis; - auto p2bmp = fmt.format_code_.collect_p2b_mapping(); - for (auto &ax : blocking_axis) { - for (size_t i = 0; i < p2bmp.size(); i++) { - auto blk_axis_i = p2bmp[i]; - if (blk_axis_i.end() != - std::find(blk_axis_i.begin(), blk_axis_i.end(), ax)) { - plain_axis.emplace_back(i); - break; - } - } - } - // check if empty to make g++12 happy - if (!plain_axis.empty()) { - // remove duplicated axis. - std::sort(plain_axis.begin(), plain_axis.end()); - plain_axis.erase(std::unique(plain_axis.begin(), plain_axis.end()), - plain_axis.end()); - } - return plain_axis; -} - -bool is_dynamic_slice_range_list(const slice_range_list &in_slice_range_list) { - for (auto &range : in_slice_range_list) { - auto shapes = get_slice_shape(range); - for (auto &shape : shapes) { - auto folded_shape = do_cast_and_fold(shape); - if (!folded_shape.isa()) { - return true; - } - } - } - return false; -} - -/** - * Compare left and right fsmap - * */ -cmp_res cmp_slice_range(const slice_range_list &left_slice_range_list, - const slice_range_list &right_slice_range_list) { - size_t left_slice_size = 0, right_slice_size = 0; - COMPILE_ASSERT(!left_slice_range_list.empty() && - !right_slice_range_list.empty(), - "slice range should be set"); - if (is_dynamic_slice_range_list(left_slice_range_list) || - is_dynamic_slice_range_list(right_slice_range_list)) { - auto &left_slice_range = left_slice_range_list[0]; - auto &right_slice_range = right_slice_range_list[0]; - assert(left_slice_range.size() == right_slice_range.size()); - for (size_t i = left_slice_range.size(); i > 0; i--) { - auto left_shape = do_cast_and_fold(left_slice_range[i - 1].second); - auto right_shape = do_cast_and_fold(right_slice_range[i - 1].second); - if (!left_shape->equals(right_shape) && - !(left_shape.isa() && right_shape.isa() && - get_expr_as_int(left_shape) == get_expr_as_int(right_shape))) { - if (left_shape.isa() && right_shape.isa()) { - if (get_expr_as_int(left_shape) > get_expr_as_int(right_shape)) { - left_slice_size++; - } else { - right_slice_size++; - } - } else if (left_shape.isa()) { - right_slice_size++; - } else if (right_shape.isa()) { - left_slice_size++; - } - } - } - } else { - for (auto &left_slice_range : left_slice_range_list) { - auto left_slice_shape = - get_expr_to_dims(get_slice_shape(left_slice_range)); - if (get_dims_product(left_slice_shape) > left_slice_size) { - left_slice_size = get_dims_product(left_slice_shape); - } - } - for (auto &right_slice_range : right_slice_range_list) { - auto right_slice_shape = - get_expr_to_dims(get_slice_shape(right_slice_range)); - if (get_dims_product(right_slice_shape) > right_slice_size) { - right_slice_size = get_dims_product(right_slice_shape); - } - } - } - // if right anchor is more smaller than the leftrent one - if (left_slice_size == right_slice_size) { - return cmp_res::equal; - } else if (left_slice_size < right_slice_size) { - return cmp_res::l_less_r; - } else { - return cmp_res::l_larger_r; - } -} - -bool is_reshaped_tensor(const expr &tsr) { - COMPILE_ASSERT(tsr.isa(), - "except for tensor node, only tensorptr node is expected, but " - "got " - << tsr); - if (tsr.static_as()->is_slice_) - return false; - auto base = tsr.static_as()->base_; - COMPILE_ASSERT(base.isa(), - "tensor_ptr base should be indexing, but got: " << base); - for (auto &idx : base.static_as()->idx_) { - if (!idx.isa() || get_expr_as_int(idx) != 0) - return false; - } - auto base_tensor = base.static_as()->ptr_; - COMPILE_ASSERT(base_tensor.isa(), "Tensor type is expected") - auto base_dims = base_tensor.static_as()->dims_; - auto new_dims = tsr.static_as()->shape_; - return get_dims_product(get_expr_to_dims(base_dims)) == - get_dims_product(get_expr_to_dims(new_dims)); -} - -static std::vector get_dense_stride(const std::vector &shape) { - std::vector result(shape.size(), 1); - for (int i = shape.size() - 2; i >= 0; --i) { - result[i] = result[i + 1] * shape[i + 1]; - } - return result; -} - -expr transform_tsr2stsr_with_range(const expr &tsr, const slice_range &range) { - auto new_dims = get_slice_shape(range); - std::vector new_strides; - tensor t; - if (tsr.isa()) { - t = tsr.static_as(); - new_strides = t->strides_; - } else { - COMPILE_ASSERT(is_reshaped_tensor(tsr), - "reshaped tensor is expected, but got " << tsr); - t = tsr.static_as() - ->base_.static_as() - ->ptr_.static_as(); - new_strides = get_dense_stride(tsr.static_as()->shape_); - } - return builder::make_stensor(t->name_ + "_strd", new_dims, new_strides, - t->elem_dtype_); -} - -expr transform_tsl2stsr(const tensor_slice &tsl) { - return transform_tsr2stsr_with_range(tsl.get_real_tensor(), tsl.get_ranges()); -} - -expr transform_tsr2tptr_with_range(const expr &tsr, const slice_range &range) { - auto new_dims = get_slice_shape(range); - return builder::tensor_ptr(tsr, get_slice_idx(range), get_slice_shape(range), - true); -} - -expr transform_tptr2stsr(const expr &tptr) { - COMPILE_ASSERT(tptr.isa(), - "tensort pointer node is expected, but got " << tptr); - auto tp = tptr.static_as(); - COMPILE_ASSERT(tp->base_.isa(), "indexing node is expected"); - auto tsr = tp->base_->ptr_; - COMPILE_ASSERT(tsr.isa(), "tensor node is expected, but got " << tsr); - auto t = tsr.static_as(); - return builder::make_stensor(t->name_ + "_strd", tp->shape_, t->strides_, - t->elem_dtype_); -} - -float evaluate_loop_parallel_balance(const std::vector &loops, - bool check_use_full_threads) { - expr dummy; - return evaluate_loop_parallel_balance(loops, dummy, check_use_full_threads); -} - -float evaluate_loop_parallel_balance(const std::vector &loops, - expr &cond, bool check_use_full_threads) { - expr dyn_loops = 1; - sc_dim stc_loops = 1; - const int run_threads = runtime_config_t::get().get_num_threads(); - cond = false; - // the minor dyn cost is used for case `dyn_var1 + dyn_var2` and - // `dyn_var1` only comparison. - float minor_dyn_parallelism = 0.f; - const float minor_dyn_parallelism_step = 1e-4f; - for (size_t i = 0; i < loops.size(); i++) { - auto &loop = loops[i]; - if (!(loop->iter_begin_.isa() && - loop->iter_end_.isa())) { - dyn_loops = dyn_loops * (loop->iter_end_ - loop->iter_begin_); - minor_dyn_parallelism += minor_dyn_parallelism_step; - } else { - auto begin = get_expr_as_int(loop->iter_begin_), - end = get_expr_as_int(loop->iter_end_); - COMPILE_ASSERT(end > begin, "loop end is expected to larger than begin") - stc_loops = stc_loops * (end - begin); - dyn_loops = dyn_loops * static_cast(stc_loops); - } - } - if (check_use_full_threads) { - cond = dyn_loops < run_threads; - return stc_loops >= run_threads; - } - cond = !((dyn_loops % run_threads == 0) || (dyn_loops > run_threads * 8)); - if (stc_loops == 1) { - return 0.0f + minor_dyn_parallelism; - } - bool parallelism = (stc_loops / run_threads > 8) || - (stc_loops % run_threads == 0 && stc_loops >= run_threads); - float cal_parallelism = ((stc_loops % run_threads) / float(run_threads)); - if (stc_loops < run_threads) { - cal_parallelism += minor_dyn_parallelism; - } - return parallelism ? 1.0f : cal_parallelism; -} - -bool range_from_outer_loop(const std::pair &range) { - return !range.first.isa() && range.second.isa() && - get_expr_as_int(range.second) == 1; -} - -bool slice_full_on_axis(const sc_dims &dim, const slice_range &ranges, - const std::vector &axis) { - for (auto &ax : axis) { - auto first = do_cast_and_fold(ranges[ax].first); - auto second = do_cast_and_fold(ranges[ax].second); - // slice range length should equal to dims - if (second.isa()) { - if (get_const_as_int(second.checked_as()) != dim[ax]) { - return false; - } else if (dim[ax] == 1) { - continue; - } - } - if (!first.isa()) { - if (first->node_type_ == sc_expr_type::mul) { - auto rv = constant_folding::get_operand_from_binary(first).second; - // {i * block, block} case where `block_size==dims[i]` - if (rv.isa() && - get_const_as_int(rv.static_as()) == dim[ax]) - continue; - } - return false; - } else if (get_const_as_int(first.static_as()) != 0) { - return false; - } - } - return true; -} - -bool slice_divisible_on_axis(const sc_dims &dim, const slice_range &ranges, - const std::vector &axis) { - for (auto &ax : axis) { - auto second = do_cast_and_fold(ranges[ax].second); - // slice range length should be divisible by dims - if (second.isa() && - dim[ax] % get_const_as_int(second.checked_as()) != 0) { - return false; - } - } - return true; -} - -bool slice_divisible_by_factor(const slice_range &ranges, - const std::vector &axis, const int factor) { - for (auto &ax : axis) { - auto second = do_cast_and_fold(ranges[ax].second); - // slice range length should be divisible by dims - if (second.isa()) { - if (get_const_as_int(second.checked_as()) % factor != 0) { - return false; - } - } else { - return false; - } - } - return true; -} - -bool slice_larger_than_bound_on_axis(const slice_range &ranges, - const std::vector &axis, - const int factor, const int lower_bound) { - auto total_len = 1; - for (auto &ax : axis) { - auto second = do_cast_and_fold(ranges[ax].second); - if (second.isa()) { - total_len *= get_const_as_int(second.checked_as()); - } else { - return false; - } - } - if (total_len / factor < lower_bound) { - return false; - } - return true; -} - -bool innermost_slice_with_non_dividable_lanes(const context_ptr &ctx, - const slice_range &slice, - const sc_data_type_t &dtype, - sc_dim &floor, sc_dim &tail) { - if (is_dynamic_slice_range_list({slice})) - return false; - auto shape = get_slice_shape(slice); - auto dims = get_expr_to_dims(shape); - auto last_dim = dims.back(); - // check dims except last are all one - if (get_dims_product(dims) != (size_t)last_dim) - return false; - // get max lanes - auto lanes = vectorize_step(ctx, dtype.type_code_); - if ((last_dim > lanes) && (last_dim % lanes != 0)) { - floor = last_dim / lanes * lanes; - tail = last_dim % lanes; - return true; - } - return false; -} - -int get_slice_size(const slice_range &ranges, const int dtype_size) { - auto total_size = dtype_size; - for (auto &range : ranges) { - auto second = do_cast_and_fold(range.second); - if (second.isa()) { - total_size *= get_const_as_int(second.checked_as()); - } else { - return -1; - } - } - return total_size; -} - -bool slice_expr_equals(const expr &in1, const expr &in2) { - auto fin1 = do_cast_and_fold(in1); - auto fin2 = do_cast_and_fold(in2); - if (fin1->equals(fin2)) { - return true; - } - // datatype may not equal during slice compare. - if (fin1.isa() && fin2.isa()) { - return get_expr_as_int(fin1) == get_expr_as_int(fin2); - } - return false; -} - -expr cast_to_s32(const expr &in) { - return builder::make_cast(datatypes::s32, in); -} - -std::vector -get_sorted_inputs_by_layout_input(const sc_op_ptr &op) { - int layout_input_index = - op->attrs_.get_or_else(op_attr_key::layout_input_index, 0); - if (layout_input_index == 0) { - return op->get_inputs(); - } - std::vector ret; - auto inps = op->get_inputs(); - ret.reserve(inps.size()); - ret.push_back(inps[layout_input_index]); - for (int i = 0; i < static_cast(inps.size()); i++) { - if (i == layout_input_index) { - continue; - } - ret.push_back(inps[i]); - } - return ret; -} - -variant numeric_limits_minimum(sc_data_etype type_code) { - if (type_code == sc_data_etype::F32 || type_code == sc_data_etype::BF16) { - return -std::numeric_limits::infinity(); - } else if (type_code == sc_data_etype::F16) { - return (float)-65504; - } else if (type_code == sc_data_etype::U8 || - type_code == sc_data_etype::U32) { - return int64_t(0); - } else if (type_code == sc_data_etype::S8) { - return int64_t(-128); - } else if (type_code == sc_data_etype::S32) { - return int64_t(std::numeric_limits::min()); - } else { - COMPILE_ASSERT(0, "unsupported data_etype"); - } -} - -variant numeric_limits_maximum(sc_data_etype type_code) { - if (type_code == sc_data_etype::F32 || type_code == sc_data_etype::BF16) { - return std::numeric_limits::infinity(); - } else if (type_code == sc_data_etype::F16) { - return (float)65504; - } else if (type_code == sc_data_etype::S8) { - return int64_t(127); - } else if (type_code == sc_data_etype::S32) { - return int64_t(std::numeric_limits::max()); - } else if (type_code == sc_data_etype::U8) { - return int64_t(255); - } else if (type_code == sc_data_etype::U32) { - return int64_t(std::numeric_limits::max()); - } else { - COMPILE_ASSERT(0, "unsupported data_etype"); - } -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/fusible_op_utils.hpp b/legacy/core/src/compiler/ir/graph/fusible_op_utils.hpp deleted file mode 100644 index e71918da8..000000000 --- a/legacy/core/src/compiler/ir/graph/fusible_op_utils.hpp +++ /dev/null @@ -1,230 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_FUSIBLE_OP_UTILS_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_FUSIBLE_OP_UTILS_HPP - -#include "fusible_op.hpp" -#include "fusion_data.hpp" -#include "util/variant.hpp" -#include -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -enum class cmp_res : int { - unknown = -1, - equal = 0, - l_less_r = 1, - l_larger_r = 2, -}; - -using slice_range_map = std::unordered_map; -slice_range_map search_known_input_slice(sc_op *cur, fslice_map &fsmap); -void set_unknown_input_slice(fusible_op_t *cur, - const slice_range_map &known_ranges_map, - fslice_map &fsmap); -infer_status_code infer_binary_slice_ranges(fusible_op_t *cur, - fslice_map &fsmap); - -std::unordered_map -search_known_input_axis(sc_op *cur, binding_axis_map &bdax_map); -void set_unknown_binding_axis( - sc_op *cur, const std::unordered_map &known_axis_map, - binding_axis_map &bdax_map); -void call_output_user_axis_binding(sc_op *cur, binding_axis_map &bdax_map); -void infer_identical_binding_axis(fusible_op_t *cur, - binding_axis_map &bdax_map); -void pre_infer_identical_binding_axis(fusible_op_t *cur, - binding_axis_map &bdax_map); - -sc_dims get_expr_to_dims(const std::vector &dims); -size_t get_dims_product(const sc_dims &dims); -// the dim can be squeezed is 1 -int get_number_of_squeeze_dims(const sc_dims &dims); - -bool range_from_outer_loop(const std::pair &range); - -bool slice_full_on_axis(const sc_dims &dim, const slice_range &ranges, - const std::vector &axis); - -bool slice_divisible_on_axis(const sc_dims &dim, const slice_range &ranges, - const std::vector &axis); - -bool slice_divisible_by_factor(const slice_range &ranges, - const std::vector &axis, const int factor); - -bool slice_larger_than_bound_on_axis(const slice_range &ranges, - const std::vector &axis, - const int factor, const int lower_bound); - -int get_slice_size(const slice_range &ranges, const int dtype_size = 1); - -inline uint16_t vectorize_step(const context_ptr &ctx, sc_data_etype detype) { - // eg: bf16 or s8u8 always promote to f32 or s32 to do calculation, we need - // to limited bf16 max lanes is 8 under avx2 environment. - auto avx2_lanes_require_dtype = [](const sc_data_etype detype) { - return detype == sc_data_etype::BF16 || detype == sc_data_etype::S8 || - detype == sc_data_etype::U8; - }; - if (!ctx->machine_.cpu_flags_.fAVX512F && avx2_lanes_require_dtype(detype)) { - assert(ctx->machine_.cpu_flags_.fAVX2); - return std::min(uint16_t(8), ctx->get_max_vector_lanes(detype)); - } - return std::min(uint16_t(16), ctx->get_max_vector_lanes(detype)); -} -bool loop_can_be_fused(const for_loop &loop); - -// use new fusion mgr to lowering single fusible op -ir_module_ptr fusible_op_get_func(fusible_op_t *op, const context_ptr &ctx); - -struct mask_compute_func_t { - mask_compute_func_t( - const std::function &, - std::vector &)> &func) - : impl_(func) {} - stmt operator()(const std::vector &in, - std::vector &out, - const expr &cur_idx = expr(), - const expr &upper_bound = expr(), uint32_t lanes = 16) const; - std::function &, - std::vector &)> - impl_; -}; - -using fusion_compute_func_t = std::function &, std::vector &)>; -bool is_op_input_blocking_shape(const sc_op_info_t &info); - -void compute_vectorized_op(const context_ptr &ctx, sc_graph_t &graph, - const std::vector &src, - const tensor_slice &dst, sc_op_info_t &info, - const vectorized_info_t &vx_info, - const mask_compute_func_t &compute_lanes, - const mask_compute_func_t &compute_scalar, - any_map_t &attrs, const graph_tensor_ptr &expand_gt, - size_t wkld = 0UL, bool use_mask = false, - const tensor_slice *expand_loop_by = - nullptr /*by default expand loop by dst*/, - bool unroll_inner_loop = false); -expr make_select_by_mask(const expr &, const expr &, const expr &, uint32_t); -expr generate_mask_var_by_step(stmt &mask_def, const expr &cur_step, - int32_t step, const expr &sup_condition = expr(), - bool direct_sup_cond = false); -expr generate_mask_by_step_directly(const expr &cur_step, int32_t step, - const expr &sup_condition = expr(), - bool direct_sup_cond = false); -expr calculate_mask_cur_step(const expr &len, const expr &iter_var, - const int32_t lanes); -expr indexing_from_diff_cond(const bool is_last_dim_1, const bool has_tail, - const tensor_slice &input, - std::vector &input_idx, const int32_t lanes, - expr &res_idx, const expr &axis_len, - const expr &iter_var, const expr &floor, - bool just_tail_part = false); -expr last_dim_generate_mask(const expr &iter_var, const expr &floor, - expr const &last_dim_len, int const &lanes, - bool just_tail_part = false); -void vec_backend_require(const context_ptr &ctx, bool &use_vectorized); -void compute_mask_and_generate_condition( - sc_graph_t &graph, const std::vector &src, - const sc_dims &plain_dims, sc_data_format_t format, - const std::vector &iter_vars, int lanes, - std::unordered_map> &conditions, - int &last_axis_mask); -void compute_block_elemwise(const std::vector &src, - const tensor_slice &dst, sc_op_info_t &info, - fusion_compute_func_t compute); - -std::vector -transform_axis_plain2blocking(const logical_tensor_t <, - const std::vector &plain_axis); - -std::vector -transform_axis_plain2blocking(const graph_tensor_ptr >, - const std::vector &plain_axis); - -std::vector -transform_axis_blocking2plain(const logical_tensor_t <, - const std::vector &blocking_axis); - -std::string fusion_create_var_idx(); -std::string fusion_create_idx(); - -cmp_res cmp_slice_range(const slice_range_list &left_slice_range_list, - const slice_range_list &right_slice_range_list); - -bool is_dynamic_slice_range_list(const slice_range_list &in_slice_range_list); - -// workload penalty coefficient for transpose/reorder measured by -// for(i, 0, 128){ -// for(j, 0, 256){ -// B[j, i] = A[i, j]; -// } -// } -// TODO(xxx): currently we mark this penalty on op, we will add loop -// analysis pass for tensor sequential access analysis in future -static constexpr size_t workload_penalty_coefficient = 16UL; - -float evaluate_loop_parallel_balance(const std::vector &loops, - bool check_use_full_threads = false); -// return static loop parallelism coefficient to satisfy the parallelism and -// the related condition expr. -float evaluate_loop_parallel_balance(const std::vector &loops, - expr &cond, - bool check_use_full_threads = false); -expr cast_to_s32(const expr &in); -// compare expr in slice equal or not, constant slice may have different -// datatypes but same value as we use `int` for static. -bool slice_expr_equals(const expr &in1, const expr &in2); -// sort op inputs by layout input index, the order is [layout input index, -// others...] -std::vector -get_sorted_inputs_by_layout_input(const sc_op_ptr &op); - -/** - * @return Bool: return if given slice comes from the inner most anchor with non - * dividable lanes - * @param ctx: context, used to query max lanes - * @param slice: given slice range - * @param dtype: data type kind, used to query max lanes - * @param floor: if return is True, recording `floor` value for last dims - * divided by max lanes - * @param tail: if return is True, recording `tail` value for last dims divided - * by max lanes - * */ -bool innermost_slice_with_non_dividable_lanes(const context_ptr &ctx, - const slice_range &slice, - const sc_data_type_t &dtype, - sc_dim &floor, sc_dim &tail); - -variant numeric_limits_minimum(sc_data_etype type_code); -variant numeric_limits_maximum(sc_data_etype type_code); - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/graph/fusion_anchor.cpp b/legacy/core/src/compiler/ir/graph/fusion_anchor.cpp deleted file mode 100644 index 8e7d386e4..000000000 --- a/legacy/core/src/compiler/ir/graph/fusion_anchor.cpp +++ /dev/null @@ -1,596 +0,0 @@ -/******************************************************************************* - * Copyright 2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "fusion_anchor.hpp" -#include "fusible_op_utils.hpp" -#include "mixed_partition.hpp" -#include -#include - -SC_MODULE(graph.fusion_anchor); - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -// get parent loop for the given node, if not found, return the top parent node -static stmt get_parent_loop_or_node(const stmt_c &node) { - stmt cur_node = node.remove_const(); - stmt parent_node = get_parent_node(cur_node); - while (parent_node.defined()) { - if (parent_node.isa()) { - return parent_node; - } - cur_node = parent_node; - parent_node = get_parent_node(parent_node); - } - return cur_node; -} - -stmt fusion_anchor_t::get_parent_loop() const { - return get_parent_loop_or_node(anchor_position_); -} - -void fusion_anchor_t::append_content(anchor_content_t content) { - std::vector contents; - // if content is fusion anchor, need to append all of its contents - if (auto fanchor = content.as_or_null()) { - contents = (*fanchor)->get_contents(); - } - contents.emplace_back(content); - append_contents(contents, content_number_map_.size()); - - // recursively attach to parent anchor - auto root = this; - while (root->parent_) { - auto num_id = root->parent_->content_number_map_[root]; - root = root->parent_.get(); - root->append_contents(contents, num_id); - } -} - -void fusion_anchor_t::clear_content(anchor_content_t content) { - if (auto fanchor = content.as_or_null()) { - COMPILE_ASSERT((*fanchor)->get_contents().empty(), - "the content of fusion anchor is not empty") - } - auto erase_content = [](fusion_anchor_t *fanchor, - const anchor_content_t &content, bool assert_last) { - auto iter = fanchor->content_number_map_.find(content); - // erase - if (iter != fanchor->content_number_map_.end()) { - COMPILE_ASSERT(!assert_last || iter->second + 1 == - fanchor->content_number_map_.size(), - "erased content is not last one") - fanchor->content_number_map_.erase(iter); - } - }; - // recursively remove content from parent anchor - auto root = this; - while (root) { - erase_content(root, content, root == this); - root = root->parent_.get(); - } -} - -static void inherit_parent_offset(slice_range_list &cur_value, - const slice_range_list &parent_value) { - // if slice range list size is not equal - if (cur_value.size() != parent_value.size()) - return; - for (size_t i = 0; i < cur_value.size(); i++) { - auto &cur_slice = cur_value[i]; - auto &parent_slice = parent_value[i]; - COMPILE_ASSERT(cur_slice.size() == parent_slice.size(), - "slice range size is not equal for the same graph " - "tensor") - for (size_t j = 0; j < cur_slice.size(); j++) { - // inherit offset - cur_slice[j].first = - do_cast_and_fold(parent_slice[j].first + cur_slice[j].first); - } - } -} - -void fusion_anchor_t::attach_parent_anchor( - const std::shared_ptr &parent, - const std::shared_ptr &repl_parent, bool inherit_offset) { - if (!parent) - return; - if (inherit_offset) { - for (auto &kv : fsmap_.datamap_) { - auto > = kv.first; - auto &cur_value = kv.second; - if (parent->fsmap_.hasvalue(gt)) { - inherit_parent_offset(cur_value, parent->fsmap_.get(gt)); - } - } - } - auto root = this; - while (root->parent_ && (root->parent_ != repl_parent)) { - COMPILE_ASSERT(root != root->parent_.get(), - "Ring parent anchor relationship found"); - root = root->parent_.get(); - } - if (root == parent.get()) - return; - root->parent_ = parent; - parent->append_content(root); -} - -void fusion_anchor_t::merge(const std::shared_ptr &other) { - fsmap_.datamap_.insert(other->fsmap_.datamap_.begin(), - other->fsmap_.datamap_.end()); - blocked_gt_set_.insert(other->blocked_gt_set_.begin(), - other->blocked_gt_set_.end()); - borrowed_fanchor_map_.insert(other->borrowed_fanchor_map_.begin(), - other->borrowed_fanchor_map_.end()); - auto contents_size = content_number_map_.size(); - for (auto &cont_numb_pair : other->content_number_map_) { - content_number_map_.insert(std::make_pair( - cont_numb_pair.first, cont_numb_pair.second + contents_size)); - } -} - -bool fusion_anchor_t::has_view_of(sc_op *op) { - auto iter = binded_mxp_->op_anchor_map_.find(op); - // if not found - if (iter == binded_mxp_->op_anchor_map_.end()) - return true; - auto op_anchor = iter->second; - while (true) { - if (op_anchor.get() == this) - return true; - if (op_anchor->content_number_map_.find(this) == - op_anchor->content_number_map_.end()) { - if (op_anchor->parent_) { - op_anchor = op_anchor->parent_; - } else - break; - } else if (*utils::find_map_value(op_anchor->content_number_map_, this) - .get() > - *utils::find_map_value(op_anchor->content_number_map_, op) - .get()) { - return true; - } else { - return false; - } - } - auto ths_root_scope = get_root()->get_parent_scope()->seq_; - stmt ths_anchor_ss = get_root()->anchor_position_; - auto ths_pos = std::find_if( - ths_root_scope.begin(), ths_root_scope.end(), - [&ths_anchor_ss](const stmt &s) { return ths_anchor_ss.ptr_same(s); }); - auto op_anchor_loop = op_anchor->get_parent_loop(); - while (true) { - auto op_anchor_pos = - std::find_if(ths_root_scope.begin(), ths_root_scope.end(), - [&op_anchor_loop](const stmt &s) { - return op_anchor_loop.ptr_same(s); - }); - if (op_anchor_pos != ths_root_scope.end()) { - return op_anchor_pos < ths_pos; - } - op_anchor_loop = get_parent_node(op_anchor_loop); - if (!op_anchor_loop.defined()) - return false; - } -} - -bool fusion_anchor_t::check_dep_for_op(const sc_op *op) { - if (op->isa() || op->isa()) - return true; - auto parti = binded_mxp_; - auto dep_m = parti->dep_m_; - for (auto &inp : op->get_inputs()) { - // check dep ops - for (auto &cur : parti->ops) { - if (std::any_of(cur->get_outputs().begin(), cur->get_outputs().end(), - [&parti](const graph_tensor_ptr >) { - return parti->is_parti_out(gt); - }) && - (dep_m->lookup(cur.get(), inp->producer_owner_) == 1 || - cur.get() == inp->producer_owner_)) { - // this fanchor should has view of depent op - if (!this->has_view_of(cur.get())) - return false; - } - } - } - return true; -} - -bool fusion_anchor_t::validate_input_for_op( - const sc_op *op, const std::unordered_set &known_gt) { - auto fanchor = this; - return std::all_of( - op->get_inputs().begin(), op->get_inputs().end(), - [&fanchor, &known_gt](const graph_tensor_ptr >) { - auto parti = fanchor->binded_mxp_; - auto dep_op = gt->producer_owner_; - // if the producer owner of gt is successfully inferred - if (known_gt.find(gt) != known_gt.end()) - return true; - // if the producer owner of gt is excluded in current - // partition - if (!parti->contains(dep_op)) { - return fanchor->check_dep_for_op(dep_op); - } - - // fanchor should has view of depent op - if (!fanchor->has_view_of(dep_op)) - return false; - - slice_range_list inferred_slice = fanchor->fsmap_.get(gt); - fusion_anchor_t *cur = fanchor; - - COMPILE_ASSERT(!dep_op->get_inputs().empty(), - dep_op->op_name_ << " op has no input") - // find the borrowed scope - auto dep_inp = dep_op->get_inputs()[0]; - while (cur->parent_) { - cur = cur->parent_.get(); - // if input is ready on cur anchor, its input slice range - // should not be empty - if (!cur->fsmap_.hasvalue(gt) || (!parti->is_parti_inp(dep_inp) && - !cur->fsmap_.hasvalue(dep_inp))) { - continue; - } - slice_range_list cur_slice = cur->fsmap_.get(gt); - auto res = cmp_slice_range(inferred_slice, cur_slice); - if (res != cmp_res::l_larger_r) { - fanchor->borrowed_fanchor_map_[gt] = cur->shared_from_this(); - return true; - } - } - return false; - }); -} - -void fusion_anchor_t::forbid_op( - const sc_op *op, const std::unordered_set &known_gt) { - auto fanchor = this; - std::for_each(op->get_inputs().begin(), op->get_inputs().end(), - [&known_gt, &fanchor](const graph_tensor_ptr >) { - if (known_gt.find(gt) == known_gt.end()) { - fanchor->fsmap_.datamap_.erase(gt.get()); - } - }); - std::for_each(op->get_outputs().begin(), op->get_outputs().end(), - [&known_gt, &fanchor](const graph_tensor_ptr >) { - if (known_gt.find(gt) == known_gt.end()) { - fanchor->fsmap_.datamap_.erase(gt.get()); - fanchor->blocked_gt_set_.insert(gt); - } - }); -} - -bool fusion_anchor_t::check_input_for_op( - const sc_op *op, std::unordered_set &known_gt) { - auto fanchor = this; - bool input_blocked = false; - for (auto > : op->get_inputs()) { - if (blocked_gt_set_.find(gt) != blocked_gt_set_.end()) { - std::for_each(op->get_outputs().begin(), op->get_outputs().end(), - [&fanchor](const graph_tensor_ptr >) { - fanchor->blocked_gt_set_.insert(gt); - }); - input_blocked = true; - break; - } - if (fanchor->fsmap_.hasvalue(gt)) { - if (fanchor->borrowed_fanchor_map_.find(gt) != - fanchor->borrowed_fanchor_map_.end()) - continue; - known_gt.insert(gt); - } - } - return !(input_blocked || known_gt.empty()); -} - -bool fusion_anchor_t::is_small_op_workload(const sc_op *op) { - if (op->is_dynamic() || !op->isa()) - return false; - auto fusible_op = op->dyn_cast(); - // collect inputs and outputs - auto gt_list = fusible_op->get_inputs(); - gt_list.insert(gt_list.end(), fusible_op->get_outputs().begin(), - fusible_op->get_outputs().end()); - // zero-init workload value - size_t wkld = 0; - // count of avaliable computation - size_t computed_count = 0; - // SigMa workload computation - for (auto > : gt_list) { - // if slice info is not found - if (!fsmap_.hasvalue(gt)) - continue; - auto &slice_list = fsmap_.get(gt); - // if unexpected cases found, directly return threshold - if (is_dynamic_slice_range_list(slice_list) || slice_list.size() != 1) - return false; - auto slice_shape = get_expr_to_dims(get_slice_shape(slice_list[0])); - // compute workload: tensor shrink shape / dtype step - wkld += utils::divide_and_ceil( - get_dims_product(slice_shape), - vectorize_step(get_binded_mxp()->ctx_, gt->details_.dtype_.type_code_)); - computed_count++; - }; - COMPILE_ASSERT(computed_count, "No workload computed") - // roughly compute if necessary - if (computed_count != gt_list.size()) { - wkld = wkld / computed_count * gt_list.size(); - } - return wkld < mixed_partition_hint::small_op_workload_threshold; -} - -void iter_fusion_anchor_t::commit(const stmt &s) { - if (cached_iter_anchor_.empty()) { - if (dispatch_helper_.isa()) { - anchor_position_->seq_.insert( - anchor_position_->seq_.end(), - dispatch_helper_.static_as()->seq_.begin(), - dispatch_helper_.static_as()->seq_.end()); - } else { - anchor_position_->seq_.emplace_back(dispatch_helper_); - } - } - // create cached_iter_anchor_ if necessary - if (cached_iter_anchor_.size() < iter_size_) { - stmts ss = s.isa() - ? s.static_as() - : builder::make_stmts_unattached({s}).checked_as(); - anchor_position_->seq_.emplace_back( - make_stmt(iter_ == iter_cnt_, ss, stmt())); - cached_iter_anchor_.emplace_back(ss); - } - // commit into cached_iter_anchor_ - else { - auto cached_anchor = cached_iter_anchor_.at(iter_cnt_); - if (s.isa()) { - cached_anchor->seq_.insert(cached_anchor->seq_.end(), - s.static_as()->seq_.begin(), - s.static_as()->seq_.end()); - } else { - cached_anchor->seq_.emplace_back(s); - } - } - iter_cnt_++; - if (iter_cnt_ == iter_size_) - iter_cnt_ = 0; -} - -void grouped_fusion_anchor_t::commit(const stmt &s) { - auto pos = anchor_position_->seq_.at(group_cnt_).checked_as(); - if (s.isa()) { - pos->seq_.insert(pos->seq_.end(), s.static_as()->seq_.begin(), - s.static_as()->seq_.end()); - } else { - pos->seq_.emplace_back(s); - } - group_cnt_++; - if (group_cnt_ == group_size_) - group_cnt_ = 0; -} - -stmt grouped_fusion_anchor_t::get_parent_loop() const { - // use last group anchor which is always the largest anchor - stmt cur_node = anchor_position_->seq_.front(); - if (group_size_ == 1) { - return get_parent_loop_or_node(cur_node); - } else { - stmt common_parent; - for (size_t i = 1; i < anchor_position_->seq_.size(); i++) { - auto next_node = anchor_position_->seq_[i]; - common_parent = get_common_parent_node(cur_node, next_node); - if (!common_parent.defined()) - return common_parent; - cur_node = common_parent; - } - return get_parent_loop_or_node(common_parent); - } -} - -void fusion_anchor_mgr_t::create_fusion_anchor(const slice_map &fsmap, - const fusion_anchor_ptr &parent, - bool is_input_anchor) { - COMPILE_ASSERT(!fsmap.empty(), "fusion anchor init slice not found") - // create anchor placeholder in IR - auto bld = builder::get_current_builder(); - auto pos = bld->push_anchor(); - - auto fusion_anchor = std::make_shared( - pos, fslice_map(fsmap), parent, is_input_anchor); - // append to fanchor list - fanchor_list_.emplace_back(fusion_anchor); - if (binded_mxp_) { - // if binded, directly append to mxp - binded_mxp_->append_fusion_anchor(fusion_anchor); - } -} - -void fusion_anchor_mgr_t::create_fusion_anchor(const expr &iter_var, - const slice_map &fsmap, - const stmt &dispatch_helper, - const fusion_anchor_ptr &parent, - bool is_input_anchor) { - // create anchor placeholder in IR - auto bld = builder::get_current_builder(); - auto pos = bld->push_anchor(); - - auto iter_anchor = std::make_shared( - iter_var, pos, fslice_map(fsmap), dispatch_helper, parent, - is_input_anchor); - // append to fanchor list - fanchor_list_.emplace_back(iter_anchor); - if (binded_mxp_) { - // if binded, directly append to mxp - binded_mxp_->append_fusion_anchor(iter_anchor); - } -} - -void fusion_anchor_mgr_t::create_fusion_anchor(int group_id, - const slice_map &fsmap, - const fusion_anchor_ptr &parent, - bool is_input_anchor) { - COMPILE_ASSERT(!fsmap.empty(), "grouped fusion anchor init slice not found") - COMPILE_ASSERT( - std::all_of(fsmap.begin(), fsmap.end(), - [](const std::pair &p) { - return p.second.size() == 1; - }), - "all init slice size of grouped fusion anchor should be 1") - // create anchor placeholder in IR - auto bld = builder::get_current_builder(); - auto pos = bld->push_anchor(); - // search grouped anchor map - auto res = grouped_id_map_.find(group_id); - // if group id already exist, update cached grouped anchor - if (res != grouped_id_map_.end()) { - auto &group_anchor = res->second; - auto &datamap = group_anchor->fsmap_.datamap_; - // update fsmap - for (auto &new_kv : fsmap) { - auto gt = new_kv.first; - auto slice = new_kv.second[0]; - auto cache_kv = datamap.find(gt); - COMPILE_ASSERT(cache_kv != datamap.end(), - "grouped anchor must have same graph tensor key") - // update cached slice range list - cache_kv->second.emplace_back(slice); - } - // update anchor position - group_anchor->anchor_position_->seq_.emplace_back(pos); - // update group size - group_anchor->group_size_++; - } else { - auto new_grouped_anchor = std::make_shared( - builder::make_stmts_unattached({pos}).checked_as(), - fslice_map(fsmap), parent, is_input_anchor); - // append to fanchor list - fanchor_list_.emplace_back(new_grouped_anchor); - if (binded_mxp_) { - // if binded, directly append to mxp - binded_mxp_->append_fusion_anchor(new_grouped_anchor); - } - // cache to grouped anchor map - grouped_id_map_.insert(std::make_pair(group_id, new_grouped_anchor)); - } -} - -void fusion_anchor_mgr_t::create_fusion_anchor(const graph_tensor_ptr >, - const slice_range &slice, - const fusion_anchor_ptr &parent, - bool is_input_anchor) { - create_fusion_anchor(slice_map{{gt.get(), slice_range_list{slice}}}, parent, - is_input_anchor); -} - -void fusion_anchor_mgr_t::create_fusion_anchor(const expr &tsr, - const slice_range &slice, - const fusion_anchor_ptr &parent, - bool is_input_anchor) { - // lookup graph tensor - auto iter = get_binded_mxp()->buf_alloc_.b2g_map_.find(tsr); - COMPILE_ASSERT(iter != get_binded_mxp()->buf_alloc_.b2g_map_.end(), - "No buffer to graph tensor mapping found for " << tsr) - auto gt = iter->second; - create_fusion_anchor(slice_map{{gt.get(), slice_range_list{slice}}}, parent, - is_input_anchor); -} - -void create_fusion_anchor(fusion_anchor_mgr_t *fusion, - const graph_tensor_ptr >, - const slice_range &slice) { - if (fusion) { - fusion->create_fusion_anchor(gt, slice); - } -} - -fusion_anchor_ptr try_convert_anchor(const context_ptr &ctx, - const fusion_anchor_ptr &fanchor) { - // Currently, it only converts common anchor. When all kinds of fusion - // anchor supported, this method should become one of member function - auto &anchor_type = *fanchor; - if (typeid(anchor_type) != typeid(fusion_anchor_t)) - return fanchor; - - auto &datamap = fanchor->fsmap_.datamap_; - if (datamap.size() != 1 || datamap.begin()->second.size() != 1) - return fanchor; - sc_dim floor, tail; - auto &fs_pair = *datamap.begin(); - auto base_gt = fs_pair.first; - // check attr to confirm all users of base gt agree to split - if (!std::all_of( - base_gt->uses_.begin(), base_gt->uses_.end(), - [](const std::pair - &user) { - return user.second->attrs_.get_or_else( - mixed_partition_hint::split_anchor_op, false); - })) - return fanchor; - // if last lanes is non-dividable, auto split it to grouped anchor - if (!innermost_slice_with_non_dividable_lanes( - ctx, fs_pair.second[0], base_gt->details_.dtype_, floor, tail)) - return fanchor; - - auto &pos = fanchor->anchor_position_; - // get parent pos - auto parent_pos = get_parent_node(pos).checked_as(); - // remove old anchor pos - std::vector::iterator anchor_iter = - std::find_if(parent_pos->seq_.begin(), parent_pos->seq_.end(), - [pos](stmt &s) { return s.ptr_same(pos); }); - COMPILE_ASSERT(anchor_iter != parent_pos->seq_.end(), - "Could not found anchor in current parent_pos stmts"); - parent_pos->seq_.erase(anchor_iter); - // create grouped anchor fsmap - auto &value = datamap.begin()->second[0]; - - auto floor_slice = slice_range{value.begin(), value.end() - 1}; - floor_slice.emplace_back( - std::make_pair(value.back().first, dim2unsigned(floor))); - auto tail_slice = slice_range{value.begin(), value.end() - 1}; - tail_slice.emplace_back(std::make_pair( - value.back().first + dim2unsigned(floor), dim2unsigned(tail))); - datamap.begin()->second = slice_range_list{floor_slice, tail_slice}; - // create grouped anchor pos - auto floor_pos = make_stmt(std::vector{}); - auto tail_pos = make_stmt(std::vector{}); - auto new_pos = - make_stmt(std::vector{floor_pos, tail_pos}); - add_parent_node(floor_pos, new_pos); - add_parent_node(tail_pos, new_pos); - parent_pos->seq_.emplace_back(new_pos); - add_parent_node(new_pos, parent_pos); - // remove current fanchor from content map of praent anchor - auto parent_anchor = fanchor->parent_; - auto root = parent_anchor; - while (root) { - root->content_number_map_.erase(fanchor.get()); - root = root->parent_; - } - // make gourped anchor - return std::make_shared( - new_pos, fslice_map(datamap), parent_anchor, fanchor->is_input_anchor()); -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/fusion_anchor.hpp b/legacy/core/src/compiler/ir/graph/fusion_anchor.hpp deleted file mode 100644 index 2f2c1469c..000000000 --- a/legacy/core/src/compiler/ir/graph/fusion_anchor.hpp +++ /dev/null @@ -1,483 +0,0 @@ -/******************************************************************************* - * Copyright 2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_FUSION_ANCHOR_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_FUSION_ANCHOR_HPP - -#include "graph.hpp" -#include "graph_map.hpp" -#include -#include -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -struct mixed_parti_t; - -using anchor_content_t = variant; -struct anchor_content_hasher { - size_t operator()(const anchor_content_t &v) const { - return std::hash()(v.cast()); - } -}; - -struct anchor_content_cmper { - bool operator()(const anchor_content_t &v, const anchor_content_t &v2) const { - return v.cast() == v2.cast(); - } -}; - -struct fusion_anchor_t : std::enable_shared_from_this { -private: - // control whether the fusion anchor is output anchor or input, default is - // output anchor. - bool is_input_anchor_; - mixed_parti_t *binded_mxp_ = nullptr; - -public: - friend struct mixed_parti_t; - stmts anchor_position_; - fslice_map fsmap_; - - // parent anchor - std::shared_ptr parent_; - - /* Updated when inferring */ - // blocked graph tensor set, the reason why not use empty gt for judgement - // is to distinguish non-visited gt and visited-but-failed gt - std::unordered_set blocked_gt_set_; - // borrowed fanchor map, must be the parent for current anchor - std::unordered_map> - borrowed_fanchor_map_; - - /* Updated when committing */ - // content-to-number mapping under current fusion anchor scope, includes - // either op and anchor - std::unordered_map - content_number_map_; - - fusion_anchor_t() = default; - fusion_anchor_t(stmts pos, const fslice_map &fsmap, - const std::shared_ptr &parent = nullptr, - bool is_input_anchor = false) - : is_input_anchor_(is_input_anchor), anchor_position_(std::move(pos)), - fsmap_(std::move(fsmap)) { - if (parent) { - attach_parent_anchor(parent, nullptr, true); - } - }; - - mixed_parti_t *const get_binded_mxp() const { return binded_mxp_; } - - bool defined() const { return anchor_position_.defined(); } - - bool is_input_anchor() const { return is_input_anchor_; }; - - // commit `stmt` to anchor and bind parent node to commited anchor - virtual void commit_stmt(const stmt &s) { - add_parent_node(s, anchor_position_); - anchor_position_->seq_.emplace_back(s); - } - - // commit `stmts` to anchor and bind parent node to commited anchor - virtual void commit_stmts(const stmts &ss) { - for (auto &s : ss->seq_) { - commit_stmt(s); - } - } - - // get all contents inside fusion anchor - std::vector get_contents() const { - std::vector ret; - for (auto &mp : content_number_map_) { - ret.emplace_back(mp.first); - } - return ret; - } - - // append list of contents by the given number id - void append_contents(std::vector contents, size_t num_id) { - for (auto &content : contents) { - content_number_map_.insert(std::make_pair(content, num_id)); - } - } - - // append content including either fusion anchor or sc op - void append_content(anchor_content_t content); - - // clear content - void clear_content(anchor_content_t content); - - void attach_parent_anchor( - const std::shared_ptr &parent, - const std::shared_ptr &repl_parent = nullptr, - bool inherit_offset = false); - - fusion_anchor_t *get_root() const { - auto root = this; - while (root->parent_) { - COMPILE_ASSERT(root != root->parent_.get(), - "Ring parent anchor relationship found"); - root = root->parent_.get(); - } - return const_cast(root); - } - - void merge(const std::shared_ptr &other); - - template bool isa() const { - static_assert(is_base_of_t::value, - "T is not a subclass of fusion_anchor."); - return dynamic_cast(this); - } - - template T *stc_cast() { return static_cast(this); } - - template T *dyn_cast() { return dynamic_cast(this); } - - virtual ~fusion_anchor_t() = default; - - // This function will find the nearest parent 'for_loop' node for fusion - // anchor - virtual stmt get_parent_loop() const; - - // This function will return parent body scope - stmts get_parent_scope() const { - auto loop = get_parent_loop(); - return loop.isa() - ? loop.static_as()->body_.checked_as() - : loop.checked_as(); - } - - /** - * What is parent relationship between anchor A and B - * { // anchor A - * for(){ - * // - * { // anchor B - * } - * } - * } - * */ - bool is_parent_for(const fusion_anchor_t *cur) const { - if (!cur) - return false; - if (is_input_anchor_ != cur->is_input_anchor_) - return false; - while (cur->parent_) { - cur = cur->parent_.get(); - if (cur == this) - return true; - } - return false; - } - - bool is_parent_for(const std::shared_ptr &cur) const { - return is_parent_for(cur.get()); - } - - /** - * What is sibling relationship between anchor A and B - * for(){ - * for(){ - * // - * { // anchor A - * } - * } - * { // anchor B - * } - * } - * */ - bool is_sibling_for(const fusion_anchor_t *other) const { - if (is_input_anchor_ != other->is_input_anchor_) - return false; - if (is_parent_for(other)) - return false; - auto this_loop = get_parent_loop(); - auto other_loop = other->get_parent_loop(); - // get parent loop - auto parent_loop = get_parent_node(other_loop); - while (parent_loop.defined()) { - if (this_loop.ptr_same(parent_loop)) { - return true; - } - parent_loop = get_parent_node(parent_loop); - } - return false; - } - - bool is_sibling_for(const std::shared_ptr &other) const { - return is_sibling_for(other.get()); - } - - /** - * What is cousin relationship between anchor A and B - * for(){ - * for(){ - * // - * { // anchor A - * } - * } - * for(){ - * // - * { // anchor B - * } - * } - * } - * */ - bool is_cousin_for(const fusion_anchor_t *cur) const { - if (is_input_anchor_ != cur->is_input_anchor_) - return false; - return !(this->is_parent_for(cur) || cur->is_parent_for(this) || - this->is_sibling_for(cur) || cur->is_sibling_for(this)) && - (cur->get_root() == this->get_root() || - get_common_parent_node(anchor_position_, cur->anchor_position_) - .defined()); - } - - bool is_cousin_for(const std::shared_ptr &cur) const { - return is_cousin_for(cur.get()); - } - - // check this anchor whether has view of given op - bool has_view_of(sc_op *op); - - // check inputs for op - bool check_input_for_op(const sc_op *op, - std::unordered_set &known_gt); - - // validate inferred slice range of inputs for op, excluded known gt - bool - validate_input_for_op(const sc_op *op, - const std::unordered_set &known_gt); - - // validate inferred slice range of outputs for op, usually for advanced - // fusion anchor - virtual bool validate_output_for_op(const sc_op *op) { - auto ths = this; - return !op->is_dynamic() || - std::all_of(op->get_outputs().begin(), op->get_outputs().end(), - [&ths](const graph_tensor_ptr &out) { - // dynamic case does not support multi-slice - // currently - return ths->fsmap_.get(out).size() == 1; - }); - } - - // forbid the given op in current anchor, set all gt(excluding known gt) to - // blocking gt set - void forbid_op(const sc_op *op, - const std::unordered_set &known_gt); - - // check the depedency for the given op in current anchor - bool check_dep_for_op(const sc_op *op); - - // query op committed into current fusion anchor whether is small workload - bool is_small_op_workload(const sc_op *op); -}; - -using fusion_anchor_ptr = std::shared_ptr; - -/** - * iter_anchor represents irregular slice range which is binded with loop iter. - * @param iter_: loop var - * @param iter_size: iteration size - * @param cached_iter_anchor_: real multi anchor used to commit code - * @param dispatch_helper_: the helper for IR dispatch - * @param iter_cnt_: built-in iter counter - * */ -struct iter_fusion_anchor_t : fusion_anchor_t { -private: - // iterated var - expr iter_; - size_t iter_size_; - std::vector cached_iter_anchor_; - stmt dispatch_helper_; - size_t iter_cnt_; - -public: - iter_fusion_anchor_t(expr iter_var, stmts pos, const fslice_map &fsmap, - stmt dispatch_helper = stmt(), - const fusion_anchor_ptr &parent = nullptr, - bool is_input_anchor = false) - : fusion_anchor_t(pos, fsmap, parent, is_input_anchor), - iter_(std::move(iter_var)), - dispatch_helper_(std::move(dispatch_helper)) { - COMPILE_ASSERT(!fsmap.empty(), - "iterated fusion anchor init slice not found") - iter_size_ = fsmap.datamap_.begin()->second.size(); - COMPILE_ASSERT( - std::all_of(fsmap.datamap_.begin(), fsmap.datamap_.end(), - [&](const std::pair &p) { - return p.second.size() == iter_size_; - }), - "all init slice size of iterated fusion anchor should be equal") - iter_cnt_ = 0; - cached_iter_anchor_.reserve(iter_size_); - } - - // iter anchor special inner-build `commit_` - void commit(const stmt &s); - - // override commit `stmt` to anchor - void commit_stmt(const stmt &s) override { commit(s); } - - // override commit `stmts` to anchor - void commit_stmts(const stmts &ss) override { commit(ss); } - - const size_t get_iter_size() const { return iter_size_; } - - bool validate_output_for_op(const sc_op *op) override { - auto ths = this; - return std::all_of(op->get_outputs().begin(), op->get_outputs().end(), - [&ths](const graph_tensor_ptr &out) { - return ths->get_iter_size() == - ths->fsmap_.get(out).size(); - }); - } -}; - -/** - * grouped_anchor contains grouped of basic anchors but under unified management - * and committed simultaneously. - * @param group_size_: total group amount - * @param group_pos_: real group anchor position used to commit code - * @param group_cnt_: built-in group counter - * */ -struct grouped_fusion_anchor_t : fusion_anchor_t { -private: - size_t group_size_; - size_t group_cnt_; - -public: - friend class fusion_anchor_mgr_t; - - grouped_fusion_anchor_t(stmts pos, const fslice_map &fsmap, - const fusion_anchor_ptr &parent = nullptr, - bool is_input_anchor = false) - : fusion_anchor_t(pos, fsmap, parent, is_input_anchor) { - COMPILE_ASSERT(!fsmap.empty(), "grouped fusion anchor init slice not found") - group_size_ = fsmap.datamap_.begin()->second.size(); - COMPILE_ASSERT( - std::all_of(fsmap.datamap_.begin(), fsmap.datamap_.end(), - [&](const std::pair &p) { - return p.second.size() == group_size_; - }), - "all init slice size of grouped fusion anchor should be equal") - COMPILE_ASSERT(pos->seq_.size() == group_size_, - "grouped anchor position " - << pos->seq_.size() << " should be equal to group size " - << group_size_) - group_cnt_ = 0; - } - - // group anchor special inner-build `commit_` - void commit(const stmt &s); - - // override commit `stmt` to anchor - void commit_stmt(const stmt &s) override { commit(s); } - - // override commit `stmts` to anchor - void commit_stmts(const stmts &ss) override { commit(ss); } - - // grouped anchor should override `get_parent_loop` method due to it owns - // multi anchor positions - stmt get_parent_loop() const override; - - const size_t get_group_size() const { return group_size_; } - - bool validate_output_for_op(const sc_op *op) override { - auto ths = this; - return std::all_of(op->get_outputs().begin(), op->get_outputs().end(), - [&ths](const graph_tensor_ptr &out) { - return ths->get_group_size() == - ths->fsmap_.get(out).size(); - }); - } -}; - -using slice_map = std::unordered_map; - -class fusion_anchor_mgr_t { -private: - // binded mixed partition - mixed_parti_t *binded_mxp_ = nullptr; - // maintain fusion anchor list - std::vector fanchor_list_; - // grouped id mapping - std::unordered_map> - grouped_id_map_; - -public: - fusion_anchor_mgr_t() = default; - fusion_anchor_mgr_t(mixed_parti_t *binded_mxp) : binded_mxp_(binded_mxp){}; - - void create_fusion_anchor(const graph_tensor_ptr >, - const slice_range &slice, - const fusion_anchor_ptr &parent = nullptr, - bool is_input_anchor = false); - - void create_fusion_anchor(const expr &tsr, const slice_range &slice, - const fusion_anchor_ptr &parent = nullptr, - bool is_input_anchor = false); - - // create basic fusion anchor - void create_fusion_anchor(const slice_map &fsmap, - const fusion_anchor_ptr &parent = nullptr, - bool is_input_anchor = false); - - // create iterated fusion anchor - void create_fusion_anchor(const expr &iter_var, const slice_map &fsmap, - const stmt &dispatch_helper = stmt(), - const fusion_anchor_ptr &parent = nullptr, - bool is_input_anchor = false); - - // create grouped fusion anchor - void create_fusion_anchor(int group_id, const slice_map &fsmap, - const fusion_anchor_ptr &parent = nullptr, - bool is_input_anchor = false); - - // get contained fusion anchor list - const std::vector &get_fusion_anchor() const { - return fanchor_list_; - } - - // get binded mxp - mixed_parti_t *const get_binded_mxp() const { return binded_mxp_; } -}; - -// create fusion anchor utils, usually used in template -void create_fusion_anchor(fusion_anchor_mgr_t *fusion, - const graph_tensor_ptr >, const slice_range &slice); -// split common anchor into grouped anchor -fusion_anchor_ptr try_convert_anchor(const context_ptr &ctx, - const fusion_anchor_ptr &fanchor); - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/graph/fusion_cost_model.cpp b/legacy/core/src/compiler/ir/graph/fusion_cost_model.cpp deleted file mode 100644 index dc4470f66..000000000 --- a/legacy/core/src/compiler/ir/graph/fusion_cost_model.cpp +++ /dev/null @@ -1,342 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "fusion_cost_model.hpp" -#include "fusible_op_utils.hpp" -#include "fusion_anchor.hpp" -#include "mixed_partition.hpp" -#include - -SC_MODULE(graph.fusion_cost_model); - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -static int op_num(mixed_parti_t *parti) { - int num = parti->ops.size(); - return num; -} - -static int loop_parallelism(mixed_parti_t *parti) { - auto outer_loops = parti->get_outer_loops(); - return evaluate_loop_parallel_balance(outer_loops); -} - -std::vector> create_default_evaluator() { - std::vector> inits; - inits.emplace_back(std::make_pair(0.1f, op_num)); - inits.emplace_back(std::make_pair(1.0f, loop_parallelism)); - return inits; -} - -fusion_cost_model_base_t::fusion_cost_model_base_t(mixed_parti_t *parti) - : binded_mxp_(parti), enable_(parti->ctx_->flags_.use_cost_model_) {} - -static_fusion_cost_model_t::static_fusion_cost_model_t(mixed_parti_t *parti) - : fusion_cost_model_base_t(parti), max_scores_(0), - evaluators_(create_default_evaluator()) {} - -float static_fusion_cost_model_t::evaluate() { - float new_scores = 0; - if (!enable_) - return new_scores; - for (auto &eval : evaluators_) { - new_scores += eval.first * eval.second(binded_mxp_); - } - if (new_scores > max_scores_) - max_scores_ = new_scores; - return new_scores; -} - -void static_fusion_cost_model_t::append_evaluator(float weight, - const cost_eval &eval) { - evaluators_.emplace_back(std::make_pair(weight, eval)); -} - -bool static_fusion_cost_model_t::make_decision_for_parti( - const mixed_parti_t *parti, size_t merged_loop_size, - parti_merge_kind merge_kind) { - // query if turn on - if (!enable_) - return true; - /* loop_parallelism */ - auto ths_outer_loop = binded_mxp_->get_outer_loops(); - auto other_outer_loop = parti->get_outer_loops(); - COMPILE_ASSERT(!ths_outer_loop.empty() && !other_outer_loop.empty(), - "Could not merge empty loop") - COMPILE_ASSERT((merged_loop_size <= ths_outer_loop.size()) && - (merged_loop_size <= other_outer_loop.size()), - "merge loop size should less than both loop") - if (merge_kind == parti_merge_kind::horizontal) { - return evaluate_loop_parallel_balance(ths_outer_loop) != 1.0f && - evaluate_loop_parallel_balance(other_outer_loop) != 1.0f; - } - /* for verticall merge*/ - COMPILE_ASSERT(merge_kind == parti_merge_kind::vertical, - "No cost metric found for parallel merge") - // in avoid of loss for loop optimize opportunity - if (binded_mxp_->can_optimize_outer_loop(true) ^ - parti->can_optimize_outer_loop(true)) { - return false; - } - - // check loop parallelism - auto ths_loop_parallelism = evaluate_loop_parallel_balance(ths_outer_loop); - auto other_loop_parallelism = - evaluate_loop_parallel_balance(other_outer_loop); - auto merged_outer_loop = std::vector{ - ths_outer_loop.begin(), ths_outer_loop.begin() + merged_loop_size}; - auto merged_loop_parallelism = - evaluate_loop_parallel_balance(merged_outer_loop); - if (merged_loop_parallelism < ths_loop_parallelism || - merged_loop_parallelism < other_loop_parallelism) { - // evalute workload size - if (!evaluate_loop_parallel_balance(merged_outer_loop, true) || - ((merged_loop_parallelism < ths_loop_parallelism) && - !binded_mxp_->is_small_workload()) || - ((merged_loop_parallelism < other_loop_parallelism) && - !parti->is_small_workload())) { - SC_MODULE_INFO << "rejects to merge two " - "partition: " - << binded_mxp_->func_->name_ << " and " - << parti->func_->name_ - << " from perspective of loop parallelism"; - return false; - } - } - - /* cache efficiency */ - // skip standalone parti - if (binded_mxp_->ops.size() == 1) - return true; - // get real merged trace and merged inplace map - auto merged_mem_info = - merge_real_mem_info(binded_mxp_->buf_alloc_, parti->buf_alloc_); - // get real buffer usage - auto buffer_usage = get_buffer_usage(binded_mxp_->ctx_, merged_mem_info.first, - merged_mem_info.second); - // get threshold - auto threshold = binded_mxp_->ctx_->machine_.cpu_flags_.getDCacheSize(2); - - // check cache efficiency - if (buffer_usage > threshold) { - SC_MODULE_INFO << "rejects to merge two " - "partition: " - << binded_mxp_->func_->name_ << " and " - << parti->func_->name_ - << " from perspective of cache efficiency"; - return false; - } else { - return true; - } -} - -// Some kinds of op may perform more parallism when fusion break, like tunable -// op and broadcast op, which needs double check -static bool need_double_check_standalone_parallel(const sc_op *op, - const mixed_parti_t *mxp) { - if (op->isa()) - return true; - if (auto broadcast_op = op->dyn_cast()) { - auto non_bc_index = broadcast_op->get_non_broadcast_input_index(true); - // If no broadcast semantic found - if (non_bc_index.size() == op->get_inputs().size()) - return false; - // If any non-broadcast input of op is included in mxp with tunable op, - // rely on previous decision made by cost model - if (mxp->contain_tunable_op() && - std::any_of(non_bc_index.begin(), non_bc_index.end(), - [&op, &mxp](const int &idx) { - return mxp->contains( - op->get_inputs()[idx]->producer_owner_); - })) - return false; - if (mxp->can_optimize_outer_loop(true)) - return false; - return true; - } - return false; -} - -bool static_fusion_cost_model_t::make_decision_for_op( - const sc_op *op, const fusion_anchor_ptr &fanchor) { - // query if turn on - if (!enable_) - return true; - /** Auto Skip List: - * 1. empty partition - * 2. nested parallel template - * 3. singel op lowering - * */ - if (binded_mxp_->empty() || binded_mxp_->contain_nested_parallel_for() || - is_single_op_graph(op->get_owner_graph())) - return true; - - auto orig_loop_parallelism = - evaluate_loop_parallel_balance(binded_mxp_->get_outer_loops()); - auto fanchor_loop_parallelism = - evaluate_loop_parallel_balance(binded_mxp_->get_outer_loops(fanchor)); - - bool ret = (!binded_mxp_->contain_tunable_op() && !op->isa()) || - (fanchor_loop_parallelism >= orig_loop_parallelism); - - // double check parallelism of standalone op - if (need_double_check_standalone_parallel(op, binded_mxp_)) { - SC_MODULE_INFO << "double check standalone parallellism for " - << op->op_name_ << op->logical_op_id_; - mixed_parti_t op_parti(binded_mxp_->ctx_, std::const_pointer_cast( - op->shared_from_this())); - float standalone_parallel = - evaluate_loop_parallel_balance(op_parti.get_outer_loops()); - // if original result of partition can not meet requirement - if (ret && - !evaluate_loop_parallel_balance(binded_mxp_->get_outer_loops(), true)) { - // if new parti created by op owning more loop parallelism, reject - // to fuse it - if (standalone_parallel > fanchor_loop_parallelism && - !fanchor->is_small_op_workload(op)) { - ret = false; - } - } else if (!ret && evaluate_loop_parallel_balance( - binded_mxp_->get_outer_loops(fanchor), true)) { - // if new parti created by op can not meet loop parallelism - // requirement, suggest to fuse it anyway. - if (standalone_parallel != 1.f) { - ret = true; - } - } - } - if (!ret) { - SC_MODULE_INFO << "rejects to commit op: " << op->op_name_ - << op->logical_op_id_ - << " into current fusion anchor from " - "perspective of parallellism"; - } - return ret; -} - -dynamic_fusion_cost_model_t::dynamic_fusion_cost_model_t( - mixed_parti_t *parti, dynamic_fusion_policy_t policy) - : fusion_cost_model_base_t(parti), cond_(false), policy_(policy) {} - -bool dynamic_fusion_cost_model_t::make_decision_for_parti( - const mixed_parti_t *parti, size_t merged_loop_size, - parti_merge_kind merge_kind) { - // query if turn on - if (!enable_ || policy_ == dynamic_fusion_policy_t::max_fusion) - return true; - /* loop_parallelism */ - auto ths_outer_loops = binded_mxp_->get_outer_loops(); - auto other_outer_loops = parti->get_outer_loops(); - COMPILE_ASSERT(!ths_outer_loops.empty() && !other_outer_loops.empty(), - "Could not merge empty loop") - COMPILE_ASSERT((merged_loop_size <= ths_outer_loops.size()) && - (merged_loop_size <= other_outer_loops.size()), - "merge loop size should less than both loop"); - expr res_cond, dummy_cond; - if (merge_kind == parti_merge_kind::horizontal) { - float ths_parallelism = - evaluate_loop_parallel_balance({ths_outer_loops[0]}, res_cond); - float other_parallelism = - evaluate_loop_parallel_balance({other_outer_loops[0]}, dummy_cond); - bool ret = ths_parallelism != 1.0f && other_parallelism != 1.0f; - if (!ret) { - cond_ = cond_ || !(res_cond && dummy_cond); - } - return ret; - } - /* for verticall merge*/ - COMPILE_ASSERT(merge_kind == parti_merge_kind::vertical, - "No cost metric found for parallel merge") - // in avoid of loss for loop optimize opportunity - if (binded_mxp_->can_optimize_outer_loop(true) ^ - parti->can_optimize_outer_loop(true)) { - return false; - } - - // check loop parallelism - float ths_parallelism = - evaluate_loop_parallel_balance(ths_outer_loops, dummy_cond); - float other_parallelism = - evaluate_loop_parallel_balance(other_outer_loops, dummy_cond); - auto merged_outer_loop = std::vector{ - ths_outer_loops.begin(), ths_outer_loops.begin() + merged_loop_size}; - float merged_parallelism = - evaluate_loop_parallel_balance(merged_outer_loop, res_cond); - if (merged_parallelism < ths_parallelism || - merged_parallelism < other_parallelism) { - SC_MODULE_INFO << "rejects to merge two " - "partition: " - << binded_mxp_->func_->name_ << " and " - << parti->func_->name_ - << " from perspective of loop parallelism"; - cond_ = cond_ || res_cond; - return false; - } - // don't set cond_ if accept the parti. - /* find how to describe cache efficiency */ - return true; -} - -bool dynamic_fusion_cost_model_t::make_decision_for_op( - const sc_op *op, const fusion_anchor_ptr &fanchor) { - // query if turn on - if (!enable_ || policy_ == dynamic_fusion_policy_t::max_fusion) - return true; - // auto skip - if (!binded_mxp_->contain_tunable_op() && !op->isa()) - return true; - expr res_cond, thr_cond, dummy_cond; - auto ths_outer_loops = binded_mxp_->get_outer_loops(); - auto other_outer_loops = binded_mxp_->get_outer_loops(fanchor); - - auto ths_parallelism = - evaluate_loop_parallel_balance(ths_outer_loops, dummy_cond); - auto other_parallelism = - evaluate_loop_parallel_balance(other_outer_loops, res_cond); - bool ret = ths_parallelism <= other_parallelism; - if (op->isa() && - evaluate_loop_parallel_balance(binded_mxp_->get_outer_loops(), thr_cond, - true) == 0.f) { - SC_MODULE_INFO << "double check standalone parallellism for " - << op->op_name_ << op->logical_op_id_; - mixed_parti_t tunable_parti( - binded_mxp_->ctx_, - std::const_pointer_cast(op->shared_from_this())); - if (evaluate_loop_parallel_balance(tunable_parti.get_outer_loops(), - dummy_cond) > other_parallelism) { - ret = false; - cond_ = cond_ || thr_cond; - } - } - if (!ret) { - SC_MODULE_INFO << "rejects to commit op: " << op->op_name_ - << op->logical_op_id_ - << " into current fusion anchor from " - "perspective of parallellism"; - cond_ = cond_ || res_cond; - return false; - } - // don't set cond_ when accept the op - return true; -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/fusion_cost_model.hpp b/legacy/core/src/compiler/ir/graph/fusion_cost_model.hpp deleted file mode 100644 index 7979df59c..000000000 --- a/legacy/core/src/compiler/ir/graph/fusion_cost_model.hpp +++ /dev/null @@ -1,118 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_FUSION_COST_MODEL_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_FUSION_COST_MODEL_HPP -#include -#include -#include -#include -#include -#include -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -class sc_op; -struct mixed_parti_t; -struct fusion_anchor_t; -enum class parti_merge_kind; -enum class dynamic_fusion_policy_t; - -using cost_eval = std::function; - -/** - * fusion cost model has two main function: - * 1. make final decision for fusing or not in consider of loop parallelism - * or cache locality. - * 2. evaluate fusion partition and throw warning when prediction is under - * expectation. - * */ -struct fusion_cost_model_base_t { -protected: - mixed_parti_t *binded_mxp_; - bool enable_; - -public: - fusion_cost_model_base_t(mixed_parti_t *parti); - virtual ~fusion_cost_model_base_t() {} - // disable cost model - void disable() { enable_ = false; } - // judge if it is enabled - bool is_enabled() const { return enable_; } - // evaluate current mixed partition by several evaluator - virtual float evaluate() = 0; - // make decision for partition merge - virtual bool make_decision_for_parti(const mixed_parti_t *parti, - size_t merge_loop_size, - parti_merge_kind merge_kind) = 0; - // make decision for op and fusion anchor - virtual bool - make_decision_for_op(const sc_op *op, - const std::shared_ptr &fanchor) = 0; - virtual expr get_fusion_policy_condition() const { return false; } -}; - -struct static_fusion_cost_model_t : public fusion_cost_model_base_t { -private: - float max_scores_; // cache the top scores - std::vector> evaluators_; - -public: - static_fusion_cost_model_t(mixed_parti_t *parti); - // evaluate current mixed partition by several evaluator - float evaluate() override; - // append new defined evaluator - void append_evaluator(float weight, const cost_eval &eval); - // make decision for partition merge - bool make_decision_for_parti(const mixed_parti_t *parti, - size_t merge_loop_size, - parti_merge_kind merge_kind) override; - // make decision for op and fusion anchor - bool make_decision_for_op( - const sc_op *op, - const std::shared_ptr &fanchor) override; -}; - -struct dynamic_fusion_cost_model_t : public fusion_cost_model_base_t { -private: - expr cond_; - dynamic_fusion_policy_t policy_; - -public: - dynamic_fusion_cost_model_t(mixed_parti_t *parti, - dynamic_fusion_policy_t policy); - // evaluate current mixed partition by several evaluator - float evaluate() override { return 0.f; } - expr get_fusion_policy_condition() const override { return cond_; } - // make decision for partition merge - bool make_decision_for_parti(const mixed_parti_t *parti, - size_t merge_loop_size, - parti_merge_kind merge_kind) override; - // make decision for op and fusion anchor - bool make_decision_for_op( - const sc_op *op, - const std::shared_ptr &fanchor) override; -}; - -using fusion_cost_model_ptr = std::shared_ptr; - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl -#endif diff --git a/legacy/core/src/compiler/ir/graph/fusion_data.hpp b/legacy/core/src/compiler/ir/graph/fusion_data.hpp deleted file mode 100644 index 082abf0e5..000000000 --- a/legacy/core/src/compiler/ir/graph/fusion_data.hpp +++ /dev/null @@ -1,94 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_FUSION_DATA_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_FUSION_DATA_HPP - -#include "graph_map.hpp" -#include "tensor_slice.hpp" -#include -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -using slice_range_list = std::vector; -using slice_range_map = std::unordered_map; - -enum class infer_status_code : int { - OK = 0, // Successful - RETRY, // Need retry another anchor - FAIL, // Could not infer - UNKNOWN, // Unknown - END, -}; - -expr do_cast_and_fold(const expr &); - -inline std::vector get_slice_idx(const slice_range &range) { - std::vector ret; - for (auto &r : range) { - ret.emplace_back(do_cast_and_fold(r.first)); - } - return ret; -} - -inline std::vector get_slice_shape(const slice_range &range) { - std::vector ret; - for (auto &r : range) { - ret.emplace_back(do_cast_and_fold(r.second)); - } - return ret; -} - -inline slice_range gen_slice_by_dims(const sc_dims &dims) { - slice_range ret; - for (auto &r : dims) { - ret.emplace_back(std::make_pair(expr(0), dim2unsigned(r))); - } - return ret; -} - -inline slice_range gen_slice_by_dims_expr(const std::vector &dims) { - slice_range ret; - for (auto &r : dims) { - ret.emplace_back(std::make_pair(dim2unsigned(0), r)); - } - return ret; -} - -bool is_reshaped_tensor(const expr &tsr); - -expr transform_tsr2stsr_with_range(const expr &tsr, const slice_range &range); - -expr transform_tsl2stsr(const tensor_slice &tsl); - -expr transform_tsr2tptr_with_range(const expr &tsr, const slice_range &range); - -expr transform_tptr2stsr(const expr &tptr); - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl -#endif diff --git a/legacy/core/src/compiler/ir/graph/graph.cpp b/legacy/core/src/compiler/ir/graph/graph.cpp deleted file mode 100644 index 2472232b0..000000000 --- a/legacy/core/src/compiler/ir/graph/graph.cpp +++ /dev/null @@ -1,1054 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include -#include -#include - -#include "dynamic_dispatch_key.hpp" -#include "dynamic_lower_info.hpp" -#include "visitor.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -SC_MODULE(sc_graph); - -extern void __dummy_init(); - -template valT >_map_t::get(graph_tensor *v) { - auto itr = datamap_.find(v); - if (itr != datamap_.end()) { - return itr->second; - } - auto &ret = datamap_[v]; - return ret; -} - -template valT >_map_t::get(const graph_tensor_ptr &v) { - return get(v.get()); -} - -template bool gt_map_t::haskey(graph_tensor *v) const { - return datamap_.find(v) != datamap_.end(); -} - -template -bool gt_map_t::haskey(const graph_tensor_ptr &v) const { - return haskey(v.get()); -} - -template struct gt_map_t; -template struct gt_map_t; -template struct gt_map_t>; -template struct gt_map_t; -template struct gt_map_t; -template struct gt_map_t; - -sc_op_ptr -op_traits::auto_copyable_t::copy(const std::vector &ins, - const std::vector &outs, - sc_graph_t &mgr) { - auto ths = dynamic_cast(this); - auto ret = mgr.make(ths->op_name_, ins, outs, ths->attrs_); - ret->copy_dispatch_key_set_from_op(ths->shared_from_this()); - return ret; -} - -std::vector -copy_logical_tsr(const std::vector &v) { - std::vector ret; - ret.reserve(v.size()); - for (auto &t : v) { - ret.emplace_back(std::make_shared(nullptr, t->details_)); - } - return ret; -} - -static std::vector *get_tensor_dims(const expr &tsr) { - if (tsr.isa()) { - auto t = tsr.static_as(); - return &t->dims_; - } else { - COMPILE_ASSERT( - tsr.isa(), - "tensor_slice only accepts a tensor or tensorptr, got: " << tsr); - return &tsr.static_as()->shape_; - } -} - -const std::vector &tensor_slice::get_base_dims() const { - return *get_tensor_dims(tptr_); -} - -sc_data_type_t tensor_slice::get_base_dtype() const { - return get_real_tensor()->elem_dtype_; -} - -tensor tensor_slice::get_real_tensor() const { - auto &base = tptr_->base_; - COMPILE_ASSERT(base.isa(), - "tensor_ptr base should be indexing, but got: " << base); - auto tsr = base->ptr_; - while (!tsr.isa()) { - COMPILE_ASSERT( - tsr.isa(), - "tensor_slice only accepts a tensor or tensorptr, got: " << tsr); - auto base = tsr.static_as()->base_; - COMPILE_ASSERT(base.isa(), - "tensor_ptr base should be indexing, but got: " << base); - tsr = base.checked_as()->ptr_; - } - return tsr.static_as(); -} - -slice_range tensor_slice::get_ranges() const { - COMPILE_ASSERT(get_shape().size() == tptr_->base_->idx_.size(), - "Unmatched shape and idx found"); - auto shape = get_shape(); - auto offset = tptr_->base_->idx_; - slice_range ranges; - for (int64_t i = 0; i < nslice_dims(); i++) { - ranges.emplace_back(std::make_pair(offset[i], shape[i])); - } - return ranges; -} - -bool tensor_slice::full_on_axis(const std::vector &axis) const { - auto &dims = get_base_dims(); - auto &idx = tptr_->base_->idx_; - auto &shape = get_shape(); - for (auto &ax : axis) { - if (!idx[ax].isa() || !shape[ax].isa()) { - return false; - } - if (get_const_as_int(idx[ax].checked_as()) != 0 || - get_const_as_int(shape[ax].checked_as()) != - get_const_as_int(dims[ax].checked_as())) { - return false; - } - } - return true; -} - -bool tensor_slice::is_full() const { - auto &dims = get_base_dims(); - std::vector total_axis; - total_axis.reserve(static_cast(dims.size())); - for (int i = 0; i < static_cast(dims.size()); i++) { - total_axis.emplace_back(i); - } - return full_on_axis(total_axis); -} - -bool tensor_slice::is_const() const { - return std::all_of(shape_.begin(), shape_.end(), [](const expr &e) { - return do_cast_and_fold(e).isa(); - }); -} - -tensor_slice::tensor_slice(const expr &tsr) { - if (tsr.isa()) { - auto t = tsr.static_as(); - tptr_ = builder::tensor_ptr(tsr, std::vector(t->dims_.size(), 0), {}, - true) - .static_as(); - shape_ = t->dims_; - } else { - COMPILE_ASSERT( - tsr.isa(), - "tensor_slice only accepts a tensor or tensorptr, got: " << tsr); - tptr_ = tsr.static_as(); - shape_ = tptr_->shape_; - } -} - -tensor_slice::tensor_slice(const expr &tsr, slice_range &&range) { - auto dims = get_tensor_dims(tsr); - if (dims->size() != range.size()) - COMPILE_ASSERT( - dims->size() == 1 && - get_const_as_int((*dims)[0].checked_as()) == 1, - "Unmatched range found. Tensor: " - << (tsr.isa() ? tsr.static_as()->name_ : "") - << " have dims: " << utils::print_vector(*dims) - << " but got slice range: " << utils::print_pair_vector(range)); - tptr_ = - builder::tensor_ptr(tsr, - dims->size() != range.size() ? std::vector{0} - : get_slice_idx(range), - {}, true) - .static_as(); - shape_ = get_slice_shape(range); -} - -graph_tensor::graph_tensor(sc_op *owner) : producer_owner_(owner) {} -graph_tensor::graph_tensor(sc_op *owner, const logical_tensor_t <) - : details_(lt), producer_owner_(owner) {} - -graph_tensor::graph_tensor(sc_op *owner, const sc_data_format_t &format, - const sc_dims &plain_shape, - const sc_data_type_t &type, const sc_dims &stride) - : details_(format, plain_shape, type, stride), producer_owner_(owner) {} - -const sc_dims &logical_tensor_t::get_blocking_dims() const { return dims_; } - -std::vector -logical_tensor_t::get_blocking_dims_expr(sc_graph_t &g) const { - return get_blocking_shapes_expr(g, plain_dims_, format_); -} - -static bool check_stride_validity(const bool is_dynamic, const sc_dims &dims, - const sc_dims &strides) { - return strides.size() == dims.size() && - (is_dynamic || std::is_sorted(strides.begin(), strides.end(), - std::greater())); -} - -void logical_tensor_t::internal_update() { - dims_ = sc_data_format_t::get_blocking_shapes(plain_dims_, format_); - if (strides_.empty()) { - strides_ = compute_dense_stride(dims_); - } else { - COMPILE_ASSERT(check_stride_validity(is_dynamic(), dims_, strides_), - "Specified strides value invalid or not consistent with " - "real(blocking) dims.") - } -} - -bool logical_tensor_t::is_dynamic() const { - return !std::all_of(plain_dims_.begin(), plain_dims_.end(), - [](const sc_dim &dim) { return !is_dynamic_dim(dim); }); -} - -// sets the logical dims in plain format -void logical_tensor_t::set_plain_dims(const sc_dims &plain_dims) { - COMPILE_ASSERT(is_dynamic() || is_dense(), - "Forbid update format on a strided tensor."); - strides_.clear(); - plain_dims_ = plain_dims; - internal_update(); -} - -// TODO(xxx): this logic maybe not correct, just distinguish with set_plain_dims -void logical_tensor_t::set_blocking_dims(const sc_dims &blocking_dims) { - // assert(format_.format_code_ == format_kinds::any); - COMPILE_ASSERT(is_dense(), "Forbid set blocking dims on a strided tensor."); - format_.format_code_ = format_kinds::any; - plain_dims_ = blocking_dims; - dims_ = blocking_dims; - strides_ = compute_dense_stride(dims_); -} - -void logical_tensor_t::set_format(const sc_data_format_t &newv) { - COMPILE_ASSERT(is_dense(), "Forbid set format on a strided tensor."); - strides_.clear(); - format_ = newv; - internal_update(); -} - -void logical_tensor_t::set_strides(const sc_dims &strides) { - COMPILE_ASSERT(check_stride_validity(is_dynamic(), dims_, strides), - "Specified strides value invalid or not consistent with " - "real(blocking) dims.") - strides_ = strides; -} - -void logical_tensor_t::set_format_and_stride(const sc_data_format_t &newv, - const sc_dims &strides) { - format_ = newv; - strides_ = strides; - internal_update(); -} - -void logical_tensor_t::add_format_candidate(const sc_data_format_t &newv) { - format_candidates_.insert(newv); -} - -void logical_tensor_t::remove_format_candidate(const sc_data_format_t &v) { - auto it = format_candidates_.find(v); - if (it != format_candidates_.end()) { - format_candidates_.erase(it); - } -} - -void logical_tensor_t::set_format_candidates( - const std::vector &newf) { - format_candidates_.insert(newf.begin(), newf.end()); - if (format_candidates_.size() == 1) { - format_ = *format_candidates_.begin(); - } - internal_update(); -} - -size_t logical_tensor_t::get_blocking_byte_size() const { - COMPILE_ASSERT(!is_dynamic(), "blocking byte size should be static shape."); - size_t sz = utils::get_sizeof_type(dtype_); - for (auto z : get_blocking_dims()) { - sz *= z; - } - return sz; -} - -bool logical_tensor_t::is_dense() { - if (strides_.empty()) { - return true; - } - if (is_dynamic()) { - return true; - } - if (std::any_of(plain_dims_.begin(), plain_dims_.end(), - [](const sc_dim &d) { return d == 0; })) { - return true; - } - assert(strides_.size() == dims_.size()); - if (strides_.back() != 1) { - return false; - } - for (int i = dims_.size() - 2; i >= 0; --i) { - if (strides_[i] != strides_[i + 1] * dims_[i + 1]) { - return false; - } - } - return true; -} - -std::vector logical_tensor_t::get_strides_expr(sc_graph_t &g) const { - return is_dynamic() ? dims_to_dense_stride(get_blocking_dims_expr(g)) - : g.dims_to_expr(get_strides()); -} - -sc_dims logical_tensor_t::compute_dense_stride(const sc_dims &dims) { - sc_dims strides(dims.size(), 1); - for (int i = dims.size() - 2; i >= 0; --i) { - if (dims[i + 1] == 0) { - strides[i] = strides[i + 1]; - } else { - strides[i] = dims[i + 1] * strides[i + 1]; - } - } - return strides; -} - -void logical_tensor_t::to_string(std::ostream &os) { - os << '[' << dtype_ << ' ' << utils::print_vector(get_blocking_dims()) - << " @ " << format_ << ']'; -} - -size_t logical_tensor_t::hash() const { - size_t seed = 0; - hash_combine(seed, static_cast(dtype_)); - hash_combine(seed, plain_dims_); - hash_combine(seed, dims_); - hash_combine(seed, strides_); - hash_combine(seed, format_); - return seed; -} - -void graph_tensor::attach_use(sc_op_ptr op, int index) { - uses_.emplace_back(std::make_pair(index, std::move(op))); -} - -void graph_tensor::detach_use(const sc_op_ptr &op) { - for (auto itr = uses_.begin(); itr != uses_.end();) { - if (itr->second == op) { - itr = uses_.erase(itr); - } else { - ++itr; - } - } -} - -void graph_tensor::detach_use(const sc_op_ptr &op, int input_idx) { - for (auto itr = uses_.begin(); itr != uses_.end();) { - if (itr->first == input_idx && itr->second == op) { - itr = uses_.erase(itr); - } else { - ++itr; - } - } -} - -void graph_tensor::replace_with(const graph_tensor_ptr &v) { - while (!uses_.empty()) { - auto node = uses_.front(); - node.second->replace_input(node.first, v); - } -} - -graph_tensor_ptr graph_tensor::copy() { - return std::make_shared(producer_owner_, details_); -} - -void sc_op::replace_input(size_t index, const graph_tensor_ptr &new_input, - const bool skip_shape_check) { - if (!skip_shape_check) { - assert(index < info_.inputs_.size()); - assert(new_input->details_.is_dynamic() || - get_dims_product(info_.inputs_[index]->details_.get_plain_dims()) == - get_dims_product(new_input->details_.get_plain_dims())); - } - info_.inputs_[index]->detach_use(shared_from_this(), index); - info_.inputs_[index] = new_input; - new_input->attach_use(shared_from_this(), index); -} - -void sc_op::replace_uses_with_and_remove(const sc_op_ptr &replacer) { - assert(info_.outputs_.size() == replacer->info_.outputs_.size()); - for (unsigned i = 0; i < info_.outputs_.size(); i++) { - auto &ths_out = info_.outputs_[i]; - auto &replace_out = replacer->info_.outputs_[i]; - ths_out->replace_with(replace_out); - } - remove(); -} - -bool sc_op::has_graph_output() const { - for (const auto &output : info_.outputs_) { - for (const auto &use_node : output->uses_) { - if (use_node.second->isa()) { - return true; - } - } - } - return false; -} - -bool sc_op::is_single_output_single_use() { - return info_.outputs_.size() == 1 && info_.outputs_[0]->uses_.size() == 1; -} - -bool sc_op::is_dynamic() const { - return !std::all_of(info_.inputs_.begin(), info_.inputs_.end(), - [](const graph_tensor_ptr &inp) { - return !inp->details_.is_dynamic(); - }) || - !std::all_of(info_.outputs_.begin(), info_.outputs_.end(), - [](const graph_tensor_ptr &out) { - return !out->details_.is_dynamic(); - }); -} - -const dispatch_set_ptr &sc_op::get_dispatch_key_set() const { - assert(info_.dispatch_key_set_); - return info_.dispatch_key_set_; -} - -dispatch_set_ptr &sc_op::get_dispatch_key_set() { - if (!info_.dispatch_key_set_) { - info_.dispatch_key_set_ = std::make_shared(); - } - return info_.dispatch_key_set_; -} - -dispatch_set_ptr sc_op::get_internal_dispatch_key_set(const context_ptr &ctx) { - throw std::runtime_error( - "Internal dispatch key set should be implemented by concrete op."); -} - -void sc_op::copy_dispatch_key_set_from_op(const sc_op_ptr &other) { - if (other->info_.dispatch_key_set_) { - info_.dispatch_key_set_ = other->info_.dispatch_key_set_->copy(); - } -} - -void sc_op::remove() { - for (auto &in : info_.inputs_) { - in->detach_use(shared_from_this()); - } - info_.inputs_.clear(); - info_.outputs_.clear(); - attrs_.as_map().clear(); - is_removed_ = true; -} - -// template op and fusible op common constructor -sc_op::sc_op(const std::string &op_name, - const std::vector &producer_lt, - const std::vector &consumer_lt, - const any_map_t &attrs) - : attrs_(attrs), op_name_(op_name) { - info_.inputs_ = producer_lt; - info_.outputs_ = consumer_lt; - for (auto &op : info_.outputs_) { - op->producer_owner_ = this; - } -} - -void sc_op::format_to_dense_format_stride_pair( - const std::vector> &in_formats, - const std::vector> &out_formats, - std::vector> &supported_ins, - std::vector> &supported_outs) { - supported_ins.resize(in_formats.size()); - for (size_t i = 0; i < in_formats.size(); ++i) { - for (auto fmt : in_formats[i]) { - logical_tensor_t dense_lt(fmt, - info_.inputs_[i]->details_.get_plain_dims(), - info_.inputs_[i]->details_.dtype_); - supported_ins[i].emplace_back( - std::make_pair(fmt, dense_lt.get_strides())); - } - } - supported_outs.resize(out_formats.size()); - for (size_t i = 0; i < out_formats.size(); ++i) { - for (auto fmt : out_formats[i]) { - logical_tensor_t dense_lt(fmt, - info_.outputs_[i]->details_.get_plain_dims(), - info_.outputs_[i]->details_.dtype_); - supported_outs[i].emplace_back(fmt, dense_lt.get_strides()); - } - } -} - -sc_graph_t::sc_graph_t(sc_graph_t &&other) - : ops_(std::move(other.ops_)), attrs_(std::move(other.attrs_)), - dyn_info_(std::move(other.dyn_info_)) { - for (auto &op : ops_) { - op->set_owner_graph(this); - } -} - -sc_graph_t &sc_graph_t::operator=(sc_graph_t &&other) { - ops_ = std::move(other.ops_); - attrs_ = std::move(other.attrs_); - dyn_info_ = std::move(other.dyn_info_); - for (auto &op : ops_) { - op->set_owner_graph(this); - } - return *this; -} - -size_t sc_graph_t::hash_contents( - const std::function &filter) - const { - size_t seed = 0; - hash_combine(seed, attrs_.get_or_else("fpmath_mode", 0)); - op_visitor_t vis = op_visitor_t::bfs_topology_sort(this->ops_.size()); - vis.visit_graph(*this, [&](op_visitor_t *vis, const sc_op_ptr &op) { - hash_combine(seed, op->hash_contents(filter)); - }); - return seed; -} - -void sc_graph_t::reset_op_ids() { - for (auto it = ops_.begin(); it != ops_.end();) { - if ((*it)->is_removed_ || - ((*it)->get_inputs().empty() && (*it)->get_outputs().empty())) { - it = ops_.erase(it); - } else { - ++it; - } - } - for (size_t i = 0; i < ops_.size(); ++i) { - ops_[i]->logical_op_id_ = i; - } -} - -void sc_graph_t::resort_op_ids( - const std::unordered_map &op_id_map) { - std::sort(ops_.begin(), ops_.end(), - [&op_id_map](const sc_op_ptr &A, const sc_op_ptr &B) { - auto A_iter = op_id_map.find(A), B_iter = op_id_map.find(B); - COMPILE_ASSERT(A_iter != op_id_map.end() && - B_iter != op_id_map.end(), - "op id map is not enough, could not do sorting") - return A_iter->second < B_iter->second; - }); - for (size_t i = 0; i < ops_.size(); ++i) { - ops_[i]->logical_op_id_ = i; - } -} - -sc_dim sc_graph_t::get_next_dynamic_placeholder() { - if (!dyn_info_) { - dyn_info_ = std::make_shared(); - } - COMPILE_ASSERT(std::numeric_limits::min() < - dyn_info_->cur_dynamic_placeholder_, - "Dynamic shapes are too many to mark!"); - return dyn_info_->cur_dynamic_placeholder_--; -} - -expr sc_graph_t::dim_to_expr(const sc_dim &v) { - const std::string dynamic_prefix = "dynamic_var_"; - if (is_dynamic_dim(v)) { - if (!dyn_info_) { - dyn_info_ = std::make_shared(); - } - assert(v != dimensions::dynamic_any); - auto &m = dyn_info_->dim2expr_map_; - auto it = m.find(v); - if (it == m.end()) { - auto dyn_var = make_expr(datatypes::index, - dynamic_prefix + std::to_string(-v)); - dyn_var->attr().set(attr_key::const_attr, true); - m[v] = dyn_var; - return dyn_var; - } else { - return it->second; - } - } - return dim2unsigned(v); -} - -std::vector sc_graph_t::dims_to_expr(const sc_dims &dim) { - std::vector dim_expr; - dim_expr.reserve(dim.size()); - for (auto d : dim) { - dim_expr.emplace_back(dim_to_expr(d)); - } - return dim_expr; -} - -std::vector sc_graph_t::dims_to_expr_c(const sc_dims &dim) { - std::vector dim_expr; - dim_expr.reserve(dim.size()); - for (auto d : dim) { - dim_expr.emplace_back(dim_to_expr(d)); - } - return dim_expr; -} - -float sc_graph_t::get_gflop() const { - float gflop = 0.f; - for (auto &op : ops_) { - if (op->is_removed_) - continue; - gflop += op->get_gflop(); - } - return gflop; -} - -bool sc_graph_t::is_dynamic() const { - return !std::all_of(ops_.begin(), ops_.end(), - [](const sc_op_ptr &op) { return !op->is_dynamic(); }); -} - -bool sc_graph_t::is_non_dense() const { - if (is_dynamic()) { - // currently dynamic graph is always dense - return false; - } - for (const sc_op_ptr &op : ops_) { - for (const graph_tensor_ptr > : op->get_inputs()) { - if (!gt->details_.is_dense()) { - return true; - } - } - } - return false; -} - -void sc_graph_t::add(const sc_op_ptr &ret) { - assert(ret->logical_op_id_ == 0); - ret->logical_op_id_ = ops_.size(); - ret->set_owner_graph(this); - for (auto &outs : ret->info_.outputs_) { - assert(outs->producer_owner_ == nullptr || - outs->producer_owner_ == ret.get()); - outs->producer_owner_ = ret.get(); - } - for (unsigned i = 0; i < ret->info_.inputs_.size(); i++) { - ret->info_.inputs_[i]->attach_use(ret, i); - } - ops_.emplace_back(ret); -} - -std::shared_ptr sc_graph_t::make( - const std::string &op_name, const std::vector &inputs, - const std::vector &outputs, const any_map_t &attrs) { - // make a reference to all needed ops to prevent them from being removed by - // linker - __dummy_init(); - std::shared_ptr ret, in_ret; - // internally create input_op - // todo: LLGA-sc front end should create input_op first, instead of creating - // it internally - for (auto &ins : inputs) { - if (!ins->producer_owner_) { - auto in_ret = - std::make_shared(std::vector{ins}); - in_ret->logical_op_id_ = ops_.size(); - ops_.emplace_back(std::move(in_ret)); - } - } - - std::string decay_op_name = graph::decay_quantized_op_name(op_name); - // firstly search template, secondly search fusible - // todo: add all tunable ops - if (auto f = get_op_factory(decay_op_name)) { - ret = f(inputs, outputs, attrs); - } else { - COMPILE_ASSERT(false, "Unsupported op: " << decay_op_name); - } - bool is_quantized = utils::string_startswith(op_name, "quantized"); - if (is_quantized) { - ret->dyn_cast()->is_quantized_ = true; - } - add(ret); - // As owner graph is initialzed after op's constructor and add, some ops - // like conv need infer output plain shapes at this step. - ret->infer_out_tensor_details(); - return ret; -} - -std::shared_ptr -sc_graph_t::make_output(const std::vector &inputs, - const any_map_t &attrs) { - auto ret = std::make_shared(inputs); - ret->owner_graph_ = this; - ret->attrs_ = attrs; - for (unsigned i = 0; i < inputs.size(); i++) { - inputs[i]->attach_use(ret, i); - } - ret->logical_op_id_ = ops_.size(); - ops_.emplace_back(ret); - return ret; -} - -std::shared_ptr -sc_graph_t::make_input(const std::vector &inputs, - const any_map_t &attrs) { - auto ret = std::make_shared(inputs); - ret->owner_graph_ = this; - ret->attrs_ = attrs; - ret->logical_op_id_ = ops_.size(); - if (ret->is_dynamic()) { - ret->initialize_dynamic_placeholder(); - } - ops_.emplace_back(ret); - return ret; -} - -std::vector sc_graph_t::get_output_ops() { - std::vector output_ops; - for (auto &op : ops_) { - if (op->isa()) { - output_ops.push_back(op); - } - } - return output_ops; -} -std::vector sc_graph_t::get_input_ops() { - std::vector input_ops; - for (auto &op : ops_) { - if (op->isa()) { - input_ops.push_back(op); - } - } - return input_ops; -} - -std::vector sc_graph_t::get_input_or_const_ops() const { - std::vector input_ops; - for (auto &op : ops_) { - if (op->isa() || op->isa()) { - input_ops.push_back(op); - } - } - return input_ops; -} - -std::unordered_set sc_graph_t::get_external_dynamic_vars() { - std::unordered_set ext_vars; - auto extract_vars = [&ext_vars](const std::vector &ops) { - for (auto &op : ops) { - for (auto &ins : op->get_inputs()) { - for (auto &d : ins->details_.get_plain_dims()) { - if (is_dynamic_dim(d)) { - ext_vars.insert(d); - } - } - } - for (auto &outs : op->get_outputs()) { - for (auto &d : outs->details_.get_plain_dims()) { - if (is_dynamic_dim(d)) { - ext_vars.insert(d); - } - } - } - } - }; - extract_vars(get_input_ops()); - extract_vars(get_output_ops()); - // dynamic reshape is also traited as external var. - std::vector dyn_reshapes; - for (auto &op : ops_) { - if (op->op_name_ == "dynamic_reshape") { - dyn_reshapes.push_back(op); - } - } - extract_vars(dyn_reshapes); - return ext_vars; -} - -bool sc_graph_t::need_dynamic_internal_query() { - return !std::all_of(ops_.begin(), ops_.end(), [](const sc_op_ptr &op) { - return !op->need_dynamic_internal_query(); - }); -} - -bool sc_op::need_dynamic_internal_query() { - bool ret = need_dynamic_internal_query_impl(); - if (ret && !info_.internal_info_) { - info_.internal_info_ = std::make_shared(); - } - return ret; -} - -bool sc_op::compare_contents( - const sc_op *other, // NOLINT - const std::function &filter) - const { - if (op_name_ != other->op_name_) { - return false; - } - int numattrs = 0, othernumattrs = 0; - auto &othermap = other->attrs_.as_map(); - for (auto &kv : attrs_.as_map()) { - if (utils::string_startswith(kv.first, "temp.")) { - continue; - } - if (filter && !filter(this, kv.first)) { - continue; - } - numattrs++; - auto otherkv = othermap.find(kv.first); - if (otherkv == othermap.end()) { - return false; - } - if (kv.second.cmp(otherkv->second) != 0) { - return false; - } - } - for (auto &kv : othermap) { - if (utils::string_startswith(kv.first, "temp.")) { - continue; - } - if (filter && !filter(this, kv.first)) { - continue; - } - othernumattrs++; - } - if (numattrs != othernumattrs) { - return false; - } - - return true; -} - -size_t sc_op::standard_hash_contents( - const sc_op *p, - const std::function &filter) { - size_t seed = 0; - for (auto &in : p->info_.inputs_) { - hash_combine(seed, in->details_.hash()); - } - for (auto &out : p->info_.outputs_) { - hash_combine(seed, out->details_.hash()); - } - hash_combine(seed, p->op_name_); - for (auto &kv : p->attrs_.as_map()) { - if (utils::string_startswith(kv.first, "temp.")) { - continue; - } - if (filter && !filter(p, kv.first)) { - continue; - } - // To hash unordered_map, use `XOR`, which satisfies commutative law. - // Otherwise, for ordered containers (like arrays), use `hash_combine` - // to distinguish result from the differnt sequence order. - if (!kv.second.empty()) { - seed ^= kv.second.hash(); - } - } - return seed; -} - -size_t sc_op::hash_contents( // NOLINT - const std::function &filter) - const { - return standard_hash_contents(this, filter); -} - -static std::unordered_map &get_op_factory_map() { - static std::unordered_map op_map; - return op_map; -} - -op_factory_func get_op_factory(const std::string &name) { - auto &op_map = get_op_factory_map(); - auto itr = op_map.find(name); - if (itr != op_map.end()) { - return itr->second; - } - return nullptr; -} - -void set_op_factory(const std::string &name, op_factory_func f) { - auto &op_map = get_op_factory_map(); - COMPILE_ASSERT(op_map.find(name) == op_map.end(), - "The op has already registered!"); - op_map[name] = f; -} - -float sc_op::get_gflop() { return 0.0f; } - -std::vector sc_op::get_impl_dispatch_candidates(const context_ptr &ctx) { - return {}; -} - -reflection::shared_general_object_t sc_op::get_dynamic_runtime_info() { - return reflection::shared_general_object_t(); -} - -namespace graph { -std::string decay_quantized_op_name(const std::string &op_name) { - bool is_quantized = utils::string_startswith(op_name, "quantized"); - std::string qstring = "quantized_"; - std::string decay_op_name = - is_quantized - ? op_name.substr(qstring.size(), op_name.size() - qstring.size()) - : op_name; - return decay_op_name; -} - -void get_logical_tensors(ltensors *ins, - const std::vector &flts) { - ins->reserve(flts.size()); - for (auto &in : flts) { - ins->emplace_back(in->details_); - } -} - -expr tensor_detail_to_ir_tensor(sc_graph_t &graph, const std::string &name, - const logical_tensor_t &tsrd) { - auto blocking_dims = tsrd.get_blocking_dims(); - auto strides = tsrd.get_strides(); - COMPILE_ASSERT(blocking_dims.size() == strides.size(), - "Dims and strides does not match."); - auto blocking_exprs = tsrd.get_blocking_dims_expr(graph); - auto tsr = - builder::make_stensor(name, blocking_exprs, tsrd.get_strides_expr(graph), - tsrd.dtype_, address_space::automatic, nullptr); - tsr->attr().set(attr_keys::plain_dims, - graph.dims_to_expr(tsrd.get_plain_dims())); - if (graph.is_dynamic()) { - tsr->attr().set(attr_keys::always_trans, true); - } - return tsr; -} - -std::vector -tensor_detail_to_ir_tensor(sc_graph_t &graph, const std::string &name_prefix, - const std::vector &tsrs) { - std::vector ret; - ret.reserve(tsrs.size()); - for (size_t i = 0; i < tsrs.size(); i++) { - ret.emplace_back(tensor_detail_to_ir_tensor( - graph, name_prefix + std::to_string(i), tsrs[i])); - } - return ret; -} - -std::vector -tensor_detail_to_ir_tensor(sc_graph_t &graph, const std::string &name_prefix, - const std::vector &tsrs) { - std::vector ret; - ret.reserve(tsrs.size()); - for (size_t i = 0; i < tsrs.size(); i++) { - ret.emplace_back(tensor_detail_to_ir_tensor( - graph, name_prefix + std::to_string(i), tsrs[i]->details_)); - } - return ret; -} - -ltensors extract_detail_from_tensors( - const std::vector> &flts) { - std::vector ret; - ret.reserve(flts.size()); - for (auto &in : flts) { - ret.emplace_back(in->details_); - } - return ret; -} - -sc_graph_t make_single_op_graph(const std::string &opname, - const std::vector &inputs, - const std::vector &outputs, - const any_map_t &attr) { - sc_graph_t ret; - ret.make_input(inputs); - auto op = ret.make(opname, inputs, outputs, attr); - ret.make_output(op->get_outputs()); - return ret; -} - -bool check_shape_equal(const sc_dims &lhs_shape, const sc_dims &rhs_shape) { - if (lhs_shape.size() != rhs_shape.size()) { - return false; - } - if (!std::equal(lhs_shape.begin(), lhs_shape.end(), rhs_shape.begin(), - [](const int x, const int y) { - return is_dynamic_dim(x) || is_dynamic_dim(y) || x == y; - })) { - return false; - } - return true; -} - -void check_logical_tensor_shape_dtype_identical(const logical_tensor_t &lhs, - const logical_tensor_t &rhs) { - COMPILE_ASSERT(check_shape_equal(lhs.get_plain_dims(), rhs.get_plain_dims()), - "The given logical tensors shall have the same shape.") - COMPILE_ASSERT(lhs.dtype_.type_code_ == rhs.dtype_.type_code_, - "The given logical tensors shall have the same dtype.") -} - -} // namespace graph -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -namespace std { -std::size_t hash::operator()( - const dnnl::impl::graph::gc::logical_tensor_t &k) const { - size_t seed = 0; - dnnl::impl::graph::gc::hash_combine(seed, k.dtype_); - dnnl::impl::graph::gc::hash_combine(seed, k.format_); - for (size_t i = 0; i < k.plain_dims_.size(); i++) { - dnnl::impl::graph::gc::hash_combine(seed, k.plain_dims_[i]); - } - return seed; -} -} // namespace std diff --git a/legacy/core/src/compiler/ir/graph/graph.hpp b/legacy/core/src/compiler/ir/graph/graph.hpp deleted file mode 100644 index cd578ab63..000000000 --- a/legacy/core/src/compiler/ir/graph/graph.hpp +++ /dev/null @@ -1,555 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_GRAPH_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_GRAPH_HPP - -#include "tensor_detail.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#if SC_MEMORY_LEAK_CHECK > 0 -#include -#endif - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -namespace reflection { -template struct type_registry; -struct shared_general_object_t; -} // namespace reflection - -struct graph_tensor; -class sc_op; -using graph_tensor_ptr = std::shared_ptr; -using sc_op_ptr = std::shared_ptr; - -// the additional data related to fusion manager, attached to logical_tensor_t -struct tensor_slice; -struct fusion_anchor_t; - -template struct gt_map_t; -using gt2gt_map = gt_map_t; -using gt2axis_map = gt_map_t>; -using gt2buf_map = gt_map_t; - -using slice_range = std::vector>; -using slice_range_list = std::vector; -using fslice_map = gt_map_t; - -using format_stride_pair = std::pair; -using shape_rl_vec = std::vector>; - -struct dispatch_key_set_base_t; -using dispatch_set_ptr = std::shared_ptr; -struct dyn_internal_info_t; - -/** VConst struct record possible varible in constant value, e.g. - * - * const int a = k * b; - * - * where `k` maybe variable related on other factor such as blocking dims. - * */ -struct VConst { - // variable value - int64_t var_; -}; - -// a weak pointer which always asserts the object exists -struct sc_op_weak_ptr_t : public std::weak_ptr { - using parent = std::weak_ptr; - using parent::parent; - using parent::operator=; - - sc_op_weak_ptr_t(const sc_op_weak_ptr_t &) = default; - sc_op_weak_ptr_t(sc_op_weak_ptr_t &&) = default; - - sc_op_weak_ptr_t &operator=(sc_op *); - sc_op_weak_ptr_t &operator=(const sc_op_weak_ptr_t &other) { - parent::operator=(other); - return *this; - } - bool operator!() const { return get(); } - bool operator==(const sc_op_weak_ptr_t &other) const { - return get() == other.get(); - } - bool operator==(const sc_op_ptr &other) const { return get() == other.get(); } - sc_op *operator*() const { return get(); } - sc_op *operator->() const { return get(); } - - sc_op_ptr get_shared() const { - auto ret = lock(); - assert(ret); - return ret; - } - sc_op *get() const { return get_shared().get(); } - sc_op_weak_ptr_t &operator=(const sc_op *other); - operator sc_op_ptr() const { return get_shared(); } -}; - -// the logical tensor for in the graph to represent a result value in the graph. -// It contains the tensor details (shape, dtype, etc.) and the connectivity of -// the value in the graph -struct SC_API graph_tensor : public std::enable_shared_from_this - SC_EXTENDS_LEAK_CHECK(graph_tensor) { - logical_tensor_t details_; - // todo(zhichen/yijie): producer_owner should be used weak pointer. - sc_op *producer_owner_{nullptr}; - // the op nodes that use this tensor as input - std::vector> uses_; - - any_map_t attrs_; - - graph_tensor(sc_op *owner); - graph_tensor(sc_op *owner, const logical_tensor_t <); - graph_tensor(sc_op *owner, const sc_data_format_t &format, - const sc_dims &plain_shape, const sc_data_type_t &type, - const sc_dims &stride = {}); - - // adds a node to the `uses_` list, given the index of the tensor in the - // inputs of the op - void attach_use(sc_op_ptr op, int index); - // removes a node from the `uses_` list (may need a linear search) - void detach_use(const sc_op_ptr &op); - // removes a node to the `uses_` list, given the index of the tensor in the - // inputs of the op - void detach_use(const sc_op_ptr &op, int input); - - // replaces all uses of this tensors with `v`. Will call `replace_input` of - // all uses of this - void replace_with(const graph_tensor_ptr &v); - graph_tensor_ptr copy(); - ~graph_tensor() = default; - - static graph_tensor_ptr - make(const sc_dims &shape, const sc_data_format_t &fmt = sc_data_format_t(), - sc_data_type_t dtype = sc_data_type_t(sc_data_etype::F32, 1), - const sc_dims &stride = {}) { - return std::make_shared(nullptr, fmt, shape, dtype, stride); - } - bool is_dynamic() const { return details_.is_dynamic(); } -}; - -using ltensors = std::vector; - -struct sc_op_info_t { - std::vector outputs_; - std::vector inputs_; - // set of all dynamic dispatch keys combinations of op, this field is mainly - // prepared for dynamic dispatch during lowering and is created during - // layout propagation. - dispatch_set_ptr dispatch_key_set_; - // Extra info for op who could be internal queried. - std::shared_ptr internal_info_; - // current used impl type - int cur_impl_ = 0; -}; - -struct op_base_trait_t {}; - -namespace op_attr_key { -// Boolean. If true, don't fuse this Op. Default = false -constexpr const char *no_fuse = "no_fuse"; -// Boolean. If true, don't break the fusion partition after this Op -// (this Op can still be fused with previous Ops). Default = false -constexpr const char *break_post_fuse = "break_post_fuse"; -// Boolean. If true, don't break the fusion partition before this Op -// (this Op can still be fused with post Ops). Default = false -constexpr const char *break_pre_fuse = "break_pre_fuse"; -// Fuse Anchor -constexpr const char *inner_anchor = "inner_anchor"; -// Inner anchor created in fusible ops and will be add to mixed partition when -// committing fusible op. -constexpr const char *fusible_inner_anchors = "fusible_inner_anchors"; -// the name of the layer. Will be used to name the IR function -constexpr const char *layer_name = "temp.name"; -// op marked with not_redundant will not be removed in horizontal same op -// elimination -constexpr const char *not_redundant = "temp.not_redundant"; -// binary/ternary elementwise op layout propagation source input index. -constexpr const char *layout_input_index = "layout_input_index"; -// Could use mask select to process output for reduce, matmul or other memory -// movement op. -constexpr const char *use_padded_mask = "use_padded_mask"; -// Boolean. If true, it will skip graph pass div_bcast_transform. The precision -// requirements are high, and division must be used in the calculation of op. -constexpr const char *must_div = "must_div"; -// Boolean. If true, the optimized formula will be used when norm calculates -// mean and var. -constexpr const char *use_norm_opt = "use_norm_opt"; -}; // namespace op_attr_key - -class sc_graph_t; -class SC_INTERNAL_API sc_op : public virtual op_base_trait_t, - public std::enable_shared_from_this - SC_LEAK_CHECK(sc_op) { -public: - sc_graph_t *owner_graph_{nullptr}; - sc_op_info_t info_; - any_map_t attrs_; - // the logical op ID in the op in the manager, default is 0. - int logical_op_id_ = 0; - // todo(zhichen): remove this, replace by dynamic cast check - // is or not template op - bool is_fusible_ = false; - // align template name - std::string op_name_; - // if the node is removed from the manager - bool is_removed_ = false; - // get op infos - const sc_op_info_t &get_info() const { return info_; } - // get input logical tensors - const std::vector &get_inputs() const { - return info_.inputs_; - } - // get output logcial tesnors - const std::vector &get_outputs() const { - return info_.outputs_; - } - - virtual const dispatch_set_ptr &get_dispatch_key_set() const; - virtual dispatch_set_ptr &get_dispatch_key_set(); - // internal query disaptch keys, mainly for impl kind. - virtual dispatch_set_ptr - get_internal_dispatch_key_set(const context_ptr &ctx); - // call impl_func if return true, create the internal info. - bool need_dynamic_internal_query(); - // if the op needs internal query, default false. - virtual bool need_dynamic_internal_query_impl() const { return false; } - void copy_dispatch_key_set_from_op(const sc_op_ptr &other); - /** - * Repalces an input logical tensor - * @param index the index within get_inputs() - * @param new_input the new logical tensor - * @param skip_shape_check whether to check new shape vs original shape, for - * padding op the shape could be different - * */ - void replace_input(size_t index, const graph_tensor_ptr &new_input, - const bool skip_shape_check = false); - - // Replaces the current Op in the graph using another Op. All other Ops - // using the output tensors of current Op will use the corresponding tensors - // in the replacer Op instead. Finally the current node will be removed - // Requires that the replacer has the same number of outputs of the current - // node. Will detach from the input tensors. The replacer should manually - // attach to the input tensors when it is needed - void replace_uses_with_and_remove(const sc_op_ptr &replacer); - - /** - * Checks whether any output tensor of this op is the output of the graph. - */ - bool has_graph_output() const; - - // the op is single output and the output is single used - bool is_single_output_single_use(); - - // the op contains dynamic shape calculation, we didn't store a boolean - // value cache for op/graph, because they are dynamic themselves. - bool is_dynamic() const; - - // Get relationship of shapes from input/output plain shapes. Return - // a vector of a pair of parent shape placeholder and child shape - // placeholder. If one of the shapes corresponding to parent and child is - // static, the other is also inferred as static, e.g. binary elemwise op - // input shapes [-1, 64] and [16, 64], -1 will be inferred as 16. - virtual shape_rl_vec get_dynamic_shape_relations() const { return {}; } - // Get calculation expressions between different dynamic vars in - // output/input. The expressions will be used by internal dynamic vars - // inside kernel which does not infer shape. The expression will store in - // var expr with attribute `pass.cal_expression`. - virtual void calculate_dynamic_shape_expression() {} - // the op share given graph tensor with opT(except itself) - template bool share_gt_with_op(const graph_tensor_ptr >) { - auto ths = this; - return std::any_of(gt->uses_.begin(), gt->uses_.end(), - [&ths](const std::pair &user) { - return user.second->isa() && - (user.second.get() != ths); - }); - } - - // Marks this node invalid and detach_use from all input tensors - void remove(); - - // get op's owner graph - sc_graph_t &get_owner_graph() const { return *owner_graph_; } - void set_owner_graph(sc_graph_t *owner_graph) { owner_graph_ = owner_graph; } - // infer output details like plain shapes, most op infer in their - // constructors except for conv now. - virtual void infer_out_tensor_details() {} - /** - * Checks if the node is of a subclass. - * @param T the subclass - * */ - - template bool isa() const { - static_assert(is_base_of_t::value || - is_base_of_t::value, - "T is not a subclass of sc_op/op_trait."); - return dynamic_cast(this); - } - - /** - * Dynamic cast the node to a subclass or a trait. - * @param T the subclass - * */ - template T *dyn_cast() const { return dynamic_cast(this); } - - template T *dyn_cast() { return dynamic_cast(this); } - - /** - * Static cast the node to a subclass or a trait. - * @param T the subclass - * */ - template T *stc_cast() { - assert(isa() && "check T failed."); - return static_cast(this); - } - - /** - * Compares the contents (op_name/attrs/other fields in the op). The default - * implementation only compares op_name and attrs. This function does not - * check the op-tensor connections, which will be checked by the - * graph_comparer. - * @param filter the filter for the attr name. If it returns false, the attr - * will be ignored. By default it only filters out keys starting with - * "temp." - * @note we ingore the attrs with keys starting with "temp." - * @return true if the contents (not including the op connections) are the - * same - * */ - virtual bool compare_contents( - const sc_op *other, - const std::function &filter = - nullptr) const; - - /** - * Hash the contents. The default implementation only hashs op_name and - * attrs. The function can be used to make hash map. When hash conflict - * happened, we can compare them with `compare_contents`. - * @param filter the filter for the attr name. If it returns false, the attr - * will be ignored. By default it only filters out keys starting with - * "temp." - * @note we ingore the attrs with keys starting with "temp." - * @return hash value with size_t datatype. - * */ - virtual size_t hash_contents( - const std::function &filter = - nullptr) const; - - // the default implementation of hash_contents - static size_t standard_hash_contents( - const sc_op *p, - const std::function &filter); - - // constructor - sc_op(const std::string &op_name, - const std::vector &producer_lt, - const std::vector &consumer_lt, - const any_map_t &attrs); - sc_op() = default; - sc_op(const sc_op &) = delete; - sc_op(sc_op &&) = delete; - - virtual bool is_valid(const context_ptr &ctx) { return true; } - virtual ir_module_ptr get_func(context_ptr ctx) = 0; - virtual ir_module_ptr get_internal_func(const context_ptr &ctx) { - throw std::runtime_error("Unimplement."); - } - - virtual void query_format( - context_ptr ctx, - std::vector> &supported_ins, - std::vector> &supported_outs) = 0; - void format_to_dense_format_stride_pair( - const std::vector> &in_formats, - const std::vector> &out_formats, - std::vector> &supported_ins, - std::vector> &supported_outs); - - virtual ~sc_op() = default; - virtual float get_gflop(); - // Get op impl type candidates for dynamic dispatch. Default candidates are - // padding/no_padding. Return the impl alg candidates vector, element is - // int(not enum) because different ops have different impl algs. - virtual std::vector get_impl_dispatch_candidates(const context_ptr &ctx); - virtual reflection::shared_general_object_t get_dynamic_runtime_info(); -}; - -inline sc_op_weak_ptr_t &sc_op_weak_ptr_t::operator=(sc_op *other) { - *this = other->shared_from_this(); - return *this; -} - -class SC_INTERNAL_API input_op; -class SC_INTERNAL_API output_op; - -std::vector -copy_logical_tsr(const std::vector &v); - -struct dynamic_lower_info_t; -class SC_API sc_graph_t { -public: - // the attr keys for graph - struct attr_key_t { - static constexpr const char *gflop = "gflop"; - static constexpr const char *quantize = "quantize"; - static constexpr const char *bf16 = "bf16"; - static constexpr const char *fp16 = "fp16"; - // if false, will keep the blocking format of output tensor - static constexpr const char *is_output_plain = "is_output_plain"; - // if false, when an input_op is plain format and has only one use, will - // set the input's data format to the expected format of the use - static constexpr const char *is_input_plain = "is_input_plain"; - // if true, will allow output to be in channel last format - static constexpr const char *allow_channel_last_output = - "allow_channel_last_output"; - static constexpr const char *fpmath_mode = "fpmath_mode"; - }; - - std::vector ops_; - any_map_t attrs_; - // lowering info related to dynamic - std::shared_ptr dyn_info_; - - // adds an existing node to the graph - void add(const sc_op_ptr &node); - - // return true if the graph is empty - bool empty() const { return ops_.empty(); } - - std::shared_ptr make(const std::string &op_name, - const std::vector &inputs, - const std::vector &outputs, - const any_map_t &attrs); - - template - std::shared_ptr make(Args &&...args) { - static_assert(!(std::is_same::value), - "output_op should go to specialized function"); - static_assert(!(std::is_same::value), - "output_op should go to specialized function"); - auto ret = std::make_shared(std::forward(args)...); - add(ret); - return ret; - } - - /** - * Hash the contents. The default implementation only hashs ops and - * attrs. The function can be used to make hash map. When hash conflict - * happened, we can compare them with `compare_graph`. - * @param filter the filter for the attr name. If it returns false, the attr - * will be ignored. By default it only filters out keys starting with - * "temp." - * @note we ingore the attrs with keys starting with "temp." - * @return hash value with size_t datatype. - * */ - size_t hash_contents( - const std::function &filter = - nullptr) const; - - // This function removes the Ops with is_removed_=true. And it compresses - // the ops_ array by removing the holes of removed ops. It finally resets - // all ids of each ops, to keep the continuity of ids. - void reset_op_ids(); - // This function is aimed to re-sort the ops in the graph's `ops_` array - // using the op-ids in the op_id_map - void resort_op_ids(const std::unordered_map &op_id_map); - - // get next valid dynamic placeholder (decreasing) in graph. - // We use [-2, -min_of_int64_t] to represent relationships between dynamic - // shapes. It will be called by infer shapes(constructor of each op). - // E.g. plain shape [-2, 64, -2, 128, -3, -4], first and third dimension are - // the same dynamic var, and the last two are separated. - sc_dim get_next_dynamic_placeholder(); - // return a expr (vector) from input dim (vector), if dim is dynamic, first - // find in dim2expr_map_, if not found, create a var based on placeholder. - expr dim_to_expr(const sc_dim &); - std::vector dims_to_expr(const sc_dims &dim); - std::vector dims_to_expr_c(const sc_dims &dim); - - // Get the total gflop from all tunable ops contained in the graph. - float get_gflop() const; - bool is_dynamic() const; - bool is_non_dense() const; - std::vector get_output_ops(); - std::vector get_input_ops(); - std::vector get_input_or_const_ops() const; - - // sync dynamic placeholder and dim2expr map with other graph. When you - // create a subgraph b of graph a, you need to call - // b.sync_dynamic_info_with_graph(a). - void sync_dynamic_info_with_graph(const sc_graph_t &other) { - dyn_info_ = other.dyn_info_; - } - // Get external dynamic vars existed in inputs/outputs. - std::unordered_set get_external_dynamic_vars(); - // Judge if the ops in graph need dynamic internal query - bool need_dynamic_internal_query(); - // output op - std::shared_ptr - make_output(const std::vector &inputs, - const any_map_t &attrs = any_map_t()); - // input op - std::shared_ptr make_input(const std::vector &inputs, - const any_map_t &attrs = any_map_t()); - sc_graph_t() = default; - sc_graph_t(sc_graph_t &&other); - sc_graph_t &operator=(sc_graph_t &&other); - sc_graph_t(const sc_graph_t &other) = delete; -}; - -std::vector get_blocking_shapes_expr(sc_graph_t &graph, - const sc_dims &plain_shapes, - const sc_data_format_t &format); - -using op_factory_func = sc_op_ptr (*)(const std::vector &, - const std::vector &, - const any_map_t &); -op_factory_func get_op_factory(const std::string &name); -void set_op_factory(const std::string &name, op_factory_func f); - -template struct register_helper_t { - static bool op_call(const std::string &op_name) { - auto func = [](const std::vector &ins, - const std::vector &outs, - const any_map_t &attrs) -> sc_op_ptr { - return std::shared_ptr(new T(ins, outs, attrs)); - }; - set_op_factory(op_name, func); - return false; - } -}; -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl -#define OP_REGISTER(IDENTIFIER, NAME) \ - volatile bool __help_dummy_##NAME = \ - dnnl::impl::graph::gc::register_helper_t::op_call(#NAME); -#endif diff --git a/legacy/core/src/compiler/ir/graph/graph_config.cpp b/legacy/core/src/compiler/ir/graph/graph_config.cpp deleted file mode 100644 index 697d6e8c3..000000000 --- a/legacy/core/src/compiler/ir/graph/graph_config.cpp +++ /dev/null @@ -1,66 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "graph_config.hpp" -#include "graph_op.hpp" -#include "traits.hpp" -#include "tunable_op.hpp" -#include "util/utils.hpp" -#include "visitor.hpp" -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -// clang-format off -SC_CLASS(graph_config) - SC_FIELD(op_cfgs_) -SC_CLASS_END(); -// clang-format on - -namespace graph { - -void set_graph_config(sc_graph_t &g, const graph_config &tcfg) { - size_t visited_num = 0; - for (auto &op : g.ops_) { - if (auto tune_op = op->dyn_cast()) { - tune_op->set_config(tcfg.op_cfgs_.at(visited_num++)); - } - } -} - -graph_config get_graph_default_config(context_ptr ctx, const sc_graph_t &g) { - graph_config cfg; - op_visitor_t vis = op_visitor_t::bfs_topology_sort(g.ops_.size()); - vis.visit_graph(g, [&](op_visitor_t *vis, const sc_op_ptr &op) { - if (auto tune_op = op->dyn_cast()) { - auto obj = tune_op->get_default_config(ctx); - cfg.op_cfgs_.emplace_back(std::move(obj)); - } - }); - return cfg; -} -} // namespace graph - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/graph_config.hpp b/legacy/core/src/compiler/ir/graph/graph_config.hpp deleted file mode 100644 index 35e2c380a..000000000 --- a/legacy/core/src/compiler/ir/graph/graph_config.hpp +++ /dev/null @@ -1,50 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_GRAPH_CONFIG_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_GRAPH_CONFIG_HPP - -#include "graph.hpp" -#include "util/general_object.hpp" -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -// todo(zhichen): replaced by any map -struct graph_config { - std::vector op_cfgs_; - // maybe anther config item in the future -}; -namespace tuner { -struct config_space; -} // namespace tuner - -namespace graph { -SC_INTERNAL_API graph_config get_graph_default_config(context_ptr ctx, - const sc_graph_t &g); -SC_INTERNAL_API void set_graph_config(sc_graph_t &g, - const graph_config &config); -} // namespace graph -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/graph/graph_map.hpp b/legacy/core/src/compiler/ir/graph/graph_map.hpp deleted file mode 100644 index a42807c8c..000000000 --- a/legacy/core/src/compiler/ir/graph/graph_map.hpp +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright (C) 2025 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions - * and limitations under the License. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_GRAPH_MAP_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_GRAPH_MAP_HPP - -#include "graph.hpp" -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -template struct is_vector { - static constexpr bool value = false; -}; - -template struct is_vector> { - static constexpr bool value = true; -}; - -// the map based on graph_tensor key -template struct gt_map_t { - std::unordered_map datamap_; - - gt_map_t() = default; - template - gt_map_t(const std::unordered_map &datamap) - : datamap_(datamap) {} - // get the reference of value for the corresponding key, can be either read - // or write - valT &get(graph_tensor *); - valT &get(const graph_tensor_ptr &); - // return true if has key - bool haskey(const graph_tensor_ptr &) const; - bool haskey(graph_tensor *) const; - - // specialization for vector tpyes of valT, which owns `empty` method - template < - typename T = void, - typename dummy = typename std::enable_if< - std::is_same::value && is_vector::value, bool>::type> - bool hasvalue(graph_tensor *v) { - return haskey(v) && !get(v).empty(); - } - - template < - typename T = void, - typename dummy = typename std::enable_if< - std::is_same::value && is_vector::value, bool>::type> - bool hasvalue(const graph_tensor_ptr &v) { - return hasvalue(v.get()); - } - - // clear datamap - void clear() { datamap_.clear(); } - // return true if empty - bool empty() const { return datamap_.empty(); } - // erase by key - void erase(const graph_tensor_ptr k) { datamap_.erase(k.get()); } - gt_map_t &operator=(const gt_map_t &other) = delete; -}; -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl -#endif diff --git a/legacy/core/src/compiler/ir/graph/graph_op.cpp b/legacy/core/src/compiler/ir/graph/graph_op.cpp deleted file mode 100644 index 08b0c98f2..000000000 --- a/legacy/core/src/compiler/ir/graph/graph_op.cpp +++ /dev/null @@ -1,170 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "graph_op.hpp" -#include "lowering.hpp" -#include "util/utils.hpp" -#include "visitor.hpp" -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -std::vector -graph_op_t::remake_logical_tensors(const std::vector &flts) { - std::vector new_flts(flts.size()); - for (size_t i = 0; i < flts.size(); ++i) { - new_flts[i] = std::make_shared( - nullptr, flts[i]->details_.get_format(), - flts[i]->details_.get_plain_dims(), flts[i]->details_.dtype_); - } - return new_flts; -} - -ir_module_ptr graph_op_t::get_func(context_ptr ctx) { - auto graph = get_graph(); - return lower_graph(ctx, *graph, {graph->ops_.back(), graph->ops_[0]}, false); -} - -std::shared_ptr graph_op_t::get_graph() { - auto g = std::make_shared(); - g->sync_dynamic_info_with_graph(get_owner_graph()); - get_graph_impl(g); - return g; -} - -graph_tensor_ptr -graph_op_t::cast_input_dtype(graph_tensor_ptr &inp, - std::shared_ptr &graph, - const any_map_t &attrs) { - graph_tensor_ptr input = inp; - if (inp->details_.dtype_.is_etype(sc_data_etype::BF16) || - inp->details_.dtype_.is_etype(sc_data_etype::F16)) { - auto cast_input = graph->make( - "cast", {inp}, {}, - attrs.as_map().empty() ? any_map_t{{"dtype", datatypes::f32}} : attrs); - input = cast_input->get_outputs()[0]; - } - return input; -} - -std::shared_ptr graph_op_t::cast_output_dtype( - graph_tensor_ptr &inp, std::shared_ptr &graph, - std::shared_ptr &last_op, const any_map_t &attrs) { - std::shared_ptr &cast_output = last_op; - if (inp->details_.dtype_.is_etype(sc_data_etype::BF16) || - inp->details_.dtype_.is_etype(sc_data_etype::F16)) { - auto target_dtype = inp->details_.dtype_; - cast_output = graph->make( - "cast", last_op->get_outputs(), {}, - attrs.as_map().empty() ? any_map_t{{"dtype", target_dtype}} : attrs); - } - return cast_output; -} - -std::shared_ptr configurable_graph_op_t::get_graph() { - auto g = std::make_shared(); - g->sync_dynamic_info_with_graph(get_owner_graph()); - get_graph_impl(g); - // set config space; - return g; -} - -config_ptr configurable_graph_op_t::get_config() { - return reflection::general_object_t::make(config_data_); -} - -void configurable_graph_op_t::set_config(const config_ptr &config) { - config_data_ = *config.get_as(); -} - -config_ptr configurable_graph_op_t::get_default_config(context_ptr ctx) { - auto op_graph = this->get_graph(); - return reflection::general_object_t::make( - graph::get_graph_default_config(ctx, *op_graph)); -} - -nested_graph_op_t::nested_graph_op_t(const std::string &op_name, - const std::vector &ins, - const std::vector &outs, - const any_map_t &attrs, sc_graph_t &&graph) - : configurable_graph_op_t(op_name, ins, outs, attrs), - graph_(std::move(graph)) { - if (outs.empty()) { - for (auto &out_op : graph_.get_output_ops()) { - info_.outputs_.insert(info_.outputs_.end(), out_op->get_inputs().begin(), - out_op->get_inputs().end()); - } - info_.outputs_ = remake_logical_tensors(info_.outputs_); - for (auto &op : info_.outputs_) { - op->producer_owner_ = this; - } - } - - auto required_input_tsenor_num = 0ul; - for (auto &in_op : graph_.get_input_ops()) { - required_input_tsenor_num += in_op->get_outputs().size(); - } - COMPILE_ASSERT(required_input_tsenor_num == ins.size(), - "The number of input tensor " - << ins.size() << " is incorrect. The required number is " - << required_input_tsenor_num); - - // create new input logical tensors - std::vector inputs, outputs; - inputs = remake_logical_tensors(info_.inputs_); - - // combine all input op into one - int counter = 0; - for (auto &op : graph_.get_input_ops()) { - for (size_t idx = 0; idx < op->get_outputs().size(); ++idx) { - op->get_outputs()[idx]->replace_with(inputs.at(counter++)); - } - op->remove(); - } - auto in_op = graph_.make_input(inputs); - - // combine all output op into one - counter = 0; - for (auto &op : graph_.get_output_ops()) { - outputs.insert(outputs.end(), op->get_inputs().begin(), - op->get_inputs().end()); - op->remove(); - } - auto out_op = graph_.make_output(outputs); - - // delete those removed op - graph_.reset_op_ids(); -} - -void nested_graph_op_t::get_graph_impl(std::shared_ptr &graph) { - graph = std::make_shared(copy_graph(graph_)); -} - -// linter has a false alarm to treat copy here as a STL function -sc_op_ptr nested_graph_op_t::copy( // NOLINT - const std::vector &ins, - const std::vector &outs, sc_graph_t &mgr) { - auto ret = mgr.make(this->op_name_, ins, outs, attrs_, - copy_graph(graph_)); - return ret; -} -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/graph_op.hpp b/legacy/core/src/compiler/ir/graph/graph_op.hpp deleted file mode 100644 index f2f18401c..000000000 --- a/legacy/core/src/compiler/ir/graph/graph_op.hpp +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Copyright (C) 2025 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions - * and limitations under the License. - * SPDX-License-Identifier: Apache-2.0 - */ -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_GRAPH_OP_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_GRAPH_OP_HPP - -#include "graph.hpp" -#include "util/general_object.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -class graph_op_t : public sc_op { -public: - graph_op_t() = default; - - graph_op_t(const std::string &op_name, - const std::vector &producer_lt, - const std::vector &consumer_lt, - const any_map_t &attrs) - : sc_op(op_name, producer_lt, consumer_lt, attrs) {} - - ir_module_ptr get_func(context_ptr ctx) override; - - void query_format( - context_ptr ctx, - std::vector> &supported_ins, - std::vector> &supported_outs) override {}; - - // the param graph is created by upper function and passed to this function. - // It should be an empty graph and already synced with external graph. - // For the alignment of outer and inner dynamic graph. - virtual void get_graph_impl(std::shared_ptr &graph) = 0; - - virtual std::shared_ptr get_graph(); - - static std::vector - remake_logical_tensors(const std::vector &flts); - - float get_gflop() override { return get_graph()->get_gflop(); } - - static graph_tensor_ptr cast_input_dtype(graph_tensor_ptr &inp, - std::shared_ptr &graph, - const any_map_t &attrs = {}); - - static std::shared_ptr - cast_output_dtype(graph_tensor_ptr &inp, std::shared_ptr &graph, - std::shared_ptr &last_op, - const any_map_t &attrs = {}); -}; - -class configurable_graph_op_t : public graph_op_t, - public op_traits::configurable_t { -public: - configurable_graph_op_t() = default; - - configurable_graph_op_t(const std::string &op_name, - const std::vector &producer_lt, - const std::vector &consumer_lt, - const any_map_t &attrs) - : graph_op_t(op_name, producer_lt, consumer_lt, attrs) {} - - std::shared_ptr get_graph() override; - - config_ptr get_config() override; - - void set_config(const config_ptr &config) override; - - reflection::shared_general_object_t - get_default_config(context_ptr ctx) override; - - config_ptr_vec - get_dynamic_config_candidates(const context_ptr &ctx) override { - throw std::runtime_error("Unimplement."); - } - impl_kind_map convert_config_candidates_to_impl_map( - const config_ptr_vec &configs) override { - throw std::runtime_error("Unimplement."); - } - -protected: - graph_config config_data_; -}; - -/** - * The nested graph op - * Used to convert a graph to a graph op which could be - * reused in other graph. - * Ins: - * - The corresponding input tensors of the nested graph - * Outs: - * - The corresponding output tensors of the nested graph - * Attrs: - * - The attribute of the op - * Graph: - * - The graph used to convert a graph to a nested graph op. - * */ -class SC_INTERNAL_API nested_graph_op_t : public configurable_graph_op_t, - public op_traits::copyable_t { -public: - nested_graph_op_t(const std::string &op_name, - const std::vector &ins, - const std::vector &outs, - const any_map_t &attrs, sc_graph_t &&graph); - - void get_graph_impl(std::shared_ptr &graph) override; - - // linter has a false alarm to treat copy here as a STL function - sc_op_ptr copy(const std::vector &ins, // NOLINT - const std::vector &outs, - sc_graph_t &mgr) override; - -protected: - sc_graph_t graph_; -}; - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/graph/lowering.cpp b/legacy/core/src/compiler/ir/graph/lowering.cpp deleted file mode 100644 index 5fc511325..000000000 --- a/legacy/core/src/compiler/ir/graph/lowering.cpp +++ /dev/null @@ -1,1512 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "lowering.hpp" -#include "fusible_op.hpp" -#include "graph.hpp" -#include "pass/pass.hpp" -#include "visitor.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -SC_MODULE(graph.lowering) - -static expr make_global_string(const ir_module_ptr &mod, const std::string &v, - int &counter) { - std::string name = "__gstring"; - name += std::to_string(counter++); - auto contents = std::make_shared(v.c_str(), v.size() + 1); - auto ret = builder::make_tensor(name, {v.size() + 1}, datatypes::s8, - address_space::automatic, contents); - auto def = - builder::make_var_tensor_def_unattached(ret, linkage::private_global); - mod->add_global_var(def.checked_as()); - return ret; -} - -static graph_tensor_ptr get_linked_output_tsr(const graph_tensor_ptr <ensor) { - if (ltensor->producer_owner_->isa()) { - return nullptr; - } - if (!ltensor->uses_.empty()) { - for (size_t i = 0; i < ltensor->uses_.size(); i++) { - if (ltensor->uses_[i].second->isa() || - ltensor->uses_[i].second->isa()) { - auto reshape = ltensor->uses_[i].second; - auto next_ltensor = reshape->get_outputs()[0]; - for (auto &cld : next_ltensor->uses_) { - if (cld.second->isa()) { - return cld.second->get_inputs()[cld.first]; - } else if (cld.second->isa() || - cld.second->isa()) { - auto cur_linked_out = get_linked_output_tsr(next_ltensor); - if (cur_linked_out) { - return cur_linked_out; - } - } - } - } - } - } - return nullptr; -} - -static bool has_output_uses(const graph_tensor_ptr <ensor) { - if (!ltensor->uses_.empty()) { - for (size_t i = 0; i < ltensor->uses_.size(); i++) { - if (ltensor->uses_[i].second->isa()) { - return true; - } - } - } - return false; -} - -struct lowering_visitor_state_t { - std::unordered_map tensor_pending_refcount_; - op_visitor_t::updater_func topo_sorter_; - std::vector op_exec_tick_; - std::vector op_visited_; - // need to visit the input outs in reversed order to align to old lowering - // input argument order (like pop_back_selector). Our visitor must visit - // the input ops first - std::list::iterator input_op_itr; - size_t cur_tick_ = 0; - size_t max_tensor_size_; - bool is_dynamic_; - - lowering_visitor_state_t(sc_graph_t &g) - : topo_sorter_{op_visitor_t::create_DAG_updater(g.ops_.size())}, - op_exec_tick_(g.ops_.size()), op_visited_(g.ops_.size()) { - max_tensor_size_ = 0; - is_dynamic_ = g.is_dynamic(); - if (!is_dynamic_) { - for (auto &op : g.ops_) { - for (auto &tsr : op->get_outputs()) { - max_tensor_size_ = std::max(max_tensor_size_, - tsr->details_.get_blocking_byte_size()); - } - } - } - } - - size_t &get_tensor_pending_refcount(const graph_tensor_ptr &p) { - auto itr = tensor_pending_refcount_.find(p); - if (itr == tensor_pending_refcount_.end()) { - auto ret = - tensor_pending_refcount_.insert(std::make_pair(p, p->uses_.size())); - return ret.first->second; - } - return itr->second; - } - - op_visitor_t::updater_func get_updater() { - auto ths = this; - return [ths](op_visitor_t *vis, const sc_op_ptr &op) { - for (auto &in : op->get_inputs()) { - ths->get_tensor_pending_refcount(in)--; - } - auto tick = ths->cur_tick_++; - if (op->isa() || op->isa()) { - ths->op_exec_tick_[op->logical_op_id_] = 0; - } else { - ths->op_exec_tick_[op->logical_op_id_] = tick; - } - ths->op_visited_[op->logical_op_id_] = true; - ths->topo_sorter_(vis, op); - }; - } - - // find the distance of an op to the visited ops - int get_op_distance_to_visited_set(sc_op *op, std::vector &d) { - auto id = op->logical_op_id_; - if (op_visited_[id]) { - return 0; - } - if (d[id] != 0) { - return d[id]; - } - if (op->isa()) { - d[id] = 0; - return 0; - } - int ret = -1; - for (auto &v : op->get_inputs()) { - int cur_d = get_op_distance_to_visited_set(v->producer_owner_, d) + 1; - ret = std::max(ret, cur_d); - } - d[id] = ret; - return ret; - } - - static constexpr float distance_factor = 2.0f; - // for each input tensor, check if the refcount=1. If so, it means that - // after the Op is visited, the input tensor is no longer needed compute the - // score of each visitable candidate op. the score is "SUM_{each input - // tensor}(normalized_sizeof(tensor)/ref_count_modifier*heat_modifier) - - // SUM_{each output tensor}(normalized_sizeof(tensor)+ distance_modifier)" - float evaluate_op_score(sc_op *op, std::vector &distance_to_visited) { - float cur_score = 0; - - for (auto &in : op->get_inputs()) { - // if the input tensor is input_op, there is no temp buffer to be - // free'd - if (!in->producer_owner_->isa()) { - // compute the heat modifier of the tensor. The hotter - // the tensor is (computed lately), the larger the - // modifier. - auto owner = in->producer_owner_; - auto tick_diff = cur_tick_ - op_exec_tick_[owner->logical_op_id_]; - assert(cur_tick_ > op_exec_tick_[owner->logical_op_id_]); - float heat_modifier; - switch (tick_diff) { - case 0: - case 1: - heat_modifier = 2.5f; - break; - case 2: - heat_modifier = 1.5f; - break; - default: - heat_modifier = 1.0f; - } - // if it is last use, ref_count_modifier=1. If not, - // ref_count_modifier=number of uses - size_t ref_count_modifier; - if (this->get_tensor_pending_refcount(in) == 1) { - ref_count_modifier = 1; - } else { - ref_count_modifier = in->uses_.size(); - } - float cur_tsr = is_dynamic_ - ? heat_modifier - : float(in->details_.get_blocking_byte_size()) / - ref_count_modifier / max_tensor_size_ * - heat_modifier; - cur_score += cur_tsr; - } - } - for (auto &out : op->get_outputs()) { - // if this output is connected to output op, it is not a temp - // buffer, and we don't need to count its size - if (out->uses_.size() == 1UL && out->uses_[0].second->isa()) { - continue; - } - int distance = 1; - for (auto &use : out->uses_) { - distance = - std::max(distance, get_op_distance_to_visited_set( - use.second.get(), distance_to_visited)); - } - float cur_tsr = - (distance - 1) * distance_factor + - (is_dynamic_ ? 1.f - : float(out->details_.get_blocking_byte_size()) / - max_tensor_size_); - cur_score -= cur_tsr; - } - return cur_score; - } - - using queue_iterator_t = std::list::iterator; - op_visitor_t::selector_func get_selector() { - auto ths = this; - return [ths](op_visitor_t *vis) -> sc_op_ptr { - if (ths->cur_tick_ == 0) { - ths->input_op_itr = vis->to_visit_.end(); - --ths->input_op_itr; - } - if (ths->input_op_itr != vis->to_visit_.end()) { - // if there is input ops, return and advance the input_op_itr - auto ret = *ths->input_op_itr; - auto to_remove = ths->input_op_itr; - if (ths->input_op_itr == vis->to_visit_.begin()) { - ths->input_op_itr = vis->to_visit_.end(); - } else { - --ths->input_op_itr; - } - vis->to_visit_.erase(to_remove); - - SC_MODULE_INFO << "Scheduling const/input: iter " << ths->cur_tick_ - << ", Op " << ret->op_name_ << "_" - << ret->logical_op_id_; - return ret; - } - // fast path: if there is only one op, just pop it - if (vis->to_visit_.size() == 1) { - auto ret = vis->to_visit_.back(); - vis->to_visit_.pop_back(); - return ret; - } - float best_score = std::numeric_limits::lowest(); - std::list::reverse_iterator to_remove; - - std::vector distance(ths->op_visited_.size()); - // visit the queue in reversed order to align to old lowering input - // argument order (like pop_back_selector) - for (auto itr = vis->to_visit_.rbegin(); itr != vis->to_visit_.rend(); - ++itr) { - auto &op = *itr; - assert(!op->isa() && !op->isa()); - float cur_score = ths->evaluate_op_score(op.get(), distance); - SC_MODULE_INFO << "Scheduling score: iter " << ths->cur_tick_ << ", Op " - << op->op_name_ << "_" << op->logical_op_id_ << " = " - << cur_score; - if (cur_score > best_score) { - best_score = cur_score; - to_remove = itr; - } - } - auto ret = *to_remove; - SC_MODULE_INFO << "Scheduling selects: iter " << ths->cur_tick_ << ", Op " - << ret->op_name_ << "_" << ret->logical_op_id_; - vis->to_visit_.erase(std::next(to_remove).base()); - return ret; - }; - } -}; - -namespace graph { -std::string get_tensor_name(graph_tensor *t, sc_op *linked_output) { - std::string tensor_name; - if (t->producer_owner_->get_outputs().size() == 1UL) { - tensor_name = - t->producer_owner_->attrs_.get_or_else("temp.name", tensor_name); - } - if (tensor_name.empty() && linked_output && - linked_output->get_inputs().size() == 1UL) { - tensor_name = linked_output->attrs_.get_or_else("temp.name", tensor_name); - } - for (auto &ch : tensor_name) { - if (ch == '*') { - ch = '_'; - } - } - return tensor_name; -} -} // namespace graph - -class tv_tsr_replacer_t : public ir_copier_impl_t { -public: - using ir_copier_impl_t::dispatch; - using ir_copier_impl_t::view; - tv_tsr_replacer_t(std::unordered_map &replace_map, - bool create_var_tensor = false) - : ir_copier_impl_t(replace_map, create_var_tensor) {} - void view(define_c v) override { - if (replace_map_.find(v->var_) != replace_map_.end()) { - returned_stmt_ = builder::make_stmts_unattached({}); - } else { - ir_copier_impl_t::view(v); - } - } -}; - -static void add_def_comments(const stmt &def_node, graph_tensor *t) { - std::stringstream ss; - t->details_.to_string(ss); - if (auto old_comments = - def_node->attr().get_or_null>("comments")) { - old_comments->emplace_back(ss.str()); - } else { - def_node->attr()["comments"] = std::vector{ss.str()}; - } -} - -enum op_kinds : int { - kother = 0, - kinput, - koutput, - kconstant, - kreorder, - kreshape, -}; - -struct general_lower_params_t { - ir_module_ptr ret_mod; - std::unordered_map <sr_rtsr; - sc_graph_t &graph; - stmts func_body; - stmts init_body; - int &tensor_counter; - int &global_tensor_counter; - bool is_graph_dynamic; - // the number of lazy initialzied constant tensors in shared const cache - size_t num_lazy_init_shared_const_tsr_; - // the number of compile-time initialzied constant tensors in shared const - // cache - size_t num_inited_shared_const_tsr_; - std::unordered_set external_dyn_vars; - std::vector &shared_consts; -}; - -expr get_or_create_tensor(general_lower_params_t &gp, const graph_tensor_ptr &t, - bool is_arg, int const_type, - info_etype_t type = info_etype_t::real_tensor) { - bool tsr_is_dynamic = t->details_.is_dynamic(); - sc_op *linked_output = nullptr; - if (!is_arg) { - for (auto &use : t->uses_) { - // finds if any of the use of the tensor is marked output - if (use.second->isa()) { - is_arg = true; - linked_output = use.second.get(); - break; - } - } - } - if (is_arg || const_type != const_kind::not_const) { - // input/output and const tsr don't need placeholder - if (gp.is_graph_dynamic) { - COMPILE_ASSERT(t->details_.get_format_candidates().size() <= 1, - "Input/output/constant tsr should have only empty or " - "one format candidate"); - } - if (type == info_etype_t::placeholder) { - type = info_etype_t::real_tensor; - } - } - auto itr = gp.ltsr_rtsr.find(t); - if (itr == gp.ltsr_rtsr.end()) { - gp.ltsr_rtsr[t] = tsr_info_t(); - itr = gp.ltsr_rtsr.find(t); - itr->second.count_ = gp.tensor_counter++; - } else { - if (type == info_etype_t::real_tensor && itr->second.tensor_.defined()) { - if (gp.graph.is_dynamic()) { - itr->second.tensor_->attr().set(attr_keys::always_trans, true); - } - return itr->second.tensor_; - } - if (type == info_etype_t::placeholder && - itr->second.placeholder_.defined()) { - return itr->second.placeholder_; - } - if (type == info_etype_t::format && itr->second.format_.defined()) { - return itr->second.format_; - } - if (type == info_etype_t::out_size && itr->second.size_.defined()) { - return itr->second.size_; - } - } - - std::vector dims, strides; - sc_data_type_t tsr_dtype; - expr tsr; - - std::string tensor_name = graph::get_tensor_name(t.get(), linked_output); - if (tensor_name.empty()) { - tensor_name = std::string("buffer_") + std::to_string(itr->second.count_); - } - if (type == info_etype_t::real_tensor) { - bool is_size_dynamic = - !is_arg && gp.is_graph_dynamic && - (t->details_.get_format_candidates().size() > 1 || - can_op_query_output(t->producer_owner_->shared_from_this())); - expr dyn_tsr_size; - if (is_size_dynamic) { - assert(itr->second.size_.defined()); - dyn_tsr_size = builder::make_indexing(itr->second.size_, {0}); - dyn_tsr_size->attr().set(attr_keys::no_index2var, true); - } - - dims = is_size_dynamic ? std::vector{dyn_tsr_size} - : t->details_.get_blocking_dims_expr(gp.graph); - strides = is_size_dynamic ? std::vector{UINT64_C(1)} - : t->details_.get_strides_expr(gp.graph); - tsr_dtype = t->details_.dtype_; - tsr = builder::make_stensor(tensor_name, dims, strides, tsr_dtype); - tsr->attr()[attr_keys::plain_dims] = - gp.graph.dims_to_expr(t->details_.get_plain_dims()); - if (itr->second.placeholder_.defined()) { - // for dynamic tensor transform - tsr->attr()["temp.dyn_placeholder"] = itr->second.placeholder_; - } - itr->second.tensor_ = tsr; - if (is_arg || const_type != const_kind::not_const) { - itr->second.placeholder_ = tsr; - } - if (gp.graph.is_dynamic()) { - tsr->attr().set(attr_keys::always_trans, true); - } - // this tensor is an input to a standalone concat op - if (t->attrs_.has_key(concat_optim_attr_keys::graph_memory_offset)) { - tsr->attr()[concat_optim_attr_keys::pass_memory_offset] = - t->attrs_.get>( - concat_optim_attr_keys::graph_memory_offset); - } - } else if (type == info_etype_t::placeholder) { - if (itr->second.tensor_.defined()) { - // first check if the real tensor exist - tsr = itr->second.tensor_; - itr->second.placeholder_ = tsr; - tsr->attr().set(attr_keys::always_trans, true); - } else { - tensor_name += "_placeholder"; - dims = std::vector{sizeof(runtime::dynamic_tensor_t)}; - tsr_dtype = datatypes::u8; - tsr = builder::make_tensor(tensor_name, dims, tsr_dtype); - itr->second.placeholder_ = tsr; - } - } else if (type == info_etype_t::format) { - tensor_name += "_format"; - tsr = builder::make_tensor(tensor_name, {UINT64_C(1)}, datatypes::index); - itr->second.format_ = tsr; - } else { - assert(type == info_etype_t::out_size); - tensor_name += "_size"; - tsr = builder::make_tensor(tensor_name, {UINT64_C(1)}, datatypes::index); - itr->second.size_ = tsr; - } - if (type == info_etype_t ::real_tensor) { - stmt def_node; - if (!is_arg) { - if (const_type != const_kind::not_const) { - if (const_type == const_kind::global_const) { - std::string folded_name; - auto ownerop = t->producer_owner_; - std::shared_ptr cached; - if (auto cache = - ownerop->attrs_.get_or_null< - std::vector>>( - op_attr_key::const_input_cache)) { - auto idx = std::find(ownerop->get_outputs().begin(), - ownerop->get_outputs().end(), t) - - ownerop->get_outputs().begin(); - cached = cache->at(idx); - if (cached->buf_base_->is_lazy_) { - gp.num_lazy_init_shared_const_tsr_++; - } else { - gp.num_inited_shared_const_tsr_++; - } - std::stringstream ss; - ss << "shared_const_" << gp.global_tensor_counter++; - folded_name = ss.str(); - tsr.static_as()->name_ = folded_name; - } else { - folded_name = - "folded_const_" + std::to_string(gp.global_tensor_counter++); - tsr = copy_attr(*tsr, - gp.ret_mod->make_global_stensor( - tsr.checked_as()->elem_dtype_, - folded_name, tsr.checked_as()->dims_, - tsr.checked_as()->strides_, - linkage::private_global, &def_node)); - } - if (cached) { - tsr->attr()[attr_keys::shared_const] = cached; - // const cached tensors are lowered to "local tensors" - // with special marks - def_node = builder::make_var_tensor_def_unattached(tsr); - def_node->attr()["comments"] = - std::vector{"The tensor is cached in global " - "constant cache"}; - // they are not real local tensors, don't schedule - // buffers - def_node->attr()[attr_keys::tsr_dont_buf_sched] = true; - gp.func_body->seq_.insert(gp.func_body->seq_.begin(), def_node); - gp.shared_consts.emplace_back(tsr); - } - // global tensor does not need cached dynamic var - tsr->attr_->set("temp.dyn_placeholder", expr()); - if (auto const_node = t->producer_owner_->dyn_cast()) { - auto const_value = const_node->get_constant_values(); - tsr.checked_as()->init_value_ = const_value; - } - if (gp.graph.is_dynamic()) { - tsr->attr().set(attr_keys::always_trans, true); - } - itr->second.tensor_ = tsr; - itr->second.placeholder_ = tsr; - } else { - def_node = builder::make_var_tensor_def_unattached(tsr); - gp.init_body->seq_.emplace_back(def_node); - } - } else { - def_node = builder::make_var_tensor_def_unattached(tsr); - gp.func_body->seq_.emplace_back(def_node); - } - } - if (def_node.defined()) { - add_def_comments(def_node, t.get()); - } - } else if (type == info_etype_t::placeholder) { - // placeholder - // if use tensor as plhd, do nothing. - if (!itr->second.tensor_.defined()) { - gp.func_body->seq_.emplace_back( - builder::make_var_tensor_def_unattached(tsr)); - std::string name; - if (tsr.isa()) { - name = tsr.checked_as()->name_; - } else { - assert(tsr.isa()); - name = tsr.checked_as() - ->base_->ptr_.checked_as() - ->name_ + - "_tptr"; - } - auto shape_tsr = builder::make_tensor( - std::string("dyn_shape_") + tsr.checked_as()->name_, - {t->details_.get_plain_dims().size()}, datatypes::index); - shape_tsr->attr().set(attr_keys::no_dead_write, true); - shape_tsr->attr().set(attr_keys::no_tensor2var, true); - tsr->attr().set("temp.dyn_shape_of_placeholder", shape_tsr); - gp.func_body->seq_.emplace_back( - builder::make_var_tensor_def_unattached(shape_tsr)); - gp.func_body->seq_.emplace_back(builder::make_evaluate_unattached( - builder::make_write_struct(tsr, shape_tsr, dyn_tsr_struct_t::name, - dyn_tsr_struct_t::fields::dim_ptr))); - gp.func_body->seq_.emplace_back( - builder::make_evaluate_unattached(builder::make_write_struct( - tsr, - builder::make_constant({t->details_.get_plain_dims().size()}, - datatypes::s32), - dyn_tsr_struct_t::name, dyn_tsr_struct_t::fields::ndims))); - uint64_t etype = - t->details_.dtype_.is_etype_pointer() - ? t->details_.dtype_.get_pointer_element().as_etype_int() - : t->details_.dtype_.as_etype_int(); - gp.func_body->seq_.emplace_back( - builder::make_evaluate_unattached(builder::make_write_struct( - tsr, builder::make_constant({etype}, datatypes::u32), - dyn_tsr_struct_t::name, dyn_tsr_struct_t::fields::dtype))); - auto plain_shapes_int = t->details_.get_plain_dims(); - auto plain_shapes = gp.graph.dims_to_expr(plain_shapes_int); - uint64_t dyn_mask_int = 0; - for (size_t i = 0; i < plain_shapes.size(); i++) { - if (!is_dynamic_dim(plain_shapes_int[i]) || - gp.external_dyn_vars.find(plain_shapes_int[i]) != - gp.external_dyn_vars.end()) { - gp.func_body->seq_.emplace_back(builder::make_assign_unattached( - builder::make_indexing(shape_tsr, {i}), plain_shapes[i])); - } - dyn_mask_int |= (uint64_t(!plain_shapes[i].isa()) << i); - } - gp.func_body->seq_.emplace_back( - builder::make_evaluate_unattached(builder::make_write_struct( - tsr, builder::make_constant({dyn_mask_int}, datatypes::u8), - dyn_tsr_struct_t::name, dyn_tsr_struct_t::fields::dyn_mask))); - } - } else if (type == info_etype_t::format) { - // placeholder can be replaced by tensor while format can't - gp.func_body->seq_.emplace_back( - builder::make_var_tensor_def_unattached(tsr)); - uint64_t init_format = 0; - if (t->details_.get_format_candidates().size() <= 1) { - init_format = uint64_t(t->details_.get_format().to_runtime()); - } - gp.func_body->seq_.emplace_back(builder::make_assign_unattached( - builder::make_indexing(tsr, {0}), init_format)); - } else if (type == info_etype_t::out_size) { - if (const_type == const_kind::not_const) { - gp.func_body->seq_.emplace_back( - builder::make_var_tensor_def_unattached(tsr)); - gp.func_body->seq_.back()->attr().set(attr_keys::tsr_dont_buf_sched, - true); - } - } - return tsr; -}; - -expr create_op_query_func(const context_ptr &ctx, general_lower_params_t &gp, - std::vector &op_dispatch_kernel, - const sc_op_ptr &node) { - std::vector plhd_ins, fmt_ins; - std::vector plhd_outs, fmt_outs, size_outs; - bool need_dispatch = can_op_be_queried(node); - // current input - for (auto <ensor : node->get_inputs()) { - auto const_type = ltensor->producer_owner_->attrs_.get_or_else( - "constant", const_kind::not_const); - plhd_ins.emplace_back(get_or_create_tensor(gp, ltensor, false, const_type, - info_etype_t::placeholder)); - fmt_ins.emplace_back(get_or_create_tensor(gp, ltensor, false, const_type, - info_etype_t::format)); - } - // input before reorder - if (node->isa() || (node->isa() && - !node->stc_cast() - ->get_internal_tunable_input_indices() - .empty())) { - auto &inputs = node->get_inputs(); - std::vector query_idxs; - if (node->isa()) { - size_t sz; - sz = inputs.size(); - query_idxs.reserve(sz); - for (size_t i = 0; i < sz; i++) { - query_idxs.emplace_back(i); - } - } else if (node->isa()) { - query_idxs = node->stc_cast() - ->get_internal_tunable_input_indices(); - } - for (auto i : query_idxs) { - auto ltensor = node->get_inputs()[i]; - auto node_before = ltensor->producer_owner_; - auto const_type_before = - node_before->attrs_.get_or_else("constant", const_kind::not_const); - // find the buffer before reorder. - if (node_before->isa() && - (node_before->attrs_.as_map().empty() || - node_before->attrs_.get_or_else("constant", const_kind::not_const) == - const_kind::not_const)) { - ltensor = node_before->get_inputs()[0]; - } - plhd_ins.emplace_back(get_or_create_tensor( - gp, ltensor, false, const_type_before, info_etype_t::placeholder)); - fmt_ins.emplace_back(get_or_create_tensor( - gp, ltensor, false, const_type_before, info_etype_t::format)); - } - } - auto const_type = node->attrs_.get_or_else("constant", const_kind::not_const); - for (auto <ensor : node->get_outputs()) { - expr plhd, fmt, size; - if (node->isa()) { - // use real tensor instead of placeholder. - plhd = get_or_create_tensor(gp, ltensor, true, const_type, - info_etype_t::real_tensor); - fmt = get_or_create_tensor(gp, ltensor, true, const_type, - info_etype_t::format); - } else if (node->isa()) { - plhd = get_or_create_tensor(gp, ltensor, false, const_kind::global_const, - info_etype_t::real_tensor); - fmt = get_or_create_tensor(gp, ltensor, false, const_kind::global_const, - info_etype_t::format); - } else { - plhd = get_or_create_tensor(gp, ltensor, false, const_type, - info_etype_t::placeholder); - // expect for output tsr - if (!plhd.defined()) { - plhd = get_or_create_tensor(gp, ltensor, false, const_type, - info_etype_t::real_tensor); - } - fmt = get_or_create_tensor(gp, ltensor, false, const_type, - info_etype_t::format); - size = get_or_create_tensor(gp, ltensor, false, const_type, - info_etype_t::out_size); - } - plhd_outs.emplace_back(plhd); - fmt_outs.emplace_back(fmt); - size_outs.emplace_back(size); - } - // Pruning, because the format propagation is broken after reorder, - // so it doesn't need query to deliver formats. Notes that only - // reorder could, other ops should propagate their format even does - // not need dispatch. - if ((node->isa() && !need_dispatch) || - const_type != const_kind::not_const) { - return expr(); - } - expr dyn_ker_ptr; - // update dynamic query format - if (!op_dispatch_kernel[node->logical_op_id_].defined()) { - auto &table_map = gp.ret_mod->get_op_table_map(); - auto func_name = - node->op_name_ + "__" + std::to_string(node->logical_op_id_) + "_ptr"; - auto table_name = func_name + "_table"; - auto table_it = table_map.find(table_name); - auto table_var = builder::make_var(datatypes::pointer, table_name); - auto table_ptr = table_it != table_map.end() - ? table_it->second - : std::make_shared(); - int internal_func_num = get_num_of_internal_funcs(node); - // kernel pointer vector the first is outer function. - dyn_ker_ptr = builder::make_tensor( - func_name, {static_cast(1 + internal_func_num)}, - datatypes::index); - std::vector query_func_args; - query_func_args.emplace_back(table_var); - query_func_args.insert(query_func_args.end(), plhd_outs.begin(), - plhd_outs.end()); - query_func_args.insert(query_func_args.end(), plhd_ins.begin(), - plhd_ins.end()); - query_func_args.insert(query_func_args.end(), fmt_outs.begin(), - fmt_outs.end()); - query_func_args.insert(query_func_args.end(), fmt_ins.begin(), - fmt_ins.end()); - query_func_args.insert(query_func_args.end(), size_outs.begin(), - size_outs.end()); - query_func_args.push_back(dyn_ker_ptr); - expr query_call; // call node - if (node->isa()) { - auto fused_node = node->stc_cast(); - auto query_mod = fused_node->get_dynamic_query_func(ctx); - gp.ret_mod->merge(*query_mod); - assert(table_ptr); - query_call = - builder::make_call(query_mod->get_entry_func(), query_func_args); - } else { - auto table_ptr = std::make_shared(); - gp.ret_mod->add_op_table(std::make_pair(table_name, table_ptr)); - initialize_dispatch_table_with_op(ctx, node, table_ptr); - query_call = call_op_dynamic_query_function(node, query_func_args); - } - stmts_node_t *target_body = gp.func_body.get(); - if (table_it == table_map.end()) { - auto table_def = builder::make_var_tensor_def_unattached( - table_var, linkage::private_global); - gp.ret_mod->add_global_var(table_def.checked_as()); - } - target_body->seq_.emplace_back( - builder::make_var_tensor_def_unattached(dyn_ker_ptr)); - target_body->seq_.emplace_back( - builder::make_evaluate_unattached(query_call)); - op_dispatch_kernel[node->logical_op_id_] = builder::make_reinterpret( - builder::make_indexing(dyn_ker_ptr, 0), datatypes::pointer); - op_dispatch_kernel[node->logical_op_id_]->attr().set( - attr_keys::no_index2var, true); - } - return dyn_ker_ptr; -} - -std::pair get_reshape_tptr(general_lower_params_t &gp, - const graph_tensor_ptr &old_tsr, - const graph_tensor_ptr &new_tsr, - int const_type, op_kinds kind) { - auto base_tsr = get_or_create_tensor(gp, old_tsr, kind == kinput, const_type); - size_t ndims; - if (base_tsr.isa()) { - ndims = base_tsr.static_as()->shape_.size(); - } else { - assert(base_tsr.isa()); - ndims = base_tsr.static_as()->dims_.size(); - } - std::vector base_idx(ndims, expr(0)); - std::vector new_shape_tmp = - new_tsr->details_.get_blocking_dims_expr(gp.graph); - std::vector new_shape(new_shape_tmp.begin(), new_shape_tmp.end()); - auto new_tptr = builder::tensor_ptr(base_tsr, base_idx, new_shape); - new_tptr->attr().set( - attr_keys::plain_dims, - gp.graph.dims_to_expr(new_tsr->details_.get_plain_dims())); - return std::make_pair(base_tsr, new_tptr); -} - -void create_op_tensors(general_lower_params_t &gp, std::vector &ins, - std::vector &outs, const sc_op_ptr &node, - op_kinds kind) { - int const_type = node->attrs_.get_or_else("constant", const_kind::not_const); - for (auto <ensor : node->get_inputs()) { - // As the traversal is not in order, so the constant type of - // tensor should be decided by the node before. - ins.emplace_back(get_or_create_tensor(gp, ltensor, false, const_type)); - } - for (auto <ensor : node->get_outputs()) { - if (kind == kconstant) { - get_or_create_tensor(gp, ltensor, false, const_kind::global_const); - } else if (kind == kreshape) { - COMPILE_ASSERT(node->get_inputs().size() == 1 || - node->get_inputs().size() == 2, - "Reshape should have 1 or 2(dynamic_reshape) input"); - // If the output of tensor view is output of graph - if (gp.ltsr_rtsr.find(ltensor) != gp.ltsr_rtsr.end() && - has_output_uses(ltensor)) { - break; - } - auto out_tsr_pair = get_reshape_tptr(gp, node->get_inputs()[0], ltensor, - const_type, kind); - auto it = gp.ltsr_rtsr.find(ltensor); - if (it != gp.ltsr_rtsr.end() && it->second.tensor_.defined()) { - COMPILE_ASSERT(gp.is_graph_dynamic, - "If output tsr of tensor view is defined, it " - "should in dynamic mode."); - // the tsr replace map for tensor view op. Because in - // dynamic mode, the output of tensor view may be - // traversed first. - std::unordered_map tv_replace_map; - tv_replace_map.insert( - std::make_pair(it->second.tensor_, out_tsr_pair.second)); - tv_tsr_replacer_t cpy(tv_replace_map, false); - gp.func_body = - cpy.dispatch(gp.func_body).remove_const().checked_as(); - gp.init_body = - cpy.dispatch(gp.init_body).remove_const().checked_as(); - } - gp.ltsr_rtsr[ltensor].tensor_ = out_tsr_pair.second; - } else { - graph_tensor_ptr out_tsr; - // for pattern like node->reshape->output, node != input - if (auto out_tsr = get_linked_output_tsr(ltensor)) { - gp.ltsr_rtsr[ltensor].tensor_ = - get_reshape_tptr(gp, out_tsr, ltensor, const_type, kind).second; - outs.emplace_back(gp.ltsr_rtsr[ltensor].tensor_); - } else { - outs.emplace_back( - get_or_create_tensor(gp, ltensor, kind == kinput, const_type)); - } - } - } -} - -static std::string get_dispatch_callee_name(const expr &kernel) { - assert(kernel.isa()); - return kernel.checked_as()->ptr_.checked_as()->name_; -} - -static void -dynamic_reshape_var_assignment(general_lower_params_t &gp, - std::unordered_set &dynamic_var_set, - const sc_op_ptr &node) { - auto &graph = gp.graph; - auto out_plain_dims = node->get_outputs()[0]->details_.get_plain_dims(); - auto shape_tsr = gp.ltsr_rtsr[node->get_inputs()[1]].tensor_; - for (size_t i = 0; i < out_plain_dims.size(); i++) { - auto &dyn_dim = out_plain_dims[i]; - // may be inferred by dynamic shape binding. - if (!is_dynamic_dim(dyn_dim)) { - continue; - } - auto var = graph.dim_to_expr(dyn_dim); - if (dynamic_var_set.find(var) == dynamic_var_set.end()) { - gp.func_body->seq_.emplace_back(builder::make_var_tensor_def_unattached( - graph.dim_to_expr(dyn_dim), linkage::local, - builder::make_indexing(shape_tsr, {i}))); - dynamic_var_set.insert(var); - } - } -} -static func_t -insert_prefetch(const context_ptr &ctx, - const std::vector> &op_execution_log, - sc_op *node, const std::vector &input_tsr, - ir_module_t &mod, std::vector &outbody) { - if (!ctx->flags_.prefetch_) { - return func_t(); - } - if (op_execution_log.empty()) { - return func_t(); - } - auto prefetch_op = node->dyn_cast(); - if (!prefetch_op) { - return func_t(); - } - std::vector ins; - ins.reserve(input_tsr.size()); - for (auto &in : input_tsr) { - ins.emplace_back(in); - } - auto can_prefetch_index = prefetch_op->query_prefetch(ctx, true, ins); - if (can_prefetch_index.empty()) { - return func_t(); - } - // now find the position where we insert prefetch - size_t pos = std::string::npos; - // how many Ops are executed between "pos" and current Op - size_t pos_diff = 0; - for (int64_t i = op_execution_log.size() - 1; i >= 0; i--) { - if (op_execution_log[i].second.defined()) { - // if the op is executed in main body (not in cached const/input) - auto gflop = op_execution_log[i].first->get_gflop(); - // todo(yijie): currently gflop>0 means the op has a complex op like - // matmul so we only need to compare gflop with 0. We should later - // set a reasonable threshold - bool is_good_for_insert = gflop > 0; - if (is_good_for_insert) { - auto func = op_execution_log[i] - .second.checked_as() - ->value_.checked_as() - ->func_; - if (func->attr_ && - func->attr_->get_or_else(function_attrs::has_idle_func, false)) { - // if we want to insert to a position that is already doing - // prefetch, skip - return func_t(); - } - pos = i; - break; - } - pos_diff++; - } - } - if (pos == std::string::npos || pos_diff > 3) { - // if there is no large ops, or the position is too far away from the - // current op, don't insert prefetch - return func_t(); - } - - std::vector filtered_prefetch_index; - auto cache_thres = ctx->machine_.cpu_flags_.getDCacheSize(2) * - runtime_config_t::get().get_num_threads(); - // we need to check at this point (at "pos"), which input of the Op ("node") - // is ready to prefetch - for (auto in_idx : can_prefetch_index) { - auto &tsr = node->get_inputs().at(in_idx); - // todo (yijie): add more detailed cache size management - // skip prefetch on this tensor if it is too large - if (tsr->details_.get_blocking_byte_size() * 2 > cache_thres) { - continue; - } - auto producer_op = tsr->producer_owner_; - size_t producer_idx = op_execution_log.size(); - for (size_t i = 0; i < op_execution_log.size(); i++) { - if (op_execution_log[i].first.get() == producer_op) { - if (op_execution_log[i].second.defined()) { - producer_idx = i; - } else { - // if the producer op is in cached const/input, it is always - // ready to be prefetched - producer_idx = 0; - } - break; - } - } - if (producer_idx <= pos) { - if (!input_tsr.at(in_idx).isa()) { - SC_MODULE_WARN << "Cannot prefetch tensor_view" << input_tsr.at(in_idx); - } else { - // if the input is already computed before "pos" - filtered_prefetch_index.push_back(in_idx); - } - } - } - if (filtered_prefetch_index.empty()) { - return func_t(); - } - std::vector out; - auto ret = prefetch_op->generate_prefetcher_and_set_idle( - ctx, true, ins, filtered_prefetch_index, out); - // insert the thread pool manipulation code before pos - bool found = false; - for (auto itr = outbody.begin(); itr != outbody.end(); ++itr) { - if (itr->ptr_same(op_execution_log[pos].second)) { - for (auto &set_idle_body : out) { - itr = outbody.insert(itr, set_idle_body); - } - auto func = op_execution_log[pos] - .second.checked_as() - ->value_.checked_as() - ->func_; - func->attr()[function_attrs::has_idle_func] = true; - found = true; - break; - } - } - assert(found); - SC_UNUSED(found); - mod.add_func({ret}); - return ret; -} - -static void add_func_doc_string(const func_t &f) { - std::vector comments; - comments.reserve(f->params_.size() + 1); - comments.emplace_back(f->name_); - for (auto &p : f->params_) { - std::stringstream ss; - ss << "@param " << p << ' ' << p->attr().get("temp.comments"); - comments.emplace_back(ss.str()); - p->attr().remove("temp.comments"); - } - f->attr()["comments"] = std::move(comments); -} - -// check if an op can be marked no_post_barrier -stmt_base_t *has_no_dep_with_prev( - const sc_op_ptr &node, const op_dep_matrix_t &dep_mat, - const std::vector> &op_execution_log) { - // mark no_post_barrier on eveluate-call - if (op_execution_log.empty()) { - return nullptr; - } - auto itr = op_execution_log.rbegin(); - for (; itr != op_execution_log.rend(); ++itr) { - if (!itr->second.defined()) - continue; - if (dep_mat.lookup(node, itr->first) != 0) { - return nullptr; - } - break; - } - if (itr == op_execution_log.rend()) { - return nullptr; - } - // itr now points to the previously executed op - auto ret = itr->second.get(); - // search from the previous executed op to find if the current op depends on - // any of the op in the no_post_barrier section - for (itr = itr + 1; itr != op_execution_log.rend(); ++itr) { - if (!itr->second.defined()) - continue; - // if the prev op is already marked no_dep, need to continue to check to - // ensure all consequent ops with no_dep does not depend on each other - if (itr->second->attr_ && - itr->second->attr_->has_key(attr_keys::no_post_barrier)) { - // if current op does not depend on the previous op - if (dep_mat.lookup(node, itr->first) == 0) { - continue; - } else { - return nullptr; - } - } else { - return ret; - } - } - return ret; -} - -ir_module_ptr lower_graph(context_ptr ctx, sc_graph_t &graph, - const std::vector &args, - bool mark_as_main) { - auto timer = SC_SCOPED_TIMER_INFO("graph.driver.time.lowering", ""); - lowering_visitor_state_t visiter_state(graph); - op_visitor_t vis{visiter_state.get_selector(), visiter_state.get_updater(), - true}; - visiter_state.input_op_itr = vis.to_visit_.end(); - std::vector params; - stmts func_body = make_stmt(std::vector()); - stmts init_body = make_stmt(std::vector()); - constexpr const char *default_graph_name = "main_entry"; - auto graph_name = - graph.attrs_.get_or_else("temp.name", default_graph_name); - // todo: use graph-id to generate name - auto func = - builder::make_func(graph_name, params, func_body, datatypes::void_t); - // todo: logical tensor should also have an unique id - // tsr_info_t include dynamic placeholder(dynamic tensor with empty - // datapointer) and runtime format. - std::unordered_map ltsr_rtsr; - // external dyn vars set. - std::unordered_set external_dyn_vars = - graph.get_external_dynamic_vars(); - // function pointer - std::vector op_dispatch_kernel(graph.ops_.size()); - int tensor_counter = 0; - int global_tensor_counter = 0; - auto ret_mod = ir_module_t::from_entry_func(ctx, func); - auto use_managed_tp = - std::make_shared(thread_pool_mode_t::DIRECT); - - expr dump_out_path; - if (graph.attrs_.get_or_else("folded_input", false)) { - ret_mod->attr_.set("folded_input", true); - } - bool is_graph_dynamic = graph.is_dynamic() && - !graph.attrs_.get_or_else("temp.force_static", false); - // the shared constant tensors. They need to be passed to the init_globals() - // func - std::vector shared_consts; - general_lower_params_t gp{ret_mod, - ltsr_rtsr, - graph, - func_body, - init_body, - tensor_counter, - global_tensor_counter, - is_graph_dynamic, - /*num_lazy_init_const_tsr*/ 0, - /*num_inited_shared_const_tsr_*/ 0, - external_dyn_vars, - shared_consts}; - // the set of dynamic var defined in func body.(dynamic reshape) - std::unordered_set dyn_var_set; - // record the node, index is op id. - std::vector query_visited(graph.ops_.size(), false); - op_dep_matrix_t dep_mat{graph}; - std::vector> op_execution_log; - int num_ops = 0; - // internal function related - bool need_extra_internal_func_arg = - !mark_as_main && graph.need_dynamic_internal_query(); - int total_num_of_internal_funcs = get_num_of_internal_funcs(graph); - expr extra_internal_func_arg = builder::make_tensor( - "extra_internal_funcs", {total_num_of_internal_funcs}, datatypes::index); - extra_internal_func_arg->attr()["temp.comments"] = - std::string("Extra tensor arg contains pointer to internal funcs"); - int cur_internal_idx = 0; - - vis.visit_graph(graph, [&](op_visitor_t *vis, const sc_op_ptr &node) { - std::vector ins, outs; - // special kinds of Ops that we need to take care of - op_kinds kind = kother; - if (node->isa()) { - kind = kinput; - } else if (node->isa()) { - kind = koutput; - } else if (node->isa()) { - kind = kconstant; - if (node->attrs_.get_or_else("constant", const_kind::not_const) == - const_kind::not_const) { - node->attrs_.set("constant", const_kind::global_const); - } - } else if (node->isa()) { - // todo: assume reorder is fused break in dynamic now. - kind = kreorder; - } else if (node->isa() || - node->isa()) { - kind = kreshape; - } - - // if the node is reorder or has tail reorder in its internal graph, - // query its uses op first. - if (is_graph_dynamic && can_op_be_queried(node)) { - auto create_op_query_func_wrapper = - std::bind(create_op_query_func, ctx, std::ref(gp), - std::ref(op_dispatch_kernel), std::placeholders::_1); - lower_query_function(query_visited, node, create_op_query_func_wrapper); - } - // tensor decl should put after query functions. - create_op_tensors(gp, ins, outs, node, kind); - if (is_graph_dynamic && kind == kreorder && - node->attrs_.get_or_else("constant", const_kind::not_const) == - const_kind::not_const) { - outs[0]->attr().set("temp.may_inplace", true); - } - int const_type = - node->attrs_.get_or_else("constant", const_kind::not_const); - bool executed_in_main_body = false; - switch (kind) { - case kinput: { - auto &op_outs = node->get_outputs(); - for (size_t i = 0; i < outs.size(); i++) { - auto &v = outs[i]; - params.emplace_back(v); - std::stringstream ss; - op_outs[i]->details_.to_string(ss); - v->attr()["temp.comments"] = ss.str(); - } - break; - } - case koutput: { - auto &op_ins = node->get_inputs(); - for (size_t i = 0; i < ins.size(); i++) { - auto &v = ins[i]; - params.emplace_back(v); - std::stringstream ss; - op_ins[i]->details_.to_string(ss); - v->attr()["temp.comments"] = ss.str(); - } - break; - } - case kconstant: - case kreshape: { - if (node->isa()) { - dynamic_reshape_var_assignment(gp, dyn_var_set, node); - } - break; - // nothing to do. - } - default: { - std::vector exprargs; - exprargs.insert(exprargs.end(), outs.begin(), outs.end()); - exprargs.insert(exprargs.end(), ins.begin(), ins.end()); - expr kernel_call; - std::string callee_name; - if (is_graph_dynamic && can_op_be_dispatched(node)) { - auto &base_kernel = op_dispatch_kernel[node->logical_op_id_] - .checked_as() - ->args_[0]; - assert(is_graph_dynamic); - assert(base_kernel.defined()); - callee_name = get_dispatch_callee_name(base_kernel); - std::string table_name = callee_name + "_table"; - int dyn_idx = 0; - - node->get_dispatch_key_set()->for_each_key_process( - std::bind(create_dispatch_funcs_by_keys, ctx, std::ref(ret_mod), - table_name, node, std::placeholders::_1, - std::ref(op_dispatch_kernel[node->logical_op_id_]), - std::ref(dyn_idx), use_managed_tp, - /*internal*/ false)); - if (node->need_dynamic_internal_query()) { - node->info_.internal_info_->parti_in_ltsrs_ = - graph::extract_detail_from_tensors(node->get_inputs()); - node->info_.internal_info_->parti_out_ltsrs_ = - graph::extract_detail_from_tensors(node->get_outputs()); - create_internal_dispatch_funcs_by_node(ctx, ret_mod, table_name, node, - use_managed_tp); - assert(base_kernel.isa()); - auto ker_tsr = base_kernel.static_as()->ptr_; - // index 0 is the outer dispatch kernel, elements from - // index 1 are the internal dispatch kernels. - exprargs.emplace_back(builder::tensor_ptr(ker_tsr, {1})); - } - kernel_call = make_expr( - op_dispatch_kernel[node->logical_op_id_], exprargs); - } else { - // no dispatch - auto mod = node->get_func(ctx); - auto inp_node = node->dyn_cast(); - if (inp_node && ctx->flags_.tensor_inplace_) { - auto inp_hint = inp_node->get_inplace_map(); - if (!inp_hint.empty()) { - auto func = mod->get_entry_func(); - std::vector>> - out_hint; - for (auto &kv : inp_hint) { - int output_id = kv.first; - out_hint.emplace_back(output_id, - std::vector{}); - for (auto &info : kv.second) { - auto input_id = info.used_arg_idx_; - // convert input id to argument index - int new_idx = input_id + node->get_outputs().size(); - out_hint.back().second.emplace_back( - tensor_inplace_info_t{new_idx, info.kind_}); - } - } - func->attr()[function_attrs::inplace_hint] = std::move(out_hint); - } - } - ret_mod->merge(*mod); - auto callee = mod->get_entry_func(); - if (need_extra_internal_func_arg && - node->need_dynamic_internal_query()) { - int cur_internal_funcs = get_num_of_internal_funcs(node); - exprargs.emplace_back( - builder::tensor_ptr(extra_internal_func_arg, {cur_internal_idx})); - cur_internal_idx += cur_internal_funcs; - } - callee_name = callee->name_; - kernel_call = builder::make_call(callee, exprargs); - if (node->isa()) { - kernel_call->attr()[concat_optim_attr_keys::is_standalone_concat] = - true; - } - if (mark_as_main && const_type == const_kind::not_const) { - insert_prefetch(ctx, op_execution_log, node.get(), ins, *ret_mod, - func_body->seq_); - } - } - stmts_node_t *target_body = (const_type != const_kind::not_const) - ? init_body.get() - : func_body.get(); - target_body->seq_.emplace_back( - builder::make_evaluate_unattached(kernel_call)); - if (const_type == const_kind::not_const) { - executed_in_main_body = true; - if (auto marked_stmt = - has_no_dep_with_prev(node, dep_mat, op_execution_log)) { - marked_stmt->attr()[attr_keys::no_post_barrier] = callee_name; - } - op_execution_log.emplace_back(node, target_body->seq_.back()); - num_ops++; - } - } - } - if (!executed_in_main_body) { - op_execution_log.emplace_back(node, stmt()); - } - }); - if (!args.empty()) { - std::vector new_param; - for (auto &v : args) { - if (auto inop = v->dyn_cast()) { - for (auto &in : inop->get_outputs()) { - auto itr = ltsr_rtsr.find(in); - COMPILE_ASSERT(itr != ltsr_rtsr.end(), - "Cannot find the input op in the generated " - "function"); - itr->second.tensor_->attr()["read_buffer"] = true; - new_param.emplace_back(itr->second.tensor_); - } - } else if (auto outop = v->dyn_cast()) { - for (auto &out : outop->get_inputs()) { - auto itr = ltsr_rtsr.find(out); - COMPILE_ASSERT(itr != ltsr_rtsr.end(), - "Cannot find the output op in the generated " - "function"); - itr->second.tensor_->attr()["write_buffer"] = true; - new_param.emplace_back(itr->second.tensor_); - } - } else { - COMPILE_ASSERT(false, - "The Op given in the args is not input or output"); - } - } - COMPILE_ASSERT(new_param.size() == params.size(), - "The args count does not match the count of in/out " - "tensors, new_param.size=" - << new_param.size() << ", param.size()=" << params.size() - << "."); - params = std::move(new_param); - } - if (need_extra_internal_func_arg) { - assert(cur_internal_idx == total_num_of_internal_funcs); - params.emplace_back(extra_internal_func_arg); - } - if (!init_body->seq_.empty()) { - expr is_init_var; - const size_t share_const_is_init_num_stmts = 2; - std::vector to_insert; - if (gp.num_lazy_init_shared_const_tsr_ == 0) { - // if there are no lazy inited buffer in shared const - is_init_var = ret_mod->make_global_var( - datatypes::boolean, "is_init", linkage::private_global, - graph.attrs_.get_or_else("folded_input", false)); - init_body->seq_.emplace_back( - builder::make_assign_unattached(is_init_var, true)); - } else { - // need special __is_init local tensor for lazy inited buffer - auto is_init_tensor = - builder::make_tensor("__is_init", {1}, datatypes::s32); - is_init_tensor->attr()[attr_keys::is_init_for_const_cache] = true; - is_init_tensor->attr()[attr_keys::no_index2var] = true; - to_insert.emplace_back( - builder::make_var_tensor_def_unattached(is_init_tensor)); - to_insert.back()->attr()["comments"] = std::vector{ - "the element of it is auto-filled based on states of " - "shared_const tensors"}; - to_insert.back()->attr()[attr_keys::is_shared_const_init_stmt] = true; - to_insert.emplace_back( - builder::make_assign_unattached(is_init_tensor[0], 1)); - to_insert.back()->attr()[attr_keys::is_shared_const_init_stmt] = true; - is_init_var = (is_init_tensor[0] != 0); - assert(share_const_is_init_num_stmts == to_insert.size()); - func_body->seq_.insert(func_body->seq_.begin(), to_insert.begin(), - to_insert.end()); - } - auto params_for_init = params; - params_for_init.insert(params_for_init.end(), shared_consts.begin(), - shared_consts.end()); - func_t init_func = builder::make_func( - "__init_const_globals", params_for_init, init_body, datatypes::void_t); - init_func->attr()[function_attrs::private_] = true; - ret_mod->add_func({init_func}); - stmt const_init = builder::make_if_else_unattached( - builder::make_logic_not(is_init_var), - builder::make_stmts_unattached({builder::make_evaluate_unattached( - builder::make_call(init_func, params_for_init))}), - stmts()); - // insert the stmt "if (!is_init) {__init_const_globals(...);}" - if (gp.num_inited_shared_const_tsr_ + gp.num_lazy_init_shared_const_tsr_ == - 0) { - // if there are no shared const buffers - func_body->seq_.insert(func_body->seq_.begin(), const_init); - } else { - // find the first position in the body after the definition of - // shared consts - for (auto itr = func_body->seq_.begin() + to_insert.size();; itr++) { - if (itr == func_body->seq_.end() || - !(*itr) - .cast() - .map([](const define_c &v) { return v->var_.as(); }) - .filter([](const tensor &v) { - return v->attr_ && - v->attr_->has_key(attr_keys::shared_const); - }) - .has_value()) { - func_body->seq_.insert(itr, const_init); - break; - } - } - } - } - func->params_ = std::move(params); - func->decl_->params_ = func->params_; - func->body_ = std::move(func_body); - func->attr()[function_attrs::top_level] = true; - add_func_doc_string(func); - if (mark_as_main) - func->attr()[function_attrs::is_main] = true; - if (utils::compiler_configs_t::get().print_pass_result_) { - SC_MODULE_INFO << ret_mod; - } - auto gflop = graph.attrs_.get_or_else(sc_graph_t::attr_key_t::gflop, 0.0f); - ret_mod->attr_[ir_module_t::attr_key_t::GFLOP] = gflop; - - // if the workload is too small, directly use thread pool backend instead of - // managed thread pool. if gflop per thread is large enough, or there is - // only one single thread, enable managed thread pool. For MLP workload on - // 24-core cascade lake, 1.6Gflop is turning point of choosing - // managed/native thread pool - auto &rtl_cfg = runtime_config_t::get(); - thread_pool_mode_t use_managed_thread_pool = thread_pool_mode_t::DIRECT; - if (rtl_cfg.managed_thread_pool_ == thread_pool_mode_t::MANAGED) { - if (num_ops > 2 || rtl_cfg.get_num_threads() == 1 || - gflop / rtl_cfg.get_num_threads() > 0.0666f) { - use_managed_thread_pool = thread_pool_mode_t::MANAGED; - } else if (ctx->use_amx()) { - use_managed_thread_pool = thread_pool_mode_t::MANAGED; - } - } else if (rtl_cfg.managed_thread_pool_ == thread_pool_mode_t::DYNAMIC) { - use_managed_thread_pool = thread_pool_mode_t::DYNAMIC; - } - ret_mod->attr_[ir_module_t::attr_key_t::MANAGED_THREAD_POOL] = - use_managed_thread_pool; - *use_managed_tp = use_managed_thread_pool; - if (graph_name != default_graph_name) { - ret_mod->attr_[ir_module_t::attr_key_t::NAME] = graph_name; - } - if (graph.attrs_.has_key("shared_const_bases")) { - ret_mod->attr_[ir_module_t::attr_key_t::SHARED_CONST_BASES] = - graph.attrs_["shared_const_bases"]; - } - - if (ctx->flags_.graph_default_private_) { - for (auto &f : ret_mod->get_contents()) { - f->attr()[function_attrs::private_] = true; - } - - ret_mod->get_entry_func()->attr().remove(function_attrs::private_); - for (auto &table : ret_mod->get_op_table_map()) { - for (auto &kv : table.second->kernel_table_) { - if (kv.second.already_compiled()) { - if (auto f = ret_mod->get_func(kv.second.name_or_postfix_)) { - f->attr().remove(function_attrs::private_); - } - } - } - } - } - return ret_mod; -} -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/lowering.hpp b/legacy/core/src/compiler/ir/graph/lowering.hpp deleted file mode 100644 index 8fb6ee5ab..000000000 --- a/legacy/core/src/compiler/ir/graph/lowering.hpp +++ /dev/null @@ -1,66 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_LOWERING_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_LOWERING_HPP -#include "graph.hpp" -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -namespace graph { -std::string get_tensor_name(graph_tensor *t, sc_op *linked_output); -} - -expr call_op_dynamic_query_function(const sc_op_ptr &op, - const std::vector &args); - -struct tsr_info_t { - expr tensor_; - expr placeholder_; - expr format_; - expr size_; - int count_ = 0; - tsr_info_t() = default; - tsr_info_t(const expr &tensor, const expr &placeholder, const expr &format, - const expr &size) - : tensor_(tensor), placeholder_(placeholder), format_(format), - size_(size) {} -}; - -enum info_etype_t { real_tensor, placeholder, format, out_size }; - -/** - * Generates the ir_module_t from the OP graph - * @param ctx the context - * @param graph the graph - * @param args optional order of the arguments of the generated IR function. - * Should all be input_op or output_op. If empty, use default internal order - * @param mark_as_main mark the main entry function with "is_main" attr - * */ -SC_API ir_module_ptr lower_graph(context_ptr ctx, sc_graph_t &graph, - const std::vector &args, - bool mark_as_main = true); -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/graph/mixed_partition.cpp b/legacy/core/src/compiler/ir/graph/mixed_partition.cpp deleted file mode 100644 index 6e75af34c..000000000 --- a/legacy/core/src/compiler/ir/graph/mixed_partition.cpp +++ /dev/null @@ -1,4199 +0,0 @@ -/* - * Copyright (C) 2025 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions - * and limitations under the License. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "mixed_partition.hpp" -#include "binding_axis.hpp" -#include "pass/pass.hpp" -#include "transform/transform.hpp" -#include "tunable_op.hpp" -#include "utils.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -SC_MODULE(graph.mixed_partition); - -static bool require_fusion_level(const context_ptr &ctx, - fusion_opt_level at_least_lvl) { - return ctx->flags_.fusion_level_ >= at_least_lvl; -} - -void mxp_replacer_t::replace_anchor( - const std::vector &fanchors) { - auto replace_fsmap = [&](const fusion_anchor_ptr &cur) { - for (auto &fs_pair : cur->fsmap_.datamap_) { - for (auto &slice : fs_pair.second) { - for (auto &range : slice) { - range.first = dispatch_impl(range.first); - range.second = dispatch_impl(range.second); - } - } - } - }; - for (auto &anmap : fanchors) { - replace_fsmap(anmap); - } -} - -namespace graph { -void tensor_detail_to_ir_tensor(sc_graph_t &graph, const std::string &name, - const graph_tensor_ptr >, - mxp_buffer_allocator *buf_alloc) { - COMPILE_ASSERT(buf_alloc, "No buffer allocator found") - bool is_cost_model_enabled = buf_alloc->get_binded_mxp()->cost_->is_enabled(); - if (!buf_alloc->g2b_map_.haskey(gt)) { - auto tsr = graph::tensor_detail_to_ir_tensor(graph, name, gt->details_); - buf_alloc->g2b_map_.get(gt) = tsr; - // init b2g map - buf_alloc->b2g_map_[tsr] = gt; - if (!graph.is_dynamic() && is_cost_model_enabled) { - auto dim_prod = - get_dims_product(get_expr_to_dims(tsr.checked_as()->dims_)); - auto dtype_size = utils::get_sizeof_etype( - tsr.checked_as()->elem_dtype_.type_code_); - // first touch - buf_alloc->mem_trace_.emplace_back(memory_optim::memory_alloc_trace_t{ - (uintptr_t)tsr.get(), (size_t)dim_prod * dtype_size}); - // last use - buf_alloc->mem_trace_.emplace_back( - memory_optim::memory_alloc_trace_t{(uintptr_t)tsr.get(), (size_t)0}); - } - } else if (!graph.is_dynamic() && is_cost_model_enabled) { - auto tsr = get_real_tensor(buf_alloc->g2b_map_.get(gt)); - // update last use trace - auto last_trace = std::remove_if( - buf_alloc->mem_trace_.begin(), buf_alloc->mem_trace_.end(), - [&tsr](const memory_optim::memory_alloc_trace_t &trace) { - return ((uintptr_t)tsr.get() == trace.buffer_id_) && - (trace.size_ == 0); - }); - COMPILE_ASSERT((last_trace + 1) == buf_alloc->mem_trace_.end(), - "Not found last use trace for: " << tsr); - (*last_trace) = - memory_optim::memory_alloc_trace_t{(uintptr_t)tsr.get(), (size_t)0}; - } -} - -void tensor_detail_to_ir_tensor(sc_graph_t &graph, - const std::string &name_prefix, - const std::vector &tsrs, - mxp_buffer_allocator *buf_alloc) { - COMPILE_ASSERT(buf_alloc, "No buffer allocator found") - for (size_t i = 0; i < tsrs.size(); i++) { - tensor_detail_to_ir_tensor(graph, name_prefix + std::to_string(i), tsrs[i], - buf_alloc); - } -} -} // namespace graph - -mixed_fuse_op_t *get_mixed_op_from_graph(sc_graph_t &graph) { - mixed_fuse_op_t *mixed_op = nullptr; - for (auto &op : graph.ops_) { - if (auto mx_op = op->dyn_cast()) { - COMPILE_ASSERT(!mixed_op, "Only one fused op is expected") - mixed_op = mx_op; - } - } - return mixed_op; -} - -void commit_graph_to_func(sc_graph_t &g, const func_t &func, - const fusion_anchor_mgr_t &fmgr) { - // query binding axis - query_binding_axis(g); - auto ctx = std::make_shared(*get_default_context()); - // set opt level to diable partition optimization - ctx->flags_.opt_level_ = sc_opt_level::lv0; - auto parti = std::make_shared(ctx, func, fmgr, g); - op_visitor_t visitor = - op_visitor_t::dfs_topology_speculative_sort(g.ops_.size()); - visitor.visit_graph(g, [&parti](op_visitor_t *visitor, const sc_op_ptr &op) { - if (op->isa() || op->isa()) - return; - if (parti->empty()) - parti->buf_alloc_.allocate_buffer(op.get()); - parti->add(op); - }); - parti->transform_to_mixed_op(); -} - -void mxp_buffer_allocator::set_buffer_inplace_hint(const expr &target_buf, - const expr &inplace_buf) { - // skip dynamic cases - // Buffer inplace currently does not support dynamic buffers - if (binded_mxp_->get_host_graph().is_dynamic()) { - return; - } - COMPILE_ASSERT(target_buf.defined() && inplace_buf.defined(), - "Both buffer should be defined") - // skip same buffer - if (inplace_buf.ptr_same(target_buf)) - return; - auto target_id = alias_info::get_or_create_alias_info(*target_buf.get()); - SC_MODULE_INFO << "Mark inplace hint for buffer: " << inplace_buf << " ==> " - << target_buf; - inplace_buf->attr()[attr_keys::tensor_inplace_hint] = - std::vector{ - {target_id, inplace_kind::ZERO_OFFSET}}; - // update inner inaplce map - inplace_map_[(uintptr_t)inplace_buf.get()] = - std::vector>{ - {(uintptr_t)target_buf.get(), inplace_kind::ZERO_OFFSET}}; -} - -void mxp_buffer_allocator::allocate_buffer(sc_op *op) { - auto &graph = op->get_owner_graph(); - // allocate input buffer - graph::tensor_detail_to_ir_tensor( - graph, op->op_name_ + "_" + std::to_string(op->logical_op_id_) + "_ins_", - op->get_inputs(), this); - - /* deal with special ops: explict inplace input */ - // tensorview op - if (auto tv_op = op->dyn_cast()) { - auto inp = tv_op->get_inputs()[0]; - if ((inp->uses_.size() == 1) && - (inp->details_.get_blocking_dims() == - tv_op->get_outputs()[0]->details_.get_blocking_dims()) && - (binded_mxp_->contains(inp->producer_owner_)) && - (!(g2b_map_.get(inp).isa() && - g2b_map_.get(inp).static_as()->init_value_))) { - if (graph.is_dynamic()) { - // reset plain dims hint as do inplacement here. - auto &tsr = g2b_map_.get(inp); - tsr->attr().set(attr_keys::plain_dims, - graph.dims_to_expr( - op->get_outputs()[0]->details_.get_plain_dims())); - } - g2b_map_.get(op->get_outputs()[0]) = g2b_map_.get(inp); - } else { - auto base_tsr = get_real_tensor(g2b_map_.get(inp)); - g2b_map_.get(op->get_outputs()[0]) = builder::tensor_ptr( - base_tsr, std::vector(base_tsr->dims_.size(), 0), - tv_op->get_shapes_expr()); - } - } - - // reduce collect op - if (auto collc_op = op->dyn_cast()) { - if (collc_op->is_place_holder_op()) { - // inplace reduce_compute_op output - COMPILE_ASSERT( - op->get_inputs()[0]->producer_owner_->isa(), - "reduce collect op is expected to follow reduce compute " - "op, but got " - << op->get_inputs()[0]->producer_owner_->op_name_) - // no code generated - g2b_map_.get(op->get_outputs()[0]) = g2b_map_.get(op->get_inputs()[0]); - } - } - - // allocate output buffer - graph::tensor_detail_to_ir_tensor( - graph, op->op_name_ + "_" + std::to_string(op->logical_op_id_) + "_outs_", - op->get_outputs(), this); - - // reorder op - if (auto reo_op = op->dyn_cast()) { - if (reo_op->check_padding()) { - op->get_outputs()[0]->attrs_.set(mixed_partition_hint::no_inplace, true); - } - } - - /* infer post-op inplace */ - auto query_inplace = [&](const graph_tensor_ptr &out, - const graph_tensor_ptr &in) -> bool { - return (!op->isa() && !op->isa()) && - (in->uses_.size() == 1) && (out != in) && - (out->details_.get_blocking_dims() == - in->details_.get_blocking_dims()) && - (out->details_.get_strides() == in->details_.get_strides()) && - (out->details_.dtype_ == in->details_.dtype_) && - (out->details_.get_format() == in->details_.get_format()) && - (binded_mxp_->contains( - in->producer_owner_)) // inputs of partition should not - // be inplaced - && (!in->producer_owner_->isa()) && - (!(g2b_map_.get(in).isa() && - g2b_map_.get(in) - .static_as() - ->init_value_)); // TODO(XXX): inplace inited - // tensor - }; - - for (auto &out : op->get_outputs()) { - if (out->attrs_.get_or_else(mixed_partition_hint::no_inplace, false)) { - continue; - } - // query input - for (auto &inp : op->get_inputs()) { - if (inp->attrs_.get_or_else(mixed_partition_hint::no_inplace, false)) { - continue; - } - if (query_inplace(out, inp)) { - // set buffer inplace hint for output buffer - set_buffer_inplace_hint(g2b_map_.get(inp), g2b_map_.get(out)); - break; - } - } - } - - /* deal with special ops: set tensor initial value */ - // reduce collect and compute op - if (auto rd_impl_op = op->dyn_cast()) { - auto buf = g2b_map_.get(op->get_outputs()[0]); - COMPILE_ASSERT(buf.isa(), - "output of reduce_impl op should be tensor type") - rd_impl_op->set_reduce_buffer(buf.checked_as()); - } - - /* infer pre-op inplace */ - if (op->isa() && op->get_inputs()[0]->uses_.size() == 1 && - !binded_mxp_->empty()) { - auto out = op->get_outputs()[0]; - auto ins = op->get_inputs()[0]; - auto old_input = g2b_map_.get(ins); - if (old_input.isa() || - (old_input.isa() && ins->uses_.size() == 1 && - utils::is_one_of(old_input.static_as()->base_->dtype_, - sc_data_type_t::u8(), sc_data_type_t::s8()))) { - if (old_input.isa()) { - auto &producer = ins->producer_owner_; - if (!producer->isa()) - return; - auto &tv_inp = producer->get_inputs()[0]; - if (tv_inp->attrs_.get_or_else(mixed_partition_hint::no_inplace, false)) - return; - if (producer->share_gt_with_op(tv_inp) || - producer->share_gt_with_op(tv_inp)) - return; - } - op->attrs_.set(mixed_partition_hint::inplace_optimized_op, true); - auto pad_op = op->dyn_cast(); - auto new_input = builder::tensor_ptr( - g2b_map_.get(out), pad_op->get_padding_offsets_exprs(), - pad_op->get_inputs()[0]->details_.get_blocking_dims_expr(graph), - true); - if (old_input.isa()) { - auto parent_tsr = get_real_tensor(old_input); - auto shape = parent_tsr->dims_; - new_input = builder::tensor_ptr( - new_input, - std::vector(new_input.static_as()->shape_.size(), - 0), - shape, false); - old_input = parent_tsr; - } - // Buffer replace - replace_buffer(old_input, new_input); - } - } -} - -std::vector -mxp_buffer_allocator::get_real_mem_trace( - const std::unordered_set &keep_cut_set) const { - std::vector shrink_trace = mem_trace_; - std::unordered_set ignore_buf_set; - for (auto &g2b : g2b_map_.datamap_) { - if (binded_mxp_->is_parti_cut(g2b.first) && - keep_cut_set.find(g2b.first) == keep_cut_set.end()) { - ignore_buf_set.insert(get_real_tensor(g2b.second)); - } - } - for (auto iter = shrink_trace.begin(); iter != shrink_trace.end();) { - auto tsr = ((expr_base *)iter->buffer_id_)->node_ptr_from_this(); - if (ignore_buf_set.find(tsr) != ignore_buf_set.end()) { - iter = shrink_trace.erase(iter); - } else { - iter++; - } - } - std::transform(shrink_trace.begin(), shrink_trace.end(), shrink_trace.begin(), - [&](const memory_optim::memory_alloc_trace_t &t) { - if (t.size_ == 0) - return t; - auto tsr = ((expr_base *)t.buffer_id_)->node_ptr_from_this(); - auto shrink_info = get_shrinked_info(tsr); - auto shape = get_dims_product(get_expr_to_dims( - shrink_info.empty() ? tsr.checked_as()->dims_ - : get_slice_shape(shrink_info))); - auto dtype_size = utils::get_sizeof_etype( - tsr.checked_as()->elem_dtype_.type_code_); - return memory_optim::memory_alloc_trace_t{ - t.buffer_id_, shape * dtype_size}; - }); - return shrink_trace; -} - -size_t get_buffer_usage( - const context_ptr &ctx, - const std::vector &mem_trace, - const memory_optim::inplace_info_map &inplace_map) { - std::unordered_map out_schedule; - std::unordered_map> out_inplace_selection; - return schedule_memory_allocations( - mem_trace, /*alignment*/ 64, - ctx->flags_.buffer_schedule_ == attr_keys::BUF_SCHED_HOT, inplace_map, - out_schedule, out_inplace_selection); -} - -size_t mxp_buffer_allocator::get_real_buffer_usage() const { - return get_buffer_usage(binded_mxp_->ctx_, get_real_mem_trace(), - inplace_map_); -} - -void mxp_buffer_allocator::replace_buffer(const expr &old_buffer, - const expr &new_buffer) { - // assert new buffer - COMPILE_ASSERT( - b2g_map_.find(new_buffer) == b2g_map_.end(), - "Currently, it is only expected to replace with new buffer which " - "never appear in mixed IR, but got " - << new_buffer) - // get old buffer - COMPILE_ASSERT(old_buffer.isa(), - "Replace target is expected to be Tensor node") - if (tsr2anch_map_.find(old_buffer) != tsr2anch_map_.end()) { - tsr2anch_map_.erase(old_buffer); - } - if (b2g_map_.find(old_buffer) != b2g_map_.end()) { - auto old_gt = b2g_map_[old_buffer]; - b2g_map_.erase(old_buffer); - b2g_map_[new_buffer] = old_gt; - } - // get real tsr - auto old_tsr = get_real_tensor(old_buffer), - new_tsr = get_real_tensor(new_buffer); - // update trace - bool new_tsr_already_in_trace = false; - std::for_each(mem_trace_.begin(), mem_trace_.end(), - [&old_tsr, &new_tsr, &new_tsr_already_in_trace]( - memory_optim::memory_alloc_trace_t &t) { - if (t.buffer_id_ == (uintptr_t)old_tsr.get()) { - t.buffer_id_ = (uintptr_t)new_tsr.get(); - } else if (t.buffer_id_ == (uintptr_t)new_tsr.get()) { - new_tsr_already_in_trace = true; - } - }); - if (new_tsr_already_in_trace) { - int cnt = 0; - // erase middle two times - for (auto iter = mem_trace_.begin(); iter != mem_trace_.end();) { - if (iter->buffer_id_ == (uintptr_t)new_tsr.get()) { - cnt++; - if (cnt == 2 || cnt == 3) { - iter = mem_trace_.erase(iter); - continue; - } - } - iter++; - } - COMPILE_ASSERT(cnt == 4, "Unexpected buffer occurs time in trace: " << cnt) - } - // update inplace map if necessary - for (auto iter = inplace_map_.begin(); iter != inplace_map_.end();) { - auto buf1 = ((expr_base *)iter->first)->node_ptr_from_this(); - auto buf2 = get_inplaced_buffer(buf1); - if (buf1.ptr_same(old_buffer) || buf2.ptr_same(old_buffer)) { - iter = inplace_map_.erase(iter); - } else { - iter++; - } - } - // replace g2b map - for (auto &g2b : g2b_map_.datamap_) { - auto &buf = g2b.second; - auto tsr = get_real_tensor(buf); - if (tsr.ptr_same(old_tsr.static_as())) { - set_base_tensor(buf, new_buffer); - } - } - // TIR replace - node_ptr_map buffer_map = {{old_buffer.impl, new_buffer.impl}}; - mxp_replacer_t(buffer_map).replace_func(binded_mxp_->func_); -} - -std::tuple, std::vector> -mxp_buffer_allocator::get_buffer(sc_op *op) const { - std::vector inputs(op->get_inputs().size()), - outputs(op->get_outputs().size()); - std::transform(op->get_inputs().begin(), op->get_inputs().end(), - inputs.begin(), [&](const graph_tensor_ptr >) { - COMPILE_ASSERT(g2b_map_.haskey(gt), - "please allocate buffer first") - return g2b_map_.datamap_.find(gt.get())->second; - }); - std::transform(op->get_outputs().begin(), op->get_outputs().end(), - outputs.begin(), [&](const graph_tensor_ptr >) { - COMPILE_ASSERT(g2b_map_.haskey(gt), - "please allocate buffer first") - return g2b_map_.datamap_.find(gt.get())->second; - }); - return std::make_tuple(inputs, outputs); -} - -static cmp_res cmp_op_anchor(sc_op *op, fusion_anchor_ptr cur_anchor, - fusion_anchor_ptr new_anchor) { - cmp_res res = cmp_res::unknown; - auto cmper = [&](const std::vector >_vec) { - std::for_each( - gt_vec.begin(), gt_vec.end(), [&](const graph_tensor_ptr >) { - if (utils::is_one_of(res, cmp_res::unknown, cmp_res::equal) && - cur_anchor->fsmap_.hasvalue(gt) && - new_anchor->fsmap_.hasvalue(gt)) { - res = cmp_slice_range(cur_anchor->fsmap_.get(gt), - new_anchor->fsmap_.get(gt)); - } - }); - }; - // compare input - cmper(op->get_inputs()); - // if result is unknown, continue compare output - if (res == cmp_res::unknown) { - cmper(op->get_outputs()); - } - COMPILE_ASSERT(res != cmp_res::unknown, "Unknown comparision result") - return res; -} - -void mxp_buffer_allocator::update_input_buffer_info(sc_op *op) { - auto commited_anchor_map = binded_mxp_->lookup_anchor_map(op); - auto update_inp_tensor_info = [&](const graph_tensor_ptr &inp) { - auto buf = g2b_map_.get(inp); - if (b2g_map_.find(buf) == b2g_map_.end()) - b2g_map_[buf] = inp; - auto tsr = get_real_tensor(buf); - bool is_borrowed = (commited_anchor_map->borrowed_fanchor_map_.find(inp) != - commited_anchor_map->borrowed_fanchor_map_.end()); - auto real_anchor_map = is_borrowed - ? commited_anchor_map->borrowed_fanchor_map_[inp] - : commited_anchor_map; - if (tsr2anch_map_.find(tsr) != tsr2anch_map_.end()) { - COMPILE_ASSERT(b2g_map_.find(buf) != b2g_map_.end(), - "base tensor should be visited") - // current anchor map - auto cur_anchor_map = tsr2anch_map_[tsr]; - // auto skip - if (cur_anchor_map == real_anchor_map) - return; - // redirect to common parent anchor if new anchor is cousin - // relationship of current anchor in avoid of use before define - if (cur_anchor_map->is_cousin_for(real_anchor_map)) { - // set common parent anchor no matter which anchor is larger - tsr2anch_map_[tsr] = real_anchor_map->get_root()->shared_from_this(); - } else { - auto cur_slice = cur_anchor_map->fsmap_.get(b2g_map_[buf]); - auto new_slice = real_anchor_map->fsmap_.get(inp); - auto res = cmp_slice_range(cur_slice, new_slice); - bool need_overwrite = false; - if (res == cmp_res::l_less_r) { - need_overwrite = true; - } else if (res == cmp_res::equal) { - // usually occurs in op is reduce or reduce collect op - need_overwrite = real_anchor_map->is_parent_for(cur_anchor_map) || - real_anchor_map->is_sibling_for(cur_anchor_map); - } - // update latest anchor map - if (need_overwrite) { - tsr2anch_map_[tsr] = real_anchor_map; - // update b2g map if necessary - b2g_map_[buf] = inp; - } - } - } else { - tsr2anch_map_[tsr] = real_anchor_map; - } - }; - - for (auto &inp : op->get_inputs()) { - update_inp_tensor_info(inp); - } -} - -void mxp_buffer_allocator::update_output_buffer_info(sc_op *op) { - auto commited_anchor_map = binded_mxp_->lookup_anchor_map(op); - auto sub_of_commited_anchor_map = - binded_mxp_->lookup_sub_anchor_map(commited_anchor_map); - - auto update_out_tensor_info = [&](const graph_tensor_ptr &out) { - auto buf = g2b_map_.get(out); - if (b2g_map_.find(buf) == b2g_map_.end()) - b2g_map_[buf] = out; - auto tsr = get_real_tensor(buf); - if (tsr2anch_map_.find(tsr) != tsr2anch_map_.end()) { - // only input anchor is expected for below logic - if (!commited_anchor_map->is_input_anchor()) - return; - COMPILE_ASSERT(b2g_map_.find(buf) != b2g_map_.end(), - "base tensor should be visited") - // auto skip - if (tsr2anch_map_[tsr] == commited_anchor_map) - return; - auto cur_slice = tsr2anch_map_[tsr]->fsmap_.get(b2g_map_[buf]); - auto new_slice = commited_anchor_map->fsmap_.get(out); - auto res = cmp_slice_range(cur_slice, new_slice); - if (res == cmp_res::l_less_r) { - tsr2anch_map_[tsr] = commited_anchor_map; - } - } else { - fusion_anchor_ptr min_anchor_map = nullptr; - for (auto &sub_anchor : sub_of_commited_anchor_map) { - if (!sub_anchor->fsmap_.hasvalue(out)) - continue; - if (!min_anchor_map) - min_anchor_map = sub_anchor; - else { - auto min_slice = min_anchor_map->fsmap_.get(out); - auto cur_slice = sub_anchor->fsmap_.get(out); - if (cmp_slice_range(min_slice, cur_slice) == cmp_res::l_larger_r) { - min_anchor_map = sub_anchor; - } - } - } - tsr2anch_map_[tsr] = - min_anchor_map ? min_anchor_map : commited_anchor_map; - } - }; - - for (auto &out : op->get_outputs()) { - update_out_tensor_info(out); - } -} - -void mxp_buffer_allocator::tensor_initialize() { - for (auto &pair : g2b_map_.datamap_) { - auto op = pair.first->producer_owner_; - // zero out padding area - if (auto padding = op->dyn_cast()) { - stmts decl_body; - auto pad_tsr = get_real_tensor(pair.second); - slice_range_list range_list = {}; - if (pair.second->attr().has_key(mixed_partition_hint::cut_buffer)) { - decl_body = binded_mxp_->func_->body_.checked_as(); - } else { - COMPILE_ASSERT(tsr2anch_map_.find(pad_tsr) != tsr2anch_map_.end(), - "Could not find padding tensor: " - << pad_tsr << " in tsr2anchor map") - auto anchor = tsr2anch_map_[pad_tsr]; - range_list = anchor->fsmap_.get(b2g_map_[pair.second]); - decl_body = anchor->get_parent_scope(); - } - if (!op->attrs_.has_key( - mixed_partition_hint::tensor_already_initialized)) { - auto ret = padding->get_zero_out_stmt(pad_tsr, range_list); - decl_body->seq_.insert(decl_body->seq_.begin(), ret); - op->attrs_[mixed_partition_hint::tensor_already_initialized] = true; - } - } - } -} - -void mxp_buffer_allocator::copy_concat_memory_attrs_tsr2buf() { - for (auto &op : binded_mxp_->committed_ops_) { - if (!op->isa()) { - continue; - } - auto concat = op->stc_cast(); - for (auto &input_tsr : concat->get_inputs()) { - if (input_tsr->attrs_.has_key( - concat_optim_attr_keys::graph_memory_offset)) { - auto &offset = input_tsr->attrs_.get>( - concat_optim_attr_keys::graph_memory_offset); - - auto &buf = binded_mxp_->buf_alloc_.g2b_map_.get(input_tsr); - COMPILE_ASSERT(buf.isa(), - "Buffer with memory_offset should be a tensor") - buf->attr()[concat_optim_attr_keys::pass_memory_offset] = offset; - - auto &final_tsr = input_tsr->attrs_.get( - concat_optim_attr_keys::graph_memory_offset_to); - COMPILE_ASSERT(binded_mxp_->buf_alloc_.g2b_map_.haskey(final_tsr), - "No buffer allocated for concat outputs") - auto &out_buffer = binded_mxp_->buf_alloc_.g2b_map_.get(final_tsr); - buf->attr()[concat_optim_attr_keys::pass_memory_offset_to] = out_buffer; - SC_MODULE_INFO << "Buffer: " << buf - << " has memory offset to buffer: " << out_buffer; - } - } - } -} - -inline bool is_elementwise_op(const sc_op *op) { - return op->isa() || - op->isa(); -} - -inline bool is_elementwise_producer(const graph_tensor *gt) { - return is_elementwise_op(gt->producer_owner_); -} - -inline bool is_reshape_op(const sc_op *op) { - return op->isa() || op->isa() || - op->isa(); -} - -// If last gt depends on all users of cur gt, return true -inline bool check_last_use_for_gt(const graph_tensor *cur_gt, - const graph_tensor *last_gt, - const mxp_buffer_allocator *alloc) { - return std::all_of( - cur_gt->uses_.begin(), cur_gt->uses_.end(), - [&alloc, &last_gt](const std::pair &user) { - return alloc->get_binded_mxp()->dep_m_->lookup( - user.second.get(), last_gt->producer_owner_) == 1; - }); -} - -// max step to explore preview ops -static constexpr int EXPLORE_INPLACE_MAX_STEP = 8; - -static void -collect_inplace_info(graph_tensor *cur_gt, graph_tensor *ref_gt, - std::unordered_set &inplace_set, - std::unordered_set &visited_set, - const std::unordered_set &valid_set, - mxp_buffer_allocator *alloc, int step) { - // increment by 1 recursive depth and auto skip - if (EXPLORE_INPLACE_MAX_STEP == (step++)) - return; - // skip repeated - if (inplace_set.find(cur_gt) != inplace_set.end()) - return; - // return when producer is not elementwise op - if (!is_elementwise_producer(cur_gt)) { - return; - } - // use buffer as key for map - auto cur_buf = alloc->g2b_map_.get(cur_gt); - // check inplace condition - if ((visited_set.find(cur_buf) == visited_set.end()) && - (valid_set.find(cur_gt) != valid_set.end()) && (cur_gt != ref_gt) && - (cur_gt->details_.get_blocking_dims() == - ref_gt->details_.get_blocking_dims()) && - (cur_gt->details_.get_strides() == ref_gt->details_.get_strides()) && - (cur_gt->details_.dtype_ == ref_gt->details_.dtype_) && - (cur_gt->details_.get_format() == ref_gt->details_.get_format()) && - check_last_use_for_gt(cur_gt, ref_gt, alloc)) { - inplace_set.insert(cur_gt); - // reset step - step = 0; - // reset ref_gt - ref_gt = cur_gt; - } - - // if not mark visited - if (visited_set.find(cur_buf) == visited_set.end()) { - // recursively mark visited - while (cur_buf.defined()) { - visited_set.insert(cur_buf); - cur_buf = alloc->get_inplaced_buffer(cur_buf); - } - } - // get cur op - auto elem_op = cur_gt->producer_owner_; - // recursively collect inplace information - for (auto &inp : elem_op->get_inputs()) { - collect_inplace_info(inp.get(), ref_gt, inplace_set, visited_set, valid_set, - alloc, step); - } -} - -expr mxp_buffer_allocator::get_inplaced_buffer(const expr &buf) const { - auto iter = inplace_map_.find((uintptr_t)buf.get()); - if (iter != inplace_map_.end()) { - COMPILE_ASSERT(iter->second.size() == 1, - "Unexpected inplace info size during partition") - return ((expr_base *)iter->second[0].first)->node_ptr_from_this(); - } - return expr(); -} - -void mxp_buffer_allocator::query_inplace() { - SC_MODULE_INFO << "Query buffer inplace hint..."; - // step 0: get outer loop - auto outer_loops = binded_mxp_->get_outer_loops(); - if (outer_loops.empty()) - return; - auto batch_anchor = binded_mxp_->get_anchor_inside_loop(outer_loops.back()); - if (!batch_anchor) - return; - - // step 1: search gt which defined inside outer loop - std::vector ref_gt_list; - std::unordered_set valid_set; - for (auto &tsr2anch : tsr2anch_map_) { - auto anchor = tsr2anch.second; - auto tsr = tsr2anch.first; - if (tsr->attr().has_key(mixed_partition_hint::cut_buffer)) - continue; - COMPILE_ASSERT(b2g_map_.find(tsr) != b2g_map_.end(), - "No buffer to graph tensor mapping found for " << tsr) - auto shrink_gt = b2g_map_[tsr]; - auto slice_on_batch_anchor = batch_anchor->fsmap_.get(shrink_gt); - auto slice_on_cur_anchor = slice_range_list{get_shrinked_info(tsr)}; - if (slice_on_cur_anchor.empty() || slice_on_batch_anchor.empty()) - continue; - if (cmp_slice_range(slice_on_batch_anchor, slice_on_cur_anchor) == - cmp_res::equal) { - ref_gt_list.emplace_back(b2g_map_[tsr].get()); - valid_set.insert(b2g_map_[tsr].get()); - } - } - // skip - if (ref_gt_list.empty()) - return; - // sort by op id - std::sort(ref_gt_list.begin(), ref_gt_list.end(), - [](const graph_tensor *gt1, const graph_tensor *gt2) { - return gt1->producer_owner_->logical_op_id_ > - gt2->producer_owner_->logical_op_id_; - }); - - std::unordered_set replaced_buffer, visited_buffer; - for (auto &ref_gt : ref_gt_list) { - // auto skip - if (replaced_buffer.find(g2b_map_.get(ref_gt)) != replaced_buffer.end()) - continue; - // step 2: collect inplace mapping for each gt - std::unordered_set inplace_gt_set; - collect_inplace_info(ref_gt, ref_gt, inplace_gt_set, visited_buffer, - valid_set, this, /*init_step*/ 0); - if (inplace_gt_set.empty()) - continue; - // step 3: transform map to vector sorted by op committing order - std::vector inplace_gt_list; - for (auto &commit_op : binded_mxp_->committed_ops_) { - if (commit_op.get() == ref_gt->producer_owner_) { - inplace_gt_list.emplace_back(ref_gt); - } else { - for (auto &out : commit_op->get_outputs()) { - if (inplace_gt_set.find(out.get()) != inplace_gt_set.end()) { - inplace_gt_list.emplace_back(out.get()); - } - } - } - } - // step 4: validate inplace chain to ensure all of gt satisfy last use - graph_tensor *next_gt = nullptr; - for (auto iter = inplace_gt_list.rbegin(); - iter != inplace_gt_list.rend();) { - if (!next_gt) - next_gt = (*iter); - else { - if (check_last_use_for_gt(*iter, next_gt, this)) { - next_gt = (*iter); - } else { - iter = std::vector::reverse_iterator( - inplace_gt_list.erase((++iter).base())); - continue; - } - } - ++iter; - } - // step 5: mark inplace attr for each buffer in list with the previous - // one - for (size_t i = 1; i < inplace_gt_list.size(); i++) { - // get target gt - auto target_gt = inplace_gt_list[i - 1]; - auto target_buf = g2b_map_.get(target_gt); - // get inplace gt - auto inplace_gt = inplace_gt_list[i]; - auto inplace_buf = g2b_map_.get(inplace_gt); - // skip repeated replaced - if (replaced_buffer.find(inplace_buf) != replaced_buffer.end()) - continue; - // set inplace hint - set_buffer_inplace_hint(target_buf, inplace_buf); - // in avoid of repeat try - replaced_buffer.insert(inplace_buf); - } - } -} - -void mxp_buffer_allocator::calibrate_info() { - // validate shrink info - for (auto &buf2shr : b2g_map_) { - auto &buf = buf2shr.first; - auto tsr = get_real_tensor(buf); - // sync shrink info of tensorptr and base tensor - if (!buf.ptr_same(tsr) && - (buf->attr().has_key(tensor_shrinker_attrs::should_shrink) ^ - tsr->attr().has_key(tensor_shrinker_attrs::should_shrink))) { - buf->attr().remove(tensor_shrinker_attrs::should_shrink); - buf->attr().set(tensor_shrinker_attrs::no_shrink, true); - tsr->attr().remove(tensor_shrinker_attrs::should_shrink); - tsr->attr().set(tensor_shrinker_attrs::no_shrink, true); - } - } - - // collect cut buffer - std::unordered_set cut_buffer_set; - for (auto iter = inplace_map_.begin(); iter != inplace_map_.end();) { - auto out_buf = ((expr_base *)iter->first)->node_ptr_from_this(); - if (!out_buf->attr().has_key(mixed_partition_hint::cut_buffer)) { - ++iter; - continue; - } - // temp buffer set - std::unordered_set temp_set; - // record whether tensorptr would be inplaced by cut buffer - bool inplace_tptr = false; - auto buf = out_buf; - while (buf.defined()) { - // if tensorptr found - if (buf.isa()) { - inplace_tptr = true; - break; - } - temp_set.insert(buf); - buf = get_inplaced_buffer(buf); - } - if (!inplace_tptr) { - // remove tensor shrink attr for those shared with cut buffer - for (auto &tb : temp_set) { - tb->attr().remove(tensor_shrinker_attrs::should_shrink); - } - cut_buffer_set.insert(temp_set.begin(), temp_set.end()); - ++iter; - } else { - // cut off inplace hint for the output buffer - out_buf->attr().remove(attr_keys::tensor_inplace_hint); - // remove inplace map - iter = inplace_map_.erase(iter); - } - } - - // validate inplace hint for shrink info - for (auto iter = inplace_map_.begin(); iter != inplace_map_.end();) { - auto buf1 = ((expr_base *)iter->first)->node_ptr_from_this(); - auto buf2 = get_inplaced_buffer(buf1); - // auto skip cut buffer - if (cut_buffer_set.find(buf1) != cut_buffer_set.end()) { - COMPILE_ASSERT(cut_buffer_set.find(buf2) != cut_buffer_set.end(), - "inplaced buffer should also be set cut buffer") - ++iter; - } else { - auto shrink_info1 = get_shrinked_info(buf1); - auto shrink_info2 = get_shrinked_info(buf2); - // if shrink info is not equal, remove inplace hint - if ((shrink_info1.empty() ^ shrink_info2.empty()) || - (!shrink_info1.empty() && !shrink_info2.empty() && - cmp_slice_range({shrink_info1}, {shrink_info2}) != cmp_res::equal)) { - SC_MODULE_INFO << "removing tensor inplace hint: " << buf1 << " ==> " - << buf2 << " for safety"; - // remove inplace hint to ensure correctness - buf1->attr().remove(attr_keys::tensor_inplace_hint); - // remove inplace map - iter = inplace_map_.erase(iter); - } else { - ++iter; - } - } - } -} - -inline bool check_tsr_len_under_resigter_size(size_t tsr_len, uint16_t simd_len, - uint16_t max_register_tol = 16) { - return (tsr_len % simd_len == 0 && (tsr_len / simd_len) <= max_register_tol); -} - -bool mxp_buffer_allocator::validate_tsr2var() const { - for (auto &tsr2def : tsr2anch_map_) { - auto tsr = tsr2def.first; - if (!tsr->attr().has_key(attr_keys::must_tensor2var)) - continue; - auto shrinked_info = get_shrinked_info(tsr); - if (shrinked_info.empty()) - continue; - auto shape = get_slice_shape(shrinked_info); - auto prod = get_dims_product(get_expr_to_dims(shape)); - auto tsr_simd_len = vectorize_step( - binded_mxp_->ctx_, tsr.checked_as()->elem_dtype_.type_code_); - if (!check_tsr_len_under_resigter_size(prod, tsr_simd_len)) - return false; - } - return true; -} - -int mxp_buffer_allocator::use_count(const expr &buffer) const { - int cnt = 0; - for (auto &g2b : g2b_map_.datamap_) { - if (g2b.second.ptr_same(buffer)) - cnt++; - } - return cnt; -} - -fusion_anchor_ptr -mxp_buffer_allocator::get_real_anchor_for_buffer(const expr &buffer) const { - auto tsr = get_real_tensor(buffer); - if (tsr2anch_map_.find(tsr) == tsr2anch_map_.end()) - return nullptr; - auto anch = tsr2anch_map_.find(tsr)->second; - auto parent_loop = anch->get_parent_loop(); - // use outer anchor for cases that calculation is partially done. (e.g. - // calculating part of K for matmul) - if (b2g_map_.find(tsr)->second->producer_owner_->isa() && - parent_loop.isa() && - parent_loop->attr().has_key(stmt_attr_key::reduce_root_loop)) { - auto raw = - parent_loop->attr() - .get>(stmt_attr_key::reduce_root_loop) - .lock(); - COMPILE_ASSERT(raw, "reduce_root_loop weak ptr invalidated"); - anch = - binded_mxp_->get_anchor_inside_loop(stmt(raw).checked_as()); - COMPILE_ASSERT(anch, - "No anchor found under reduce root loop, please create it " - "otherwise, it may cause correctness issue") - } - return anch; -} - -slice_range mxp_buffer_allocator::get_shrinked_info(const expr &buffer) const { - if (buffer->attr().get_or_else(tensor_shrinker_attrs::no_shrink, false)) - return {}; - auto anchor = get_real_anchor_for_buffer(buffer); - if (!anchor) - return {}; - COMPILE_ASSERT(b2g_map_.find(buffer) != b2g_map_.end(), - "Could not find " << buffer << " in b2g map"); - slice_range ret; - auto range_list = anchor->fsmap_.get(b2g_map_.find(buffer)->second); - if (range_list.size() != 1) { - if (range_list.empty() || !(anchor->isa() || - anchor->isa())) - return {}; - else - ret = *std::max_element(range_list.begin(), range_list.end(), - [&](const slice_range &A, const slice_range &B) { - return cmp_slice_range({A}, {B}) == - cmp_res::l_less_r; - }); - - } else { - ret = range_list[0]; - } - - return ret; -} - -void mxp_buffer_allocator::declare_tensor() const { - // define real tensor, and put it at the `def_pos` of ss - auto declare_tensor_ = [&](const expr &tsr) { - auto fanchor = get_real_anchor_for_buffer(tsr); - if (!fanchor) - return; - // search insert position in scope of fusion anchor - auto &ss = fanchor->get_parent_scope()->seq_; - size_t def_pos = 0; - for (; def_pos < ss.size(); def_pos++) { - // skip var def node - if (!ss[def_pos] - .cast() - .filter([](const define &d) { return d->var_.isa(); }) - .has_value()) - break; - } - ss.emplace(ss.begin() + def_pos, - builder::make_var_tensor_def_unattached(tsr)); - }; - - // collect tensors those need definition - std::vector define_list; - for (auto &tsr2def : tsr2anch_map_) { - if (tsr2def.first->attr().has_key(mixed_partition_hint::cut_buffer)) - continue; - define_list.emplace_back(tsr2def.first); - } - // sort by tsr name for stable define order - std::sort( - define_list.begin(), define_list.end(), [](const expr &a, const expr &b) { - COMPILE_ASSERT(a.isa() && b.isa(), - "tensor node is expected, but got " << a << " and " << b) - return a.static_as()->name_ < b.static_as()->name_; - }); - // declare tensors by order - for (auto &tsr : define_list) { - declare_tensor_(tsr); - } -} - -void mxp_buffer_allocator::set_shrink_info() const { - // set shrink info - auto set_shrink_info_ = [&](const expr &buffer) { - auto shrink_range = get_shrinked_info(buffer); - if (shrink_range.empty()) - return; - buffer->attr()[tensor_shrinker_attrs::should_shrink] = - tensor_shrinker_t::shrink_info_t{ - /*base*/ get_slice_idx(shrink_range), - /*shape*/ get_slice_shape(shrink_range), stmts()}; - }; - for (auto &buf2shr : b2g_map_) { - if (buf2shr.first->attr().has_key(mixed_partition_hint::cut_buffer)) - continue; - set_shrink_info_(buf2shr.first); - } -} - -std::vector merge_mem_trace( - const std::vector &mem_trace1, - const std::vector &mem_trace2, - const std::unordered_map &buffer_map) { - auto ret = mem_trace1; - for (auto &trace : mem_trace2) { - auto tsr = ((expr_base *)trace.buffer_id_)->node_ptr_from_this(); - auto trace_itr = buffer_map.find(tsr); - bool is_replaced = trace_itr != buffer_map.end(); - auto buf_id = - is_replaced ? (uintptr_t)(trace_itr->second.get()) : trace.buffer_id_; - if (trace.size_ > 0) { - auto last_use = - std::find_if(ret.begin(), ret.end(), - [&buf_id](const memory_optim::memory_alloc_trace_t &tr) { - return (tr.buffer_id_ == buf_id) && (tr.size_ == 0); - }); - if (last_use != ret.end()) { - ret.erase(last_use); - continue; - } - } - if (is_replaced) { - ret.emplace_back(memory_optim::memory_alloc_trace_t{buf_id, trace.size_}); - } else { - ret.emplace_back(trace); - } - } - return ret; -} - -memory_optim::inplace_info_map -merge_inplace_map(const memory_optim::inplace_info_map &inplace_map1, - const memory_optim::inplace_info_map &inplace_map2, - const std::unordered_map &buffer_map) { - auto ret = inplace_map1; - for (auto &inplace_pair : inplace_map2) { - COMPILE_ASSERT(inplace_pair.second.size() == 1, - "Unexpected inplace info size during partition") - memory_optim::inplace_info info = inplace_pair.second[0]; - auto tsr = ((expr_base *)info.first)->node_ptr_from_this(); - auto iter = buffer_map.find(tsr); - ret[inplace_pair.first] = std::vector>{ - {iter != buffer_map.end() ? (uintptr_t)iter->second.get() : info.first, - info.second}}; - } - return ret; -} - -mxp_mem_info merge_real_mem_info(const mxp_buffer_allocator &alloc1, - const mxp_buffer_allocator &alloc2) { - // get buffer replace map - std::unordered_map buffer_map; - std::unordered_set keep_cut_set; - for (auto &g2b : alloc2.g2b_map_.datamap_) { - auto gt = g2b.first; - if (alloc1.g2b_map_.haskey(gt)) { - buffer_map[g2b.second] = alloc1.g2b_map_.datamap_.find(gt)->second; - if (alloc1.get_binded_mxp()->is_parti_out(gt) && - alloc2.get_binded_mxp()->is_parti_inp(gt)) { - keep_cut_set.insert(gt); - } - } - } - return std::make_pair( - merge_mem_trace(alloc1.get_real_mem_trace(keep_cut_set), - alloc2.get_real_mem_trace(keep_cut_set), buffer_map), - merge_inplace_map(alloc1.inplace_map_, alloc2.inplace_map_, buffer_map)); -} - -void mxp_buffer_allocator::merge( - mxp_buffer_allocator &other, std::unordered_map &buffer_map, - const std::pair - &common_buffer_anchor_pair) { - buffer_map.clear(); - auto common_buffer_anchor = common_buffer_anchor_pair.first, - common_other_buffer_anchor = common_buffer_anchor_pair.second; - for (auto &other_g2b : other.g2b_map_.datamap_) { - auto other_gt = other_g2b.first; - // if other tensor has conflict in current tensor, redirect it to common - // buffer anchor - if (g2b_map_.haskey(other_gt)) { - auto existed_buf = g2b_map_.get(other_gt); - buffer_map[other_g2b.second] = existed_buf; - if ((binded_mxp_->is_parti_inp(other_gt) && - other.binded_mxp_->is_parti_inp(other_gt)) || - (binded_mxp_->is_parti_out(other_gt) && - other.binded_mxp_->is_parti_out(other_gt))) - continue; - COMPILE_ASSERT(common_buffer_anchor, - "Conflict buffer: " - << existed_buf - << " is detected but no common buffer anchor " - "is found for redirection") - tsr2anch_map_[get_real_tensor(existed_buf)] = common_buffer_anchor; - // the existed buffer will become intermediate buffer, which may be - // shrinked - if (b2g_map_.find(existed_buf) == b2g_map_.end()) { - b2g_map_[existed_buf] = other_gt->shared_from_this(); - } - } else { - auto buffer = other.g2b_map_.get(other_gt); - g2b_map_.get(other_gt) = buffer; - if (other.b2g_map_.find(buffer) != other.b2g_map_.end()) { - b2g_map_[buffer] = other.b2g_map_[buffer]; - } - if (other.tsr2anch_map_.find(get_real_tensor(buffer)) != - other.tsr2anch_map_.end()) { - auto other_anchor = other.tsr2anch_map_[get_real_tensor(buffer)]; - tsr2anch_map_[get_real_tensor(buffer)] = - (other_anchor == common_other_buffer_anchor) ? common_buffer_anchor - : other_anchor; - } - } - } - // merge mem trace - mem_trace_ = merge_mem_trace(mem_trace_, other.mem_trace_, buffer_map); - // merge inplace map - inplace_map_ = - merge_inplace_map(inplace_map_, other.inplace_map_, buffer_map); -} - -void mxp_buffer_allocator::clear() { - binded_mxp_ = nullptr; - g2b_map_.clear(); - tsr2anch_map_.clear(); - b2g_map_.clear(); - mem_trace_.clear(); -} - -void search_op_anchor_in_parti(sc_op *op, mixed_parti_t *parti) { - if (parti->merged_to) { - search_op_anchor_in_parti(op, parti->get_root()); - return; - } - - for (auto &fanchor : parti->fanchors_) { - // if op is marked as break_pre_fuse, only pre-op fusion is accepted - if (!parti->empty() && - op->attrs_.get_or_else(mixed_partition_hint::pre_fuse_begin_op, - false)) { - if (fanchor->fsmap_.hasvalue(op->get_outputs()[0])) { - parti->set_anchor_for_op(op, fanchor); - } - // in avoid of preop fusion select unexpected anchor - continue; - } - auto ctx = parti->ctx_; - infer_status_code status; - std::unordered_set known_gt; - // deal with fusible op which use output loop mode to create partition - if (parti->empty() && - std::any_of(op->get_outputs().begin(), op->get_outputs().end(), - [&fanchor](const graph_tensor_ptr >) { - return fanchor->fsmap_.haskey(gt); - })) { - COMPILE_ASSERT(op->isa(), - "Only fusible op is expected, but got " << op->op_name_) - // use pre-infer slice range method - status = op->dyn_cast()->pre_infer_slice_ranges( - ctx, fanchor->fsmap_); - } else { /* most common cases */ - // check input slice firstly - if (!fanchor->check_input_for_op(op, known_gt)) - continue; - // TODO(XXX): merge - if (auto fusible = op->dyn_cast()) { - status = fusible->infer_slice_ranges(ctx, fanchor->fsmap_); - } else if (auto tunable = op->dyn_cast()) { - status = tunable->infer_slice_ranges(ctx, fanchor->fsmap_); - } else { - COMPILE_ASSERT(0, "Unexpected op type found: " << op->op_name_) - } - } - bool success_flag = true; - // check status map - if (status == infer_status_code::OK) { - /** doule check list - * 1. validate input slice - * 2. validate output slice - * 3. check cost model - * */ - success_flag = fanchor->validate_input_for_op(op, known_gt) && - fanchor->validate_output_for_op(op) && - parti->cost_->make_decision_for_op(op, fanchor); - } else { - success_flag = false; - } - if (success_flag) { - // set op anchor, and auto select smallest one - parti->set_anchor_for_op(op, fanchor); - } else { - // forbid op in current anchor - fanchor->forbid_op(op, known_gt); - } - } -} - -static std::string print_loops_range(const std::vector &loops) { - std::stringstream os; - int cnt = 0; - for (auto &l : loops) { - if (cnt != 0) { - os << "X"; - } - if (l->iter_begin_.isa() && l->iter_end_.isa()) { - os << (get_expr_as_int(l->iter_end_) - get_expr_as_int(l->iter_begin_)); - } else { - os << "var"; - } - cnt++; - } - return os.str(); -} - -enum class parti_dep : int { - no_dep = 0, - l_dep_r = 1, - r_dep_l = 2, - inter_dep = 3, -}; - -/** - * Check two partition dependency - * */ -static parti_dep check_parti_dep(mixed_parti_t *A, mixed_parti_t *B) { - A = A->get_root(), B = B->get_root(); - auto dep_m = A->dep_m_; - auto A_ops = A->ops, B_ops = B->ops; - bool A_dep_B = false, B_dep_A = false; - for (auto &op_a : A_ops) { - for (auto &op_b : B_ops) { - auto dep_flag = dep_m->lookup(op_a, op_b); - if (dep_flag == 1) - B_dep_A = true; - else if (dep_flag == -1) - A_dep_B = true; - } - } - if (A_dep_B && !B_dep_A) { - return parti_dep::l_dep_r; - } else if (B_dep_A && !A_dep_B) { - return parti_dep::r_dep_l; - } else if (!A_dep_B && !B_dep_A) { - return parti_dep::no_dep; - } else { - return parti_dep::inter_dep; - } -} - -/** - * Check two partition connectionship - * */ -static bool check_parti_connectionship(mixed_parti_t *A, mixed_parti_t *B) { - A = A->get_root(), B = B->get_root(); - auto A_ops = A->ops, B_ops = B->ops; - for (auto &op_a : A_ops) { - std::unordered_set gt_set; - std::for_each(op_a->get_inputs().begin(), op_a->get_inputs().end(), - [>_set](const graph_tensor_ptr >) { gt_set.insert(gt); }); - std::for_each(op_a->get_outputs().begin(), op_a->get_outputs().end(), - [>_set](const graph_tensor_ptr >) { gt_set.insert(gt); }); - for (auto &op_b : B_ops) { - for (auto &inp : op_b->get_inputs()) { - if (gt_set.find(inp) != gt_set.end()) - return true; - } - for (auto &out : op_b->get_outputs()) { - if (gt_set.find(out) != gt_set.end()) - return true; - } - } - } - return false; -} - -/** - * Check two partition ring risk - * */ -static bool check_parti_ring_risk(mixed_parti_t *A, mixed_parti_t *B) { - A = A->get_root(), B = B->get_root(); - auto dep = check_parti_dep(A, B); - if (dep == parti_dep::inter_dep) - return true; - else if (dep == parti_dep::no_dep) - return false; - - auto append_parti = (dep == parti_dep::l_dep_r) ? A : B, - target_parti = (dep == parti_dep::l_dep_r) ? B : A; - auto append_ops = append_parti->ops; - for (auto &op_a : append_ops) { - for (auto &inp : op_a->get_inputs()) { - if (append_parti->is_parti_inp(inp) && - !target_parti->contains(inp->producer_owner_)) { - for (auto &op_in_set : target_parti->ops) { - auto result = target_parti->dep_m_->lookup( - op_in_set->logical_op_id_, inp->producer_owner_->logical_op_id_); - if (result == 1) { - return true; - } - } - } - } - } - return false; -} - -static stmts find_last_parallel_for(const stmts &scope, int64_t &out_index) { - auto &outer_body = scope.static_as()->seq_; - int64_t prefetch_insertion_point = -1; - if (!outer_body.empty()) { - for (int64_t i = outer_body.size() - 1; i >= 0; i--) { - auto &s = outer_body[i]; - if (s.isa()) { - auto ret = find_last_parallel_for(s.static_as(), out_index); - if (ret.defined()) { - return ret; - } - } - if (s.cast() - .filter([](const for_loop &v) { - return v->kind_ == for_type::PARALLEL && - (!v->attr_ || - !v->attr_->get_or_else("dont_prefetch", false)); - }) - .has_value()) { - out_index = i; - return scope; - } - } - } - return stmts(); -} - -/** - * Check two partition is forked, like - * input - * / \ - * parti A parti B - * | | - * output output - * Note that: this is different relationship from no dependency - * */ -static bool check_parti_forked(mixed_parti_t *A, mixed_parti_t *B) { - A = A->get_root(), B = B->get_root(); - auto dep_m = A->dep_m_; - // for all op depends on A, should not depends on B - for (auto &op_in_A : A->ops) { - auto A_id = op_in_A->logical_op_id_; - auto related_ids = dep_m->lookup_ops_depend_on(A_id); - for (auto &op_in_B : B->ops) { - auto B_id = op_in_B->logical_op_id_; - if (std::any_of(related_ids.begin(), related_ids.end(), - [&B_id, &dep_m](const int &rid) { - return dep_m->lookup(B_id, rid) == 1; - })) { - return false; - } - } - } - return true; -} - -/** - * Check two partition axis binding - * */ -static int check_parti_loop_axis_binding(mixed_parti_t *A, mixed_parti_t *B, - int check_loop_size) { - A = A->get_root(); - B = B->get_root(); - // skip conv workload until all conv ops implement axis binding infer - if (A->contain_convolution() || B->contain_convolution()) { - // limit to at most one outer loop (batch-dimension) - return std::min(1, check_loop_size); - } - // auto skip when A and B are forked - if (check_parti_forked(A, B)) - return check_loop_size; - return check_loop_binding_axis(A->get_outer_loops(), B->get_outer_loops(), - check_loop_size); -} - -static void merge_parti_impl(mixed_parti_t *pa_to_merge, - mixed_parti_t *parti_be_merged, - size_t merged_loop_size, - const sc_op_ptr &joint_op = nullptr) { - pa_to_merge = pa_to_merge->get_root(), - parti_be_merged = parti_be_merged->get_root(); - auto outer_loops_to_merge = pa_to_merge->get_outer_loops(), - outer_loops_be_merged = parti_be_merged->get_outer_loops(); - auto outer_loops_merged_target = outer_loops_to_merge[merged_loop_size - 1]; - auto outer_loops_merged_append = outer_loops_be_merged[merged_loop_size - 1]; - auto max_to_merge_anchor_map = - pa_to_merge->get_anchor_inside_loop(outer_loops_merged_target); - auto max_be_merged_anchor_map = - parti_be_merged->get_anchor_inside_loop(outer_loops_merged_append); - /* * * * * * * * * * * * * * * * * - * Step 1: Merge func_ - * * * * * * * * * * * * * * * * */ - COMPILE_ASSERT(max_to_merge_anchor_map, - "max-to-merge fusion anchor not found") - max_to_merge_anchor_map->fusion_anchor_t::commit_stmt( - outer_loops_merged_append->body_); - - // var and tensor replace map - node_ptr_map node_remap; - std::unordered_map buffer_map; - - /* * * * * * * * * * * * * * * * * - * Step 2: Merge fanchor_ - * * * * * * * * * * * * * * * * */ - // erase inferred but not allocated gt - for (auto &to_merge_anchor_map : pa_to_merge->fanchors_) { - for (auto iter = to_merge_anchor_map->fsmap_.datamap_.begin(); - iter != to_merge_anchor_map->fsmap_.datamap_.end();) { - if (!pa_to_merge->buf_alloc_.g2b_map_.haskey(iter->first)) { - iter = to_merge_anchor_map->fsmap_.datamap_.erase(iter); - } else { - iter++; - } - } - } - - for (auto &be_merged_anchor_map : parti_be_merged->fanchors_) { - for (auto iter = be_merged_anchor_map->fsmap_.datamap_.begin(); - iter != be_merged_anchor_map->fsmap_.datamap_.end();) { - if (!parti_be_merged->buf_alloc_.g2b_map_.haskey(iter->first)) { - iter = be_merged_anchor_map->fsmap_.datamap_.erase(iter); - } else { - iter++; - } - } - } - - // append tunable op without owner anchor manually if necessary - for (auto &op : parti_be_merged->committed_ops_) { - auto op_raw = op.get(); - // Due to current tunable template implement, the beginning tunable op - // of partition does not belong to any anchor. - if (op->isa() && - parti_be_merged->op_anchor_map_.find(op_raw) == - parti_be_merged->op_anchor_map_.end()) { - // append op to anchor - max_to_merge_anchor_map->append_content(op_raw); - // append op to partition - pa_to_merge->op_anchor_map_[op_raw] = max_to_merge_anchor_map; - } - } - - // append inner loop anchor - for (auto &be_merged_anchor_map : parti_be_merged->fanchors_) { - // skip outer anchor - if (std::any_of( - outer_loops_be_merged.begin(), - outer_loops_be_merged.begin() + merged_loop_size, - [&be_merged_anchor_map, &parti_be_merged](const for_loop &loop) { - return parti_be_merged->get_anchor_inside_loop( - loop, be_merged_anchor_map->is_input_anchor()) == - be_merged_anchor_map; - })) { - continue; - } - be_merged_anchor_map->attach_parent_anchor(max_to_merge_anchor_map, - max_be_merged_anchor_map); - pa_to_merge->append_fusion_anchor(be_merged_anchor_map); - } - - // merge outer loop anchor - for (size_t i = 0; i < merged_loop_size; i++) { - node_remap[outer_loops_be_merged[i]->var_.impl] = - outer_loops_to_merge[i]->var_.impl; - node_remap[outer_loops_be_merged[i].impl] = outer_loops_to_merge[i].impl; - auto be_merged_anchor_map = - parti_be_merged->get_anchor_inside_loop(outer_loops_be_merged[i]), - to_merge_anchor_map = - pa_to_merge->get_anchor_inside_loop(outer_loops_to_merge[i]); - if (be_merged_anchor_map && to_merge_anchor_map) { - to_merge_anchor_map->merge(be_merged_anchor_map); - // reset op anchor map if neccesary - if (i == merged_loop_size - 1) { - for (auto &op_anchor_pair : parti_be_merged->op_anchor_map_) { - if (op_anchor_pair.second == be_merged_anchor_map) { - op_anchor_pair.second = to_merge_anchor_map; - } - } - } - } - } - - /* * * * * * * * * * * * * * * * * - * Step 3: Merge buf_alloc_ - * * * * * * * * * * * * * * * * */ - pa_to_merge->buf_alloc_.merge( - parti_be_merged->buf_alloc_, buffer_map, - std::make_pair(max_to_merge_anchor_map, max_be_merged_anchor_map)); - - /* * * * * * * * * * * * * * * * * * - * Step 4: Replace expr involving: - * 1. func->body - * 2. fanchor->fsmap->slice_range - * * * * * * * * * * * * * * * * * */ - for (auto &buf_pair : buffer_map) { - node_remap.insert( - std::make_pair(buf_pair.first.impl, buf_pair.second.impl)); - } - // create mxp inplace replacer - mxp_replacer_t mxp_reper(node_remap); - // 1. func->body - mxp_reper.replace_func(pa_to_merge->func_); - // 2. fanchor->fsmap->slice_range - mxp_reper.replace_anchor(pa_to_merge->fanchors_); - - /* * * * * * * * * * * * * * * * * - * Step 5: Merge op_anchor_map_ - * * * * * * * * * * * * * * * * */ - for (auto &op_anchor_pair : parti_be_merged->op_anchor_map_) { - // override existed ones - pa_to_merge->op_anchor_map_[op_anchor_pair.first] = op_anchor_pair.second; - } - - // erase joint op in op_anchor_map - pa_to_merge->op_anchor_map_.erase(joint_op.get()); - - /* * * * * * * * * * * * * * * * * - * Step 6: Merge op_ - * * * * * * * * * * * * * * * * */ - // call base merge - // move 'ops' from src to target and set 'merged_to' of src to be target - pa_to_merge->fusion_partition_t::merge( - static_cast(parti_be_merged)->shared_from_this()); - - // Merge commited ops - pa_to_merge->committed_ops_.insert(pa_to_merge->committed_ops_.end(), - parti_be_merged->committed_ops_.begin(), - parti_be_merged->committed_ops_.end()); - - // clear merged parti - parti_be_merged->clear(); -} - -static size_t get_great_common_loop_size(const std::vector &loop_A, - const std::vector &loop_B) { - // great common size - auto gcs = std::min(loop_A.size(), loop_B.size()); - - size_t merged_loop_size = 0; - for (; merged_loop_size < gcs; merged_loop_size++) { - auto &to_merge = loop_A[merged_loop_size]; - auto &be_merged = loop_B[merged_loop_size]; - if (!(to_merge->iter_begin_.isa() && - to_merge->step_.isa() && - be_merged->iter_begin_.isa() && - be_merged->step_.isa())) { - break; - } - if (!to_merge->iter_end_.isa() && - slice_expr_equals(to_merge->iter_end_, be_merged->iter_end_)) { - continue; // for dynamic - } else if (!(to_merge->iter_end_.isa() && - be_merged->iter_end_.isa())) { - break; - } - auto A_begin = get_expr_as_int(to_merge->iter_begin_), - A_end = get_expr_as_int(to_merge->iter_end_), - A_step = get_expr_as_int(to_merge->step_), - B_begin = get_expr_as_int(be_merged->iter_begin_), - B_end = get_expr_as_int(be_merged->iter_end_), - B_step = get_expr_as_int(be_merged->step_); - auto A_num_threads = to_merge->num_threads_; - auto B_num_threads = be_merged->num_threads_; - if (A_begin != B_begin || A_end != B_end || A_step != B_step || - A_num_threads != B_num_threads) - break; - } - return merged_loop_size; -} - -/** - * find nested parallel for. E.g. - * pfor(){ - * tensor a; // optional - * pfor(){ - * } - * } - * */ -class nested_pfor_finder_t : public ir_viewer_t { -public: - using ir_viewer_t::dispatch; - using ir_viewer_t::view; - // return whether nested pfor exist - bool operator()(for_loop_c v) { - ir_viewer_t::dispatch(std::move(v)); - return pfor_cnt_ > 1; - } - expr_c dispatch(expr_c v) override { return v; } - - void view(for_loop_c f) override { - // check pfor - if (f->kind_ == for_type::PARALLEL && f->num_threads_ > 0) { - pfor_cnt_++; - } - // to be faster - if (pfor_cnt_ > 1) - return; - ir_viewer_t::view(f); - } - -private: - int pfor_cnt_ = 0; -}; - -static bool -try_merge_mixed_parti_parallel_inners(mixed_parti_t *pa_to_merge, - mixed_parti_t *parti_be_merged) { - pa_to_merge = pa_to_merge->get_root(), - parti_be_merged = parti_be_merged->get_root(); - - auto outer_loops_to_merge = pa_to_merge->get_outer_loops(), - outer_loops_be_merged = parti_be_merged->get_outer_loops(); - - if (outer_loops_to_merge.empty() || outer_loops_be_merged.empty()) - return false; - - auto merged_loop_size = - get_great_common_loop_size(outer_loops_to_merge, outer_loops_be_merged); - if (!merged_loop_size) - return false; // no outer loop can be merged - - // validate axis binding - merged_loop_size = check_parti_loop_axis_binding(parti_be_merged, pa_to_merge, - merged_loop_size); - SC_MODULE_INFO << "After axis binding, num loops to merge: " - << merged_loop_size; - if (!merged_loop_size) - return false; // no outer loop can be merged - - // change the loop var name of the first for. - // first for: for m_s, for n_s. second for: for m_s_o, m_s_i, n_s. - // merge directly: for m_s, for n_s, for n_s. - // add suffix and merge: for m_s_0, for n_s_0, for n_s. - for (size_t i = 0; i < merged_loop_size; ++i) { - std::string &varname = - outer_loops_to_merge[i]->var_.static_as()->name_; - varname += "_0"; - } - - SC_MODULE_INFO << "parallel merging two partition:"; - SC_MODULE_INFO << pa_to_merge->func_; - SC_MODULE_INFO << parti_be_merged->func_; - - if (check_parti_dep(pa_to_merge, parti_be_merged) == parti_dep::no_dep) { - auto &to_merge_body = outer_loops_to_merge[merged_loop_size - 1]->body_; - /** - * The thread-shared buffer of the previous pfor and the thread-shared - * buffer of the next pfor may share the same memory location after - * buffer scheduling and hoist. After removal of barrier, some threads - * may work on the previous pfor and others may work on the next pfor, - * and they share the same memory localtion. This will cause race - * condition. To avoid that, we need to check that: - * 1. there are no tensors defined in the immediate body of the merged - * pfor. - * 2. the buffers inside of the child pfor of the merged pfor must be - * the most inner loop (i.e. merged pfor must be second most inner - * loop). This is to avoid hoisting of the tensors. */ - bool barrier_can_remove = true; - if (to_merge_body.isa()) { - for (auto &s : to_merge_body.static_as()->seq_) { - if (s.isa()) { - // Case 1: if tensor node is defined - if (s.static_as()->var_.isa()) { - barrier_can_remove = false; - break; - } - } else if (s.isa()) { - // Case 2: nested pfor and potential hoist buffer - if (nested_pfor_finder_t()(s.static_as())) { - barrier_can_remove = false; - break; - } - } - } - } - if (barrier_can_remove) { - auto last_for = get_last_loop_in_body(to_merge_body); - if (last_for.defined()) { - last_for->attr()[stmt_attr_key::no_post_barrier] = true; - } - } - } - - // try to add prefetch code - if (pa_to_merge->ctx_->flags_.prefetch_) { - auto op_first = pa_to_merge->committed_ops_[0]; - auto op_second = parti_be_merged->committed_ops_[0]; - if (auto second_prefetch = - op_second->dyn_cast()) { - int64_t prefetch_insertion_point = -1; - auto last_for_parent = find_last_parallel_for( - outer_loops_to_merge[merged_loop_size - 1]->body_.static_as(), - prefetch_insertion_point); - - std::vector in_slice; - in_slice.reserve(op_second->get_inputs().size()); - for (auto &inp : op_second->get_inputs()) { - auto second_in = parti_be_merged->buf_alloc_.g2b_map_.get(inp); - in_slice.emplace_back(second_in); - } - auto prefetch_idx = - second_prefetch->query_prefetch(pa_to_merge->ctx_, false, in_slice); - for (auto itr = prefetch_idx.begin(); itr != prefetch_idx.end();) { - auto input = op_second->get_inputs()[*itr]->producer_owner_; - if (pa_to_merge->contains(input)) { - itr = prefetch_idx.erase(itr); - } else { - ++itr; - } - } - if (prefetch_insertion_point != -1 && !prefetch_idx.empty()) { - std::vector out_seq; - second_prefetch->generate_prefetcher_and_set_idle( - pa_to_merge->ctx_, false, in_slice, prefetch_idx, out_seq); - auto &outer_body = last_for_parent->seq_; - outer_body.insert(outer_body.begin() + prefetch_insertion_point, - out_seq.begin(), out_seq.end()); - } - } - } - - auto outer_loops_merged_target = outer_loops_to_merge[merged_loop_size - 1]; - auto max_to_merge_anchor_map = - pa_to_merge->get_anchor_inside_loop(outer_loops_merged_target); - if (!max_to_merge_anchor_map) { - auto s = builder::make_stmts_unattached({}).checked_as(); - add_parent_node(s, outer_loops_merged_target->body_); - outer_loops_merged_target->body_.checked_as()->seq_.emplace_back(s); - // dummy fsmap, the tensor belongs to this scope will not be shrinked - fslice_map fsmap; - max_to_merge_anchor_map = std::make_shared(s, fsmap); - pa_to_merge->append_fusion_anchor(max_to_merge_anchor_map); - } - - pa_to_merge->func_->name_ += - "_parallel_merge_" + parti_be_merged->func_->name_; - - merge_parti_impl(pa_to_merge, parti_be_merged, merged_loop_size); - - SC_MODULE_INFO << "parallel merging result:"; - SC_MODULE_INFO << pa_to_merge->func_; - - return true; -} - -static bool try_merge_mixed_parti_parallel(mixed_parti_t *A, mixed_parti_t *B) { - A = A->get_root(), B = B->get_root(); - if (A == B) - return false; - if (!A->func_.get() || !B->func_.get()) - return false; - if (!check_parti_connectionship(A, B)) - return false; - if (check_parti_ring_risk(A, B)) - return false; - - auto dep = check_parti_dep(A, B); - COMPILE_ASSERT(dep != parti_dep::inter_dep, - "inter-dependency is not expected"); - - auto append_parti = (dep == parti_dep::l_dep_r) ? A : B, - target_parti = (dep == parti_dep::l_dep_r) ? B : A; - SC_MODULE_INFO << "Start try_merge_mixed_parti_parallel: " << "Target: " - << target_parti->func_->name_ - << ", Append: " << append_parti->func_->name_; - - auto outer_loops_target = target_parti->get_outer_loops(), - outer_loops_append = append_parti->get_outer_loops(); - if (outer_loops_target.empty() || outer_loops_append.empty()) - return false; - - auto outermost_loop_target = outer_loops_target[0], - outermost_loop_append = outer_loops_append[0]; - - // check parallel loops attr. - if (!(outermost_loop_target->kind_ == for_type::PARALLEL && - (outermost_loop_target->num_threads_ > 0)) || - !(outermost_loop_append->kind_ == for_type::PARALLEL && - (outermost_loop_append->num_threads_ > 0))) { - return false; - } - - // check axis binding on the outmost axis - if (!check_parti_loop_axis_binding(append_parti, target_parti, 1)) { - return false; - } - // cannot do parallel merge when loop split granularity are different - if (outermost_loop_target->attr().get_or_else( - stmt_attr_key::parallel_merge_loop_granularity, 1) != - outermost_loop_append->attr().get_or_else( - stmt_attr_key::parallel_merge_loop_granularity, 1)) { - return false; - } - - if (outermost_loop_target->iter_begin_.isa() && - outermost_loop_target->iter_end_.isa() && - outermost_loop_target->step_.isa() && - outermost_loop_append->iter_begin_.isa() && - outermost_loop_append->iter_end_.isa() && - outermost_loop_append->step_.isa()) { - auto target_begin = get_expr_as_int(outermost_loop_target->iter_begin_), - target_end = get_expr_as_int(outermost_loop_target->iter_end_), - target_step = get_expr_as_int(outermost_loop_target->step_), - append_begin = get_expr_as_int(outermost_loop_append->iter_begin_), - append_end = get_expr_as_int(outermost_loop_append->iter_end_), - append_step = get_expr_as_int(outermost_loop_append->step_); - // start and step must be the same - if (!(target_begin == append_begin && target_step == append_step)) { - return false; - } - if (target_end != - append_end) { // if the end is not same, then we try to split - // the first on num_threads_ to merge, but we - // require num_iters == num_threads - if (append_parti->contain_convolution() || - target_parti->contain_convolution()) { - // currently do not support split num_threads_ on conv - return false; - } - - if (target_begin != 0 || target_step != 1 || append_begin != 0 || - append_step != 1) { - // Only support begin is 0 and step is 1 - return false; - } - if (target_end == outermost_loop_target->num_threads_ && - append_end == outermost_loop_append->num_threads_) { - // For the two fors, same start, same step, different end => - // different num_iters. - // For each for, num_iters == num_threads. So We split the for - // on num_threads. - // can not split the outermost loop in imbalanced cases - if (!outermost_loop_target->attr().get_or_else( - stmt_attr_key::parallel_loop_balanced, true) || - !outermost_loop_append->attr().get_or_else( - stmt_attr_key::parallel_loop_balanced, true)) { - SC_MODULE_INFO << "The outermost loop is imbalanced, can " - "not be split"; - return false; - } - if (outermost_loop_target->num_threads_ > - outermost_loop_append->num_threads_ && - outermost_loop_target->num_threads_ % - outermost_loop_append->num_threads_ == - 0) { - if (outermost_loop_append->num_threads_ == 1) { - // in this case, after split, outermost_loop_target - // num_threads will be 1 - return false; - } - int64_t num_groups = outermost_loop_target->num_threads_ / - outermost_loop_append->num_threads_; - target_parti->try_split_outermost_loop_on_num_threads(num_groups); - return try_merge_mixed_parti_parallel_inners(target_parti, - append_parti); - } else if (outermost_loop_append->num_threads_ > - outermost_loop_target->num_threads_ && - outermost_loop_append->num_threads_ % - outermost_loop_target->num_threads_ == - 0) { - if (outermost_loop_target->num_threads_ == 1) { - // in this case, after split, outermost_loop_append - // num_threads will be 1 - return false; - } - int64_t num_groups = outermost_loop_append->num_threads_ / - outermost_loop_target->num_threads_; - append_parti->try_split_outermost_loop_on_num_threads(num_groups); - return try_merge_mixed_parti_parallel_inners(target_parti, - append_parti); - } else { - return false; - } - } else { // do not support num_iters != num_threads case for now - return false; - } - } else { // the two fors have same (start, end, step) - // if num_threads are the same, merge them directly - if (outermost_loop_target->num_threads_ == - outermost_loop_append->num_threads_) { - return try_merge_mixed_parti_parallel_inners(target_parti, - append_parti); - } else { - return false; - } - } - } else { // if (start, end, step) is not constant - return false; - } -} - -static bool try_merge_mixed_parti_horizontally(mixed_parti_t *A, - mixed_parti_t *B) { - A = A->get_root(), B = B->get_root(); - if (A == B) - return false; - if (!A->func_.get() || !B->func_.get()) - return false; - if (!A->contain_tunable_op() || !B->contain_tunable_op()) - return false; - if (!check_parti_connectionship(A, B)) - return false; - if (check_parti_dep(A, B) != parti_dep::no_dep) - return false; - - auto outer_loops_A = A->get_outer_loops(), - outer_loops_B = B->get_outer_loops(); - if (outer_loops_A.empty() || outer_loops_B.empty()) - return false; - // skip parallel merge - if (outer_loops_A[0]->num_threads_ > 0 || outer_loops_B[0]->num_threads_ > 0) - return false; - // check cost model - if (!A->cost_->make_decision_for_parti(B, 1, parti_merge_kind::horizontal)) { - return false; - } - - SC_MODULE_INFO << "horizontally merging two partition:"; - SC_MODULE_INFO << A->func_; - SC_MODULE_INFO << B->func_; - - /* * * * * * * * * * * * * * * * * - * Step 0: Fuse func_ - * * * * * * * * * * * * * * * * */ - node_ptr_map node_remap; - schedule_loop_body(A->func_->body_, &node_remap); - schedule_loop_body(B->func_->body_, &node_remap); - - /* * * * * * * * * * * * * * * * * - * Step 1: Merge func_ - * * * * * * * * * * * * * * * * */ - auto new_body = make_stmt( - std::vector{A->func_->body_, B->func_->body_}); - - A->func_->body_ = std::move(new_body); - - /* * * * * * * * * * * * * * * * * - * Step 2: Merge fanchor_ - * * * * * * * * * * * * * * * * */ - A->append_fusion_anchor(B->fanchors_); - - /* * * * * * * * * * * * * * * * * - * Step 3: Merge buffer_ - * * * * * * * * * * * * * * * * */ - std::unordered_map buffer_map; - A->buf_alloc_.merge(B->buf_alloc_, buffer_map, - std::make_pair(nullptr, nullptr)); - - /* * * * * * * * * * * * * * * * * - * Step 4: Replace expr - * * * * * * * * * * * * * * * * */ - for (auto &buf_pair : buffer_map) { - node_remap.insert( - std::make_pair(buf_pair.first.impl, buf_pair.second.impl)); - } - mxp_replacer_t expr_reper(node_remap); - // 1. func->body - expr_reper.replace_func(A->func_); - // 2. fanchor->fsmap->slice_range - expr_reper.replace_anchor(A->fanchors_); - - /* * * * * * * * * * * * * * * * * - * Step 5: Merge op_anchor_map_ - * * * * * * * * * * * * * * * * */ - A->op_anchor_map_.insert(B->op_anchor_map_.begin(), B->op_anchor_map_.end()); - - // call base merge - A->fusion_partition_t::merge( - static_cast(B)->shared_from_this()); - - // Merge commited ops - A->committed_ops_.insert(A->committed_ops_.end(), B->committed_ops_.begin(), - B->committed_ops_.end()); - - auto &body = A->func_->body_; - /* * * * * * * * * * * * * * * * * - * Step 6: Same to Horizontal Merge - * * * * * * * * * * * * * * * * */ - COMPILE_ASSERT(body.isa(), "body has only one stmt."); - scope_flatten(body.checked_as(), -1); - std::vector &body_seq = body.checked_as()->seq_; - std::vector loops; - std::vector not_loops; - for (auto &st : body_seq) { - if (st.isa()) { - loops.push_back(st.checked_as()); - } else if (!st.isa()) { - not_loops.push_back(st); - } - } - std::vector new_seq(not_loops.begin(), not_loops.end()); - new_seq.insert(new_seq.end(), loops.begin(), loops.end()); - body_seq = std::move(new_seq); - COMPILE_ASSERT(loops.size() > 1, - "No need to horizontal fuse as parallel loop number is less " - "than " - "2."); - - constant_folder_t cf; - auto_caster_t ac; - for (size_t i = 1; i < loops.size(); i++) { - loops[0]->parallel_merge(body, loops[i]); - loops[0]->iter_end_ = cf(ac(loops[0]->iter_end_)).remove_const(); - } - - add_parent_node(loops[0], stmt()); - - A->func_->name_ += "_horizontal_merge_" + B->func_->name_; - - SC_MODULE_INFO << "horizontally merging result:"; - SC_MODULE_INFO << A->func_; - - // clear merged parti - B->clear(); - - return true; -} - -static sc_dim get_loops_range(const for_loop &loop) { - if (!(loop->iter_begin_.isa() && - loop->iter_end_.isa())) { - return (int64_t)0; - } - return get_expr_as_int(loop->iter_end_) - get_expr_as_int(loop->iter_begin_); -} - -static sc_dim get_loops_range_prod(const std::vector &loops) { - sc_dim prod_res = 1; - for (auto &l : loops) { - prod_res *= get_loops_range(l); - } - return prod_res; -} - -static void try_align_parti_outer_loops(mixed_parti_t *A, mixed_parti_t *B) { - auto outer_loops_A = A->get_outer_loops(), - outer_loops_B = B->get_outer_loops(); - if (outer_loops_A.empty() || outer_loops_B.empty()) - return; - if (outer_loops_A.size() == outer_loops_B.size()) - return; - - auto outermost_loop_A_range = get_loops_range(outer_loops_A[0]), - outermost_loop_B_range = get_loops_range(outer_loops_B[0]); - if (outermost_loop_A_range == 0 || outermost_loop_B_range == 0 || - outermost_loop_A_range == outermost_loop_B_range) { - return; - } else if (outermost_loop_A_range <= outermost_loop_B_range) { - B->try_split_outermost_loop(outermost_loop_A_range); - } else { - A->try_split_outermost_loop(outermost_loop_B_range); - } -} - -// check brgemm pre-op fusion -static bool -try_merge_brgemm_and_preop_parti(mixed_parti_t *A, mixed_parti_t *B, - const sc_op_ptr &joint_op = nullptr) { - if (!require_fusion_level(A->ctx_, fusion_opt_level::lv2)) - return false; - A = A->get_root(), B = B->get_root(); - if (A == B) - return false; - if (!A->func_.get() || !B->func_.get()) - return false; - if (!joint_op && !check_parti_connectionship(A, B)) - return false; - if (check_parti_ring_risk(A, B)) - return false; - auto dep_flag = check_parti_dep(A, B); - if (dep_flag == parti_dep::inter_dep) - return false; - - mixed_parti_t *brgemm_parti, *preop_parti; - if (A->contain_tunable_op() && B->contain_elemwise_op_only()) { - brgemm_parti = A; - preop_parti = B; - } else if (B->contain_tunable_op() && A->contain_elemwise_op_only()) { - brgemm_parti = B; - preop_parti = A; - } else { - return false; - } - - if (check_parti_dep(brgemm_parti, preop_parti) == parti_dep::l_dep_r) - return false; - - SC_MODULE_INFO << "pre-op merging two partition:"; - SC_MODULE_INFO << A->func_; - SC_MODULE_INFO << B->func_; - - /* * * * * * * * * * * * * * * * * - * Step 1: pre infer elementwise op using brgemm parti fusion anchor - * * * * * * * * * * * * * * * * */ - for (auto &brgemm_parti_anchor : brgemm_parti->fanchors_) { - fslice_map tmp_fsmap; - for (auto iter = brgemm_parti_anchor->fsmap_.datamap_.begin(); - iter != brgemm_parti_anchor->fsmap_.datamap_.end();) { - if (!brgemm_parti->buf_alloc_.g2b_map_.haskey(iter->first)) { - auto pre_fuse_last_op = iter->first->producer_owner_; - if (preop_parti->contains(pre_fuse_last_op)) { - // set known slice range and prepare for pre-infer slice - // range - tmp_fsmap.get(iter->first) = iter->second; - // pre infer ops list - std::vector pre_infer_ops; - for (auto &op : preop_parti->committed_ops_) { - if (op.get() == pre_fuse_last_op || - preop_parti->dep_m_->lookup(op.get(), pre_fuse_last_op) == 1) { - pre_infer_ops.emplace_back(op.get()); - } - } - COMPILE_ASSERT(pre_infer_ops.back() == pre_fuse_last_op, - "The first pre infer op should be the last pre " - "fuse op") - // inverse visit, pre infer from the last op to first one - for (auto op_iter = pre_infer_ops.rbegin(); - op_iter != pre_infer_ops.rend(); op_iter++) { - auto status = - (*op_iter)->dyn_cast()->pre_infer_slice_ranges( - brgemm_parti->ctx_, tmp_fsmap); - assert(status == infer_status_code::OK); - } - iter++; - } else { - iter = brgemm_parti_anchor->fsmap_.datamap_.erase(iter); - } - } else { - iter++; - } - } - // move fsmap - brgemm_parti_anchor->fsmap_.datamap_.insert(tmp_fsmap.datamap_.begin(), - tmp_fsmap.datamap_.end()); - } - - /* * * * * * * * * * * * * * * * * - * Step 2: Commit ops in pre-op parti into brgemm parti by original order - * * * * * * * * * * * * * * * * */ - brgemm_parti->func_->name_ += "_preop_merge"; - // set pre-op fusion attr - preop_parti->committed_ops_.front()->attrs_.set( - mixed_partition_hint::pre_fuse_begin_op, true); - for (auto &op_in_preop_parti : preop_parti->committed_ops_) { - brgemm_parti->add(op_in_preop_parti); - } - // remove pre-op fusion attr - preop_parti->committed_ops_.front()->attrs_.remove( - mixed_partition_hint::pre_fuse_begin_op); - - // erase joint op in op_anchor_map - brgemm_parti->op_anchor_map_.erase(joint_op.get()); - - /* * * * * * * * * * * * * * * * * - * Step 3: Merge op_ - * * * * * * * * * * * * * * * * */ - // call base merge - brgemm_parti->fusion_partition_t::merge( - static_cast(preop_parti)->shared_from_this()); - - SC_MODULE_INFO << "pre-op merging result:"; - SC_MODULE_INFO << brgemm_parti->func_; - - // clear merged parti - preop_parti->clear(); - - return true; -} - -static bool try_merge_mixed_parti_vertically(mixed_parti_t *A, mixed_parti_t *B, - const sc_op_ptr &joint_op, - bool keep_outerloop_size = false) { - A = A->get_root(), B = B->get_root(); - if (A == B) - return false; - if (!A->func_.get() || !B->func_.get()) - return false; - // in avoid conflict with parallel merge - if (A->contain_nested_parallel_for() || B->contain_nested_parallel_for()) - return false; - if (!joint_op && !check_parti_connectionship(A, B)) - return false; - if (!joint_op && check_parti_ring_risk(A, B)) - return false; - auto dep_flag = check_parti_dep(A, B); - // if two partition inter-depends each other, could not merge them - if (dep_flag == parti_dep::inter_dep) - return false; - // if both two partitions have input anchor, could not merge them - if (A->contain_input_anchor() && B->contain_input_anchor()) - return false; - mixed_parti_t *pa_to_merge = nullptr, *parti_be_merged = nullptr; - - pa_to_merge = (dep_flag == parti_dep::l_dep_r) ? B : A; - parti_be_merged = (dep_flag == parti_dep::l_dep_r) ? A : B; - - auto outer_loops_to_merge = pa_to_merge->get_outer_loops(), - outer_loops_be_merged = parti_be_merged->get_outer_loops(); - - auto merged_loop_size = - get_great_common_loop_size(outer_loops_to_merge, outer_loops_be_merged); - - if (!merged_loop_size) - return false; - - // validate axis binding - merged_loop_size = check_parti_loop_axis_binding(parti_be_merged, pa_to_merge, - merged_loop_size); - - if (!merged_loop_size || (keep_outerloop_size && - (merged_loop_size != outer_loops_to_merge.size() || - merged_loop_size != outer_loops_be_merged.size()))) - return false; - - // check cost model - if (!pa_to_merge->cost_->make_decision_for_parti( - parti_be_merged, merged_loop_size, parti_merge_kind::vertical)) - return false; - - if (auto max_to_merge_anchor_map = pa_to_merge->get_anchor_inside_loop( - outer_loops_to_merge[merged_loop_size - 1])) { - if (joint_op && - std::any_of( - joint_op->get_inputs().begin(), joint_op->get_inputs().end(), - [&max_to_merge_anchor_map, - &pa_to_merge](const graph_tensor_ptr &inp) { - return pa_to_merge->contains(inp->producer_owner_) && - max_to_merge_anchor_map->blocked_gt_set_.find(inp) != - max_to_merge_anchor_map->blocked_gt_set_.end(); - })) - return false; - } else { - return false; - } - - SC_MODULE_INFO << "merging two partition:"; - SC_MODULE_INFO << A->func_; - SC_MODULE_INFO << B->func_; - - pa_to_merge->func_->name_ += "_merge_" + parti_be_merged->func_->name_; - - merge_parti_impl(pa_to_merge, parti_be_merged, merged_loop_size, joint_op); - - SC_MODULE_INFO << "Merging result:"; - SC_MODULE_INFO << pa_to_merge->func_; - - return true; -} - -// usually used by crossover dispatcher -static bool try_merge_mixed_parti_vertically(mixed_parti_t *A, - mixed_parti_t *B) { - A = A->get_root(), B = B->get_root(); - if (A == B) - return false; - // if A and B are forked, do not merge them - if (check_parti_forked(A, B)) - return false; - // skip single op partition - if (A->is_single_op_parti() || B->is_single_op_parti()) - return false; - bool image_affinity = A->contain_convolution() && B->contain_convolution(); - return try_merge_mixed_parti_vertically(A, B, nullptr, !image_affinity); -} - -static bool try_merge_mixed_parti_with_joint_op(const mixed_parti_t::ptr &A, - const mixed_parti_t::ptr &B, - const sc_op_ptr &joint_op) { - mixed_parti_t *default_lhs, *default_rhs; - // If no dependence, first execute the partition with more tunable ops - if (joint_op->isa() || - (A->contain_tunable_op() && - (!B->contain_tunable_op() || (B->count_op_with_type() > - A->count_op_with_type())))) { - default_lhs = B.get(); - default_rhs = A.get(); - } else { - default_lhs = A.get(); - default_rhs = B.get(); - } - auto &ctx = A->ctx_; - // if pre-op fusion succeed, skip vertical merge - return try_merge_brgemm_and_preop_parti(default_lhs, default_rhs, joint_op) || - try_merge_mixed_parti_vertically(default_lhs, default_rhs, joint_op); -} - -mixed_parti_t::mixed_parti_t(const context_ptr &ctx, const sc_op_ptr &op, - const dep_mat_ptr &dep_m) - : dep_m_(dep_m), ctx_(ctx) { - auto &graph = op->get_owner_graph(); - if (graph.is_dynamic()) { - cost_ = std::make_shared( - this, graph.attrs_.get_or_else("temp.dynamic_fusion_policy", - dynamic_fusion_policy_t::max_fusion)); - } else { - cost_ = std::make_shared(this); - } - if (!op->isa() && !is_reshape_op(op.get())) { - SC_MODULE_INFO << "================ create new partition: " << op->op_name_ - << "_" << op->logical_op_id_ << " ================"; - auto mixed_op = op->dyn_cast(); - mixed_op->create_mixed_partition(this); - func_->name_ = op->op_name_ + std::to_string(op->logical_op_id_); - SC_MODULE_INFO << func_; - } - ops.insert(op); - committed_ops_.emplace_back(op); -} - -mixed_parti_t::mixed_parti_t(const context_ptr &ctx, const func_t &func, - const fusion_anchor_mgr_t &fmgr, - const sc_graph_t &graph) - : ctx_(ctx), func_(func) { - cost_ = std::make_shared(this); - dep_m_ = std::make_shared(graph); - append_fusion_anchor(fmgr.get_fusion_anchor()); -} - -bool mixed_parti_t::is_ok_to_add(sc_op *op) { - if (merged_to) { - return get_root()->is_ok_to_add(op); - } - if (empty()) { - return false; - } - if (!fusion_partition_t::is_ok_to_add(op, (*dep_m_))) { - SC_MODULE_INFO << op->op_name_ << "_" << op->logical_op_id_ - << " fail to add partition: " << func_->name_ - << ", due to potential graph " - "dependency ring risk"; - return false; - } - // TODO(yunfei): when all tunable ops completely refactored to managed - // threads mode, remove this if. - if (op->isa() && contain_nested_parallel_for()) { - return false; - } - // if not acceptable. return false - if (!op->isa()) - return false; - // search suitable anchor for op - auto mixed_op = op->dyn_cast(); - mixed_op->search_anchor(this); - if (!ready_for_op(op)) { - SC_MODULE_INFO << op->op_name_ << "_" << op->logical_op_id_ - << " fail to add partition: " << func_->name_ - << ", due to no suitable anchor found"; - return false; - } - return true; -} - -bool mixed_parti_t::ready_for_op(sc_op *op) const { - if (merged_to) { - return get_root()->ready_for_op(op); - } - return op_anchor_map_.find(op) != op_anchor_map_.end(); -} - -void mixed_parti_t::set_anchor_for_op(sc_op *op, - const fusion_anchor_ptr &fanchor_map) { - if (merged_to) { - get_root()->set_anchor_for_op(op, fanchor_map); - return; - } - - auto iter = op_anchor_map_.find(op); - // update op_anchor_map_ - if (iter != op_anchor_map_.end()) { - // auto skip - if (iter->second == fanchor_map) - return; - auto res = cmp_op_anchor(op, iter->second, fanchor_map); - // if new anchor is more smaller than the current one - if (res == cmp_res::l_larger_r) { - // overwrite new anchor - op_anchor_map_[op] = fanchor_map; - } else if (res == cmp_res::equal) { - // Except for reduce_collect_op_t with COPY kind because this mode - // will elimiate first required axis - if (auto red_coll = op->dyn_cast()) { - if (red_coll->op_ == reduce_collect_op_t::kind::COPY) - return; - } - // if sub anchor is equal to parent one, overwrite it - if (iter->second->is_parent_for(fanchor_map)) { - // overwrite inner anchor - op_anchor_map_[op] = fanchor_map; - } - } - } else { - op_anchor_map_[op] = fanchor_map; - } -} - -bool mixed_parti_t::add(const sc_op_ptr &op) { - if (merged_to) { - return get_root()->add(op); - } - // pre-check anchor for op in avoid of throw assert during following - // appending stage - search_op_anchor_in_parti(op.get(), this); - // if no anchor is ready, return false - if (!ready_for_op(op.get())) { - SC_MODULE_INFO << op->op_name_ << "_" << op->logical_op_id_ - << " fail to add partition: " << func_->name_ - << ", due to no suitable anchor found"; - return false; - } - /* adding op to partition */ - SC_MODULE_INFO << "================ adding op: " << op->op_name_ << "_" - << op->logical_op_id_ << " to partition: " << func_->name_ - << " ================"; - if (op->need_dynamic_internal_query()) { - dyn_inter_ = std::make_shared(ctx_); - } - // if not acceptable. return false - if (!op->isa()) - return false; - // append to partition - auto mixed_op = op->dyn_cast(); - mixed_op->append_mixed_partition(this); - func_->name_ += "_" + op->op_name_ + std::to_string(op->logical_op_id_); - SC_MODULE_INFO << func_; - ops.insert(op); - committed_ops_.emplace_back(op); - return true; -} - -void mixed_parti_t::append_fusion_anchor(const fusion_anchor_ptr &fanchor) { - auto cur = try_convert_anchor(ctx_, fanchor); - cur->binded_mxp_ = this; - fanchors_.emplace_back(cur); -} - -fusion_anchor_ptr mixed_parti_t::lookup_anchor_map(sc_op *op, - bool throw_assert) const { - if (merged_to) { - return get_root()->lookup_anchor_map(op); - } - auto iter = op_anchor_map_.find(op); - auto ret = (iter != op_anchor_map_.end()) ? iter->second : nullptr; - if (throw_assert) { - COMPILE_ASSERT(ret, "No dispatched fusion anchor map found for " - << op->op_name_ - << " in this partition, please try to search it " - "firstly"); - } - return ret; -} - -fusion_anchor_ptr mixed_parti_t::lookup_anchor_map(const stmts &ss) const { - if (merged_to) { - return get_root()->lookup_anchor_map(ss); - } - auto iter = std::find_if(fanchors_.begin(), fanchors_.end(), - [&ss](const fusion_anchor_ptr &amap) { - return ss.ptr_same(amap->anchor_position_); - }); - return (iter != fanchors_.end()) ? (*iter) : nullptr; -} - -std::vector mixed_parti_t::lookup_sub_anchor_map( - const fusion_anchor_ptr &parent_fanchor) const { - if (merged_to) { - return get_root()->lookup_sub_anchor_map(parent_fanchor); - } - std::vector subs; - for (auto &fanc : fanchors_) { - if (fanc->parent_ == parent_fanchor) { - subs.emplace_back(fanc); - } - } - return subs; -} - -void mixed_parti_t::clear_fanchor(fusion_anchor_ptr &fanchor) { - stmt anchor = fanchor->anchor_position_; - if (!fanchor->isa()) { - COMPILE_ASSERT(anchor.checked_as()->seq_.empty(), - "Could not remove this fanchor, because it is not empty"); - } else { - for (auto &sub_group : anchor.checked_as()->seq_) { - COMPILE_ASSERT(sub_group.checked_as()->seq_.empty(), - "Could not remove this fanchor, because it is not empty"); - } - } - stmt parent = get_parent_node(anchor); - auto ss_parent = parent.checked_as(); - // clear empty if_node outside iter anchor if necessary - if (fanchor->isa() && ss_parent->seq_.size() == 1 && - get_parent_node(parent).isa()) { - auto if_node = get_parent_node(parent); - // redirect - parent = get_parent_node(if_node); - ss_parent = parent.checked_as(); - anchor = if_node; - } - // find anchor iter - std::vector::iterator anchor_iter = - std::find_if(ss_parent->seq_.begin(), ss_parent->seq_.end(), - [anchor](stmt &s) { return s.ptr_same(anchor); }); - COMPILE_ASSERT(anchor_iter != ss_parent->seq_.end(), - "Could not found anchor in current parent stmts"); - // remove anchor - ss_parent->seq_.erase(anchor_iter); - // reset fanchor - fanchor->anchor_position_ = stmts(); - fanchor->fsmap_.clear(); -} - -void mixed_parti_t::clear_fanchors() { - for (auto iter = fanchors_.begin(); iter != fanchors_.end();) { - auto fanchor = *iter; - if (!fanchor->anchor_position_->size()) { - clear_fanchor(fanchor); - iter = fanchors_.erase(iter); - } else { - iter++; - } - } -} - -std::vector mixed_parti_t::get_outer_loops(fusion_anchor_ptr fanchor, - bool skip_def_node) const { - if (merged_to) { - return get_root()->get_outer_loops(fanchor); - } - std::vector outer_loops; - if (!func_) - return outer_loops; - auto body = func_->body_; - - fusion_anchor_ptr target_fanchor = std::move(fanchor); - while (target_fanchor && target_fanchor->parent_) { - target_fanchor = target_fanchor->parent_; - } - - if (body.isa()) { - // copy - auto ss = body.static_as()->seq_; - if (skip_def_node) { - for (auto iter = ss.begin(); iter != ss.end();) { - if (iter->isa()) { - iter = ss.erase(iter); - } else { - break; - } - } - } - // pop back unnecessary item including `return` or unused anchor - if (!ss.empty()) { - auto backs = ss.back(); - if (backs.isa()) - ss.pop_back(); - backs = ss.back(); - if (backs.isa()) { - auto last_ss = backs.static_as(); - if (last_ss->seq_.empty()) { - auto anchor_map = lookup_anchor_map(last_ss); - if (anchor_map && anchor_map != target_fanchor) { - ss.pop_back(); - } - } - } - } - if (ss.size() == 1) { - auto st = ss[0]; - if (st.isa()) { - auto loop = st.static_as(); - while (loop.defined()) { - outer_loops.emplace_back(loop); - loop = get_next_inner_loop_with_anchor(loop, target_fanchor); - } - } - } - } - - return outer_loops; -} - -/** - * for_loop(){ - * // input anchor - * {} - * // body - * ... - * // output anchor - * {} - * } - * */ -fusion_anchor_ptr -mixed_parti_t::get_anchor_inside_loop(const for_loop &loop, - bool input_anchor) const { - auto &body = loop->body_; - if (body.isa()) { - auto ss = body.static_as(); - for (auto s : ss->seq_) { - // find anchor inside if-node - if (s.isa()) { - auto if_node = s.static_as(); - if (!if_node->then_case_.defined() || if_node->else_case_.defined()) - continue; - auto then_stmts = if_node->then_case_.static_as(); - if (then_stmts->seq_.size() == 1) - s = then_stmts->seq_[0]; - else - continue; - } - if (!s.isa()) - continue; - auto inner_ss = s.static_as(); - auto anchor_map = lookup_anchor_map(inner_ss); - if (anchor_map) { - // check anchor type - if (input_anchor != anchor_map->is_input_anchor()) - continue; - return anchor_map; - } - } - } - return nullptr; -} - -for_loop mixed_parti_t::get_next_inner_loop_with_anchor( - const for_loop &cur_loop, const fusion_anchor_ptr &target_fanchor) const { - if (cur_loop->body_.isa()) { - return cur_loop->body_.checked_as(); - } else if (cur_loop->body_.isa()) { - auto ss = cur_loop->body_.static_as(); - bool target_anchor_found = false; - for_loop next_loop = for_loop(); - for (auto &s : ss->seq_) { - if (s.isa()) { - auto inner_ss = s.static_as(); - if (inner_ss->seq_.empty()) { - auto anchor_map = lookup_anchor_map(inner_ss); - if (anchor_map) { - if (anchor_map == target_fanchor) { - target_anchor_found = true; - break; - } - continue; - } else { - next_loop = for_loop(); - break; - } - } else { - next_loop = for_loop(); - break; - } - } else if (s.isa() && !next_loop.defined()) { - next_loop = s.checked_as(); - } else if (s.cast() - .map([](const evaluate &v) { - return v->value_.as(); - }) - .filter([](const intrin_call &v) { - return v->type_ == intrin_type::set_thread_idle_func; - }) - .has_value()) { - // When prefetch is enabled, a prefetcher function call - // `evaluate{set_thread_idle_func(...)}` would have been added - // to the loop. We should ignore it when finding next anchor. - continue; - } else { - next_loop = for_loop(); - break; - } - } - return target_anchor_found ? for_loop() : next_loop; - } - return for_loop(); -} - -class var_replacer_t : public ir_visitor_t { -public: - var_replacer_t(const std::unordered_map &remap) - : remap_(remap) {} - std::unordered_map remap_; - using ir_visitor_t::dispatch; - using ir_visitor_t::visit; - expr_c visit(var_c v) override { - auto itr = remap_.find(v); - if (itr != remap_.end()) { - return itr->second; - } - return v; - } -}; - -void mixed_parti_t::try_split_outermost_loop_on_num_threads( - int64_t num_groups) { - auto outer_loops = get_outer_loops(); - if (outer_loops.empty()) - return; - auto outermost_loop = outer_loops[0]; - - // cache original largest anchor inside outermost loop - auto origin_large_anchor = get_anchor_inside_loop(outermost_loop); - // node ptr replace map - node_ptr_map remap; - // change IR and record replace map - auto split_inner_loop = - outermost_loop->split_on_num_threads(num_groups, &remap); - mxp_replacer_t(remap).replace_anchor(fanchors_); - // if original largest anchor not null, create new largest anchor under new - // outermost loop - if (origin_large_anchor) { - auto s = builder::make_stmts_unattached({}).checked_as(); - add_parent_node(s, outermost_loop->body_); - outermost_loop->body_.checked_as()->seq_.emplace_back(s); - // dummy fsmap, the tensor belongs to this scope will not be shrinked - fslice_map new_fsmap; - // copy from original fsmap - new_fsmap.datamap_ = origin_large_anchor->fsmap_.datamap_; - // create var inplacer - std::unordered_map vmap = { - {split_inner_loop->var_, - make_expr(UINT64_C(0), - split_inner_loop->var_->dtype_)}}; - var_replacer_t repl(vmap); - // modify slice range - for (auto &fspair : new_fsmap.datamap_) { - for (auto &srange : fspair.second) { - for (auto &r : srange) { - auto new_v = repl.dispatch(r.first); - // if necesary - if (!new_v.ptr_same(r.first)) { - // set new offset - r.first = new_v.remove_const(); - // set new range - r.second = - dim2unsigned(get_expr_as_int(r.second) * - (get_expr_as_int(split_inner_loop->iter_end_) - - get_expr_as_int(split_inner_loop->iter_begin_))); - } - } - } - } - // append new anchor into parti - append_fusion_anchor(std::make_shared(s, new_fsmap)); - } -} - -void mixed_parti_t::try_split_outermost_loop(int64_t block) const { - auto outer_loops = get_outer_loops(); - if (outer_loops.empty()) - return; - auto outermost_loop = outer_loops[0]; - auto outermost_loop_range = get_expr_as_int(outermost_loop->iter_end_) - - get_expr_as_int(outermost_loop->iter_begin_); - if (outermost_loop_range % block != 0) - return; - if (!outermost_loop->step_.isa() || - get_expr_as_int(outermost_loop->step_) != 1) - return; - - node_ptr_map remap; - // change IR and record replace map - outermost_loop->split(outermost_loop_range / block, &remap); - mxp_replacer_t(remap).replace_anchor(fanchors_); -} - -void mixed_parti_t::buffer_schedule() { - if (merged_to) { - get_root()->buffer_schedule(); - return; - } - if (ctx_->flags_.concat_optimization_) { - buf_alloc_.copy_concat_memory_attrs_tsr2buf(); - } - buf_alloc_.tensor_initialize(); - buf_alloc_.declare_tensor(); - buf_alloc_.set_shrink_info(); - buf_alloc_.query_inplace(); - buf_alloc_.calibrate_info(); -} - -bool mixed_parti_t::is_parti_inp(const graph_tensor *gt) const { - if (merged_to) { - return get_root()->is_parti_inp(gt); - } - auto ths = this; - return std::any_of(gt->uses_.begin(), gt->uses_.end(), - [&ths](const std::pair &user) { - return ths->contains(user.second.get()); - }) && - !contains(gt->producer_owner_); -} - -bool mixed_parti_t::is_parti_inp(const graph_tensor_ptr >) const { - return is_parti_inp(gt.get()); -} - -bool mixed_parti_t::is_parti_out(const graph_tensor *gt) const { - if (merged_to) { - return get_root()->is_parti_out(gt); - } - auto ths = this; - return contains(gt->producer_owner_) && - std::any_of(gt->uses_.begin(), gt->uses_.end(), - [&ths](const std::pair &user) { - return !ths->contains(user.second.get()); - }); -} - -bool mixed_parti_t::is_parti_out(const graph_tensor_ptr >) const { - return is_parti_out(gt.get()); -} - -bool mixed_parti_t::is_const_parti() const { - return (get_ops_size() == 1 && - get_ith_op(0)->attrs_.get_or_else( - "constant", const_kind::not_const) != const_kind::not_const); -} - -bool mixed_parti_t::contain_convolution() const { - return contain_op_with_type() || - contain_op_with_type() || - contain_op_with_type(); -} - -bool mixed_parti_t::contain_nested_parallel_for() const { - auto outer_loops = get_outer_loops(); - if (outer_loops.empty()) - return false; - if (outer_loops.size() == 1) { - if (outer_loops[0]->num_threads_ == 0) - return false; - if (outer_loops[0]->body_.isa()) { - auto &ss = outer_loops[0]->body_.static_as()->seq_; - for (auto &s : ss) { - if (s.isa()) { - if (s.static_as()->num_threads_ > 0) - return true; - } - } - } - return false; - } - return (outer_loops[0]->num_threads_ > 0) && - (outer_loops[1]->num_threads_ > 0); -} - -bool mixed_parti_t::contain_tunable_op() const { - return contain_op_with_type(); -} - -bool mixed_parti_t::contain_elemwise_op_only() const { - if (merged_to) { - return get_root()->contain_elemwise_op_only(); - } - for (auto &op : ops) { - if (!is_elementwise_op(op.get())) - return false; - } - return true; -} - -bool mixed_parti_t::is_single_op_parti() const { - if (merged_to) { - return get_root()->is_single_op_parti(); - } - return ops.size() == 1; -} - -bool mixed_parti_t::is_optimized() const { - if (merged_to) { - return get_root()->is_optimized(); - } - if (committed_ops_.empty()) - return false; - return is_optimized_sub_graph(get_host_graph()); -} - -void mixed_parti_t::clear() { - // Graph-related - ops.clear(); - committed_ops_.clear(); - dep_m_ = nullptr; - - // IR-related - func_ = func_t(); - fanchors_.clear(); - buf_alloc_.clear(); - op_anchor_map_.clear(); - - // Cost Model - cost_ = nullptr; -} - -float mixed_parti_t::evaluate_perf() { - if (merged_to) { - return get_root()->evaluate_perf(); - } - return cost_->evaluate(); -} - -bool mixed_parti_t::is_small_workload() const { - if (merged_to) { - return get_root()->is_small_workload(); - } - // skip partition owning more than one ops - if (committed_ops_.size() != 1) - return false; - // get sinlge op - auto single_op = committed_ops_[0].get(); - // skip tunable op - if (single_op->isa()) - return false; - // get committed anchor - auto committed_anchor = lookup_anchor_map(single_op); - COMPILE_ASSERT(committed_anchor, "No committed anchor found") - // query small op workload - return committed_anchor->is_small_op_workload(single_op); -} - -std::vector -collect_parti_set(const std::vector &op_2_partition, - bool ignore_const) { - std::unordered_set parti_set; - std::vector parti_vec; - for (auto &parti : op_2_partition) { - if (parti_set.count(parti)) { - continue; - } - // auto skip nullptr or const parti - if (!parti || (ignore_const && parti->is_const_parti())) - continue; - parti_set.insert(parti); - parti_vec.push_back(parti); - } - return parti_vec; -} - -static bool check_repartition(const mixed_parti_t::ptr &parti) { - // if case 3's pre_fuse_break leads to standalone op - // we remove break_pre_fuse - if (parti && parti->get_ops_size() == 1 && - parti->committed_ops_[0]->attrs_.get_or_else( - mixed_partition_hint::trial_break, false)) { - parti->committed_ops_[0]->attrs_.remove(op_attr_key::break_pre_fuse); - parti->committed_ops_[0]->attrs_[mixed_partition_hint::trial_break] = false; - return true; - } - if (!parti || parti->get_ops_size() < 2) - return false; - // check tensorview in edge of partition - bool repartition = false; - // check buffer is tensorptr - auto check_parti_out_tptr = [&repartition, - &parti](const graph_tensor_ptr &out) { - auto producer = out->producer_owner_; - COMPILE_ASSERT(parti->buf_alloc_.g2b_map_.haskey(out), - "No buffer allocated found for output " - "of " - << producer->op_name_ << producer->logical_op_id_) - if (parti->buf_alloc_.g2b_map_.get(out).isa()) { - if (producer->isa()) { - auto tv_op = producer; - graph_tensor *tv_inp; - while (tv_op->isa()) { - tv_op->attrs_[op_attr_key::no_fuse] = true; - tv_inp = tv_op->get_inputs()[0].get(); - tv_op = tv_inp->producer_owner_; - } - if (parti->buf_alloc_.g2b_map_.get(tv_inp).isa()) { - tv_inp->attrs_[mixed_partition_hint::no_inplace] = true; - } - } else { - out->attrs_[mixed_partition_hint::no_inplace] = true; - } - repartition = true; - } - }; - for (auto &op : parti->ops) { - // 1. check partition output buffer should not be tensoptr - for (auto &out : op->get_outputs()) { - if (parti->is_parti_out(out)) - check_parti_out_tptr(out); - } - // 2. check reorder - if (auto reo = op->isa()) { - if (!op->dyn_cast()->support_output_loop() || - op->get_inputs()[0]->uses_.size() == 1) { - continue; - } - auto tunable_users = search_tuneop_linearly(op); - // mark this kind of reorder to do pre-op fusion during - // repartition stage - if (tunable_users && !parti->contains(tunable_users.get())) { - op->attrs_[op_attr_key::break_pre_fuse] = true; - repartition = true; - // the input of this reorder will be the new output of - // partition, check tptr in advance to reduce repartition times - check_parti_out_tptr(op->get_inputs()[0]); - } - } - // 3. check partition outputs and mark break_pre_fuse on B - // when encountering the topology below - // A__ - // / | - // out B - // | - // out - // if B becomes a standalone op, we will re-fuse it - // we requires B to have all of its consumers outside of the current - // partition - bool is_terminate_op = std::all_of( - op->get_outputs().begin(), op->get_outputs().end(), - [&](const graph_tensor_ptr >) { - return std::all_of( - gt->uses_.begin(), gt->uses_.end(), - [&parti](const std::pair &user) { - return !parti->contains(user.second.get()); - }); - }); - if (is_terminate_op) { - // find producer inputs - for (const auto &prod_input : op->get_inputs()) { - for (const auto &preced_output : - prod_input->producer_owner_->get_outputs()) { - if (parti->is_parti_out(preced_output) && - !op->attrs_.has_key(mixed_partition_hint::trial_break)) { - op->attrs_[op_attr_key::break_pre_fuse] = true; - op->attrs_[mixed_partition_hint::trial_break] = true; - repartition = true; - } - } - } - } - } - return repartition; -} - -bool mixed_parti_t::contain_input_anchor() const { - if (merged_to) { - return get_root()->contain_input_anchor(); - } - return std::any_of(fanchors_.begin(), fanchors_.end(), - [](const fusion_anchor_ptr &anchor) { - return anchor->is_input_anchor(); - }); -} - -static mixed_parti_t::ptr try_execute_pre_op_fusion( - const context_ptr &ctx, const sc_op_ptr &op, - std::vector &avaliable_input_parti) { - mixed_parti_t::ptr parent_partition = nullptr; - // only suitable for tunable op - if (!op->isa()) - return parent_partition; - if (!require_fusion_level(ctx, fusion_opt_level::lv2)) { - return parent_partition; - } - // collect reorder parti - std::vector reo_parti; - for (auto &inp_parti : avaliable_input_parti) { - if (inp_parti->get_ops_size() == 1 && - inp_parti->contain_op_with_type()) { - reo_parti.emplace_back(inp_parti->get_root()); - } - } - if (reo_parti.empty()) - return parent_partition; - // create tunable partition for possible input fusion anchor - parent_partition = std::make_shared(reo_parti[0]->ctx_, op, - reo_parti[0]->dep_m_); - if (!parent_partition->contain_input_anchor()) { - parent_partition->clear(); - return nullptr; - } - // search input anchor for reorder op - for (auto &rparti : reo_parti) { - auto reo = rparti->committed_ops_[0]; - bool input_anchor_found = false; - // set pre-op fusion attr - reo->attrs_.set(mixed_partition_hint::pre_fuse_begin_op, true); - for (auto &fanchor : parent_partition->fanchors_) { - // only search input anchor - if (!fanchor->is_input_anchor()) - continue; - // check output of reorder - auto reo_out = reo->get_outputs()[0], reo_in = reo->get_inputs()[0]; - if (!fanchor->fsmap_.hasvalue(reo_out)) - continue; - fslice_map tmp_fsmap; - // copy to tmp fsmap - tmp_fsmap.get(reo_out) = fanchor->fsmap_.get(reo_out); - auto status = reo->stc_cast()->pre_infer_slice_ranges( - parent_partition->ctx_, tmp_fsmap); - if (status == infer_status_code::OK) { - input_anchor_found = true; - fanchor->fsmap_.get(reo_in) = tmp_fsmap.get(reo_in); - } - } - if (input_anchor_found) { - // pre fuse reorder - parent_partition->add(reo); - // clear origin parti - rparti->clear(); - // reset root pointer - rparti->merged_to = parent_partition; - } - // remove pre-op fusion attr - reo->attrs_.remove(mixed_partition_hint::pre_fuse_begin_op); - } - if (parent_partition->get_ops_size() == 1) { - parent_partition->clear(); - // reset - parent_partition = nullptr; - } else { - auto &ops = parent_partition->committed_ops_; - auto old_op = std::move(ops.front()); - ops.erase(ops.begin()); - ops.emplace_back(std::move(old_op)); - } - return parent_partition; -} - -static mixed_parti_t::ptr try_execute_post_op_fusion( - const context_ptr &ctx, const sc_op_ptr &op, - std::vector &avaliable_input_parti) { - mixed_parti_t::ptr parent_partition = nullptr; - // try to preop merge the partitons of all inputs in advance - for (auto &inp_parti : avaliable_input_parti) { - if (!parent_partition) { - parent_partition = inp_parti; - } else if (!parent_partition->empty() && !inp_parti->empty()) { - COMPILE_ASSERT(op->isa(), - "Unexpected op type find: " << op->op_name_) - op->dyn_cast()->search_anchor( - parent_partition.get()); - op->dyn_cast()->search_anchor( - inp_parti.get()); - if (parent_partition->ready_for_op(op.get()) && - inp_parti->ready_for_op(op.get())) { - // the difference between later partition merge with - // joint op is that current merge need to check - // partition connection to ensure legality. - try_merge_brgemm_and_preop_parti(parent_partition.get(), - inp_parti.get(), nullptr); - } - } - } - // reset parent partition - parent_partition = nullptr; - // try to add op to input partition - for (auto &inp_parti : avaliable_input_parti) { - if (inp_parti->is_ok_to_add(op.get())) { - if (parent_partition) { - try_merge_mixed_parti_with_joint_op(parent_partition, inp_parti, op); - parent_partition = std::static_pointer_cast( - parent_partition->get_root()->shared_from_this()); - } else { - parent_partition = inp_parti; - // Do not merge input partition according hint - if (op->attrs_.get_or_else(mixed_partition_hint::no_gather_op, false)) - break; - } - } - } - if (parent_partition) { - if (!parent_partition->get_root()->add(op)) { - // set no gather input partition hint - op->attrs_.set(mixed_partition_hint::no_gather_op, true); - } - } - return parent_partition; -} - -bool do_partition(const context_ptr &ctx, sc_graph_t &g, - std::vector &op_2_partition, - const dep_mat_ptr &dep_m) { - // validate partition - bool repartition = false; - // a speculative DFS visitor - op_visitor_t visitor = - op_visitor_t::dfs_topology_speculative_sort(g.ops_.size()); - visitor.visit_graph(g, [&](op_visitor_t *visitor, const sc_op_ptr &op) { - if (op->isa() || op->isa()) - return; - mixed_parti_t::ptr parent_partition = nullptr; - if (op->attrs_.get_or_else(op_attr_key::no_fuse, false)) { - SC_MODULE_INFO << op->op_name_ << "_" << op->logical_op_id_ - << " is marked as no fuse"; - } else { - if (!op->attrs_.get_or_else(op_attr_key::break_pre_fuse, false)) { - std::vector avaliable_input_parti; - // any op marked as `break_post_fuse` should be excluded - std::vector excluded_ops; - // collect avaliable input partition - auto sorted_inputs = get_sorted_inputs_by_layout_input(op); - for (auto &in : sorted_inputs) { - // if an input is fusible and is not "break_post_fuse" - if (!in->producer_owner_->attrs_.get_or_else( - op_attr_key::break_post_fuse, false) && - !in->producer_owner_->attrs_.get_or_else(op_attr_key::no_fuse, - false) && - in->producer_owner_->attrs_.get_or_else( - "constant", const_kind::not_const) == const_kind::not_const) { - auto &cur_in_partition = - op_2_partition[in->producer_owner_->logical_op_id_]; - if (cur_in_partition) - avaliable_input_parti.emplace_back(cur_in_partition); - } else if (in->producer_owner_->attrs_.get_or_else( - op_attr_key::break_post_fuse, false)) { - SC_MODULE_INFO << op->op_name_ << "_" << op->logical_op_id_ - << " fail to add partition because its " - "producer " - << in->producer_owner_->op_name_ << "_" - << in->producer_owner_->logical_op_id_ - << " is marked as break post fuse"; - // Although input op has been marked as break post fuse, - // it maybe already fused into avaliable input partition - // belonging to another ones - excluded_ops.emplace_back(in->producer_owner_); - } - } - // filter input partition - auto filter_input_partition = [&avaliable_input_parti, - &excluded_ops]() { - // avoid duplication - std::unordered_set unique_parti_set; - for (auto iter = avaliable_input_parti.begin(); - iter != avaliable_input_parti.end();) { - // get root partition as key - auto root_parti = (*iter)->get_root(); - if (unique_parti_set.find(root_parti) != unique_parti_set.end()) { - iter = avaliable_input_parti.erase(iter); - } else { - unique_parti_set.insert(root_parti); - // double-check whether contains excluded ops - if (std::any_of(excluded_ops.begin(), excluded_ops.end(), - [&root_parti](sc_op *op) { - return root_parti->contains(op); - })) { - iter = avaliable_input_parti.erase(iter); - } else { - iter++; - } - } - } - }; - // filter duplicated ones - filter_input_partition(); - // try pre op fusion - parent_partition = - try_execute_pre_op_fusion(ctx, op, avaliable_input_parti); - if (!parent_partition) { - // try post op fusion - parent_partition = - try_execute_post_op_fusion(ctx, op, avaliable_input_parti); - } - } else { - SC_MODULE_INFO << op->op_name_ << "_" << op->logical_op_id_ - << " fail to add partition because it is marked " - "as break pre fuse"; - } - } - if (!parent_partition || !parent_partition->contains(op.get())) { - // op was not added into parent partition, usually after unexpected - // input partition merge, as the result, set repatition flag to - // trigger next round of partition from the view of performance. - if (parent_partition && !parent_partition->contains(op.get())) { - repartition = true; - } - parent_partition = std::make_shared(ctx, op, dep_m); - } - op_2_partition[op->logical_op_id_] = parent_partition; - }); - - for (auto &parti : op_2_partition) { - if (parti) { - parti = std::static_pointer_cast( - parti->get_root()->shared_from_this()); - } - } - - // collect parti set - auto parti_set = collect_parti_set(op_2_partition); - // assign checker - auto checker = &check_repartition; - std::for_each(parti_set.begin(), parti_set.end(), - [&checker, &repartition](const mixed_parti_t::ptr &parti) { - if ((*checker)(parti)) - repartition = true; - }); - if (repartition) { - SC_MODULE_INFO << "================ Repartition the whole graph " - "================"; - } - return !repartition; -} - -bool mixed_parti_t::validate_optimization() const { - return buf_alloc_.validate_tsr2var(); -} - -bool mixed_parti_t::can_optimize_outer_loop(bool allow_tensorview) const { - if (merged_to) { - return get_root()->can_optimize_outer_loop(allow_tensorview); - } - bool for_reduce = - contain_op_with_type() && - std::all_of( - ops.begin(), ops.end(), - [&](const sc_op_ptr &op) { - if (op->isa()) { - if (op->isa() && allow_tensorview) { - return is_parti_out(op->get_outputs()[0]) || - is_parti_inp(op->get_inputs()[0]); - } else if (op->isa()) { - return op->attrs_.get_or_else(op_attr_key::no_fuse, false); - } else { - return false; - } - } else { - return is_elementwise_op(op.get()) || op->isa() || - op->isa() || - (op->isa() && - !op->stc_cast() - ->is_partial_reduce()); - } - }) && - std::any_of(ops.begin(), ops.end(), [&](const sc_op_ptr &op) { - std::vector rd_axis; - if (auto rd_op = op->dyn_cast()) { - rd_axis = rd_op->get_rd_axis(); - } else if (auto rd_op = op->dyn_cast()) { - rd_axis = rd_op->get_rd_axis(); - } else { - return false; - } - std::sort(rd_axis.begin(), rd_axis.end()); - int cur = (op->get_inputs()[0]->details_.get_blocking_dims().size() - - rd_axis.size()); - /** E.g - * - reduce input dims: [32,64,16,16] - * - rd_axis: [2,3] - * It is unecessary to optimize outer loop for this kind - * of reduce op - * */ - for (auto &ax : rd_axis) { - if (ax != cur) - return true; - cur++; - } - return false; - }); - bool for_pooling = - contain_op_with_type() && - std::all_of(ops.begin(), ops.end(), [&](const sc_op_ptr &op) { - return (!op->isa() || op->isa()) && - !op->isa(); - }); - return !is_optimized() && (for_reduce || for_pooling); -} - -static bool try_optimize_reduce( - mixed_parti_t *parti, sc_graph_t &sub_graph, - const std::unordered_map &graph2orig_ops) { - // currently disable reduce optimization in dynamic - if (!parti->contain_op_with_type() || - sub_graph.is_dynamic()) - return false; - - bool redo = false; - auto &ctx = parti->ctx_; - - // if parti contains nested parallel for, it could not be ensured that the - // inner loop is not parallel - bool nested_parallel_found = parti->contain_nested_parallel_for(); - - std::unordered_set split_reduce_set; - for (auto &op : sub_graph.ops_) { - if (auto red_op = op->dyn_cast()) { - if (!red_op->can_split_op()) - continue; - if (op->isa() && !nested_parallel_found) { - split_reduce_set.insert(red_op); - } else if (auto rd_op = op->dyn_cast()) { - COMPILE_ASSERT(rd_op->is_partial_reduce(), - "Only partial reduce is expected") - // predict anchor parallelism of coming `reduce_collect_op` - auto orig_iter = graph2orig_ops.find(op); - COMPILE_ASSERT(orig_iter != graph2orig_ops.end(), - "Could not find original op in partition") - auto &input = orig_iter->second->get_inputs()[0]; - auto required_axis = rd_op->get_rd_axis(); - // get real requied axis - required_axis.erase(required_axis.begin()); - bool parallelism = true; - // get outer loops - auto outer_loops = parti->get_outer_loops(nullptr, true); - for (auto itr = outer_loops.begin(); itr != outer_loops.end(); itr++) { - // check required axis whether appears on outer loops - if (check_loop_has_axis(*itr, input, required_axis)) { - // if found, predict parallelism - if (itr == outer_loops.begin() || - evaluate_loop_parallel_balance( - {outer_loops.begin(), itr - 1}) != 1.0f) { - parallelism = false; - } - break; - } - } - if (parallelism) - split_reduce_set.insert(red_op); - } - } - } - // If split reduce op exist - for (auto &red_op : split_reduce_set) { - auto op = dynamic_cast(red_op); - reduce_operator rd_type; - // check padding except for reduce add - if (auto rd_op = op->dyn_cast()) { - rd_type = rd_op->get_rd_op(); - } else if (auto rd_op = op->dyn_cast()) { - rd_type = rd_op->get_rd_op(); - } else { - COMPILE_ASSERT(0, "Unexpected kind of op found: " << op->op_name_) - } - if (rd_type != reduce_operator::add) { - auto &plain_dims = op->get_inputs()[0]->details_.get_plain_dims(); - auto &fmt = op->get_inputs()[0]->details_.get_format(); - auto blocking_dims = - sc_data_format_t::get_blocking_shapes(plain_dims, fmt); - auto padded_dims = - sc_data_format_t::get_padded_plain_shapes(blocking_dims, fmt); - // currently, do not support split with padding - if (plain_dims != padded_dims) - continue; - } - // pre-check - if (op->isa()) { - // find original op in partition - auto orig_iter = graph2orig_ops.find(op->shared_from_this()); - COMPILE_ASSERT(orig_iter != graph2orig_ops.end(), - "Could not find original op in partition") - // get shrink info - auto slice_info = parti->buf_alloc_.get_shrinked_info( - parti->buf_alloc_.g2b_map_.get(orig_iter->second->get_outputs()[0])); - if (slice_info.empty()) - continue; - sc_dim prod = - get_dims_product(get_expr_to_dims(get_slice_shape(slice_info))); - auto tsr_simd_len = - vectorize_step(ctx, op->get_inputs()[0]->details_.dtype_.type_code_); - if (!check_tsr_len_under_resigter_size(prod, tsr_simd_len)) - continue; - } - // Do split - red_op->split_op(ctx, sub_graph, 1); - redo = true; - } - - return redo; -} - -static bool try_optimize_concat(mixed_parti_t *parti, sc_graph_t &sub_graph) { - return parti->ctx_->flags_.concat_optimization_ && - concat_memory_planning_on_graph(sub_graph); -} - -static bool try_optimize_outer_loop( - mixed_parti_t *parti, sc_graph_t &sub_graph, - const std::unordered_map &graph2orig_ops) { - // prepare stage - auto &ops = sub_graph.ops_; - // check reorder in partition which includes reduce op but - // exclude tunable op - if (parti->contain_op_with_type() && - !parti->contain_tunable_op() && - parti->contain_op_with_type()) { - bool forced_reorder_axis = false; - std::unordered_set reo_op_set; - auto run_threads = runtime_config_t::get().get_num_threads(); - for (auto &op : ops) { - if (op->is_removed_) - continue; - if (op->isa()) { - std::vector rd_axis; - if (auto rd_op = op->dyn_cast()) { - rd_axis = rd_op->get_rd_axis(); - } else if (auto rd_op = op->dyn_cast()) { - if (rd_op->is_partial_reduce()) { - forced_reorder_axis = false; - break; - } - rd_axis = rd_op->get_rd_axis(); - } - auto shape = op->get_inputs()[0]->details_.get_blocking_dims(); - int outer_rd_axis_size = 1; - for (int i = 0; i < *rd_axis.begin(); i++) { - outer_rd_axis_size *= shape[i]; - } - /* Due to loop order not only affect outer-loop parallelism, - * but also inner-loop fusion, which will affect local buffer - * size( sensitive to cache line size). Further, more - * performance data maybe required and analyzed to decide which - * strategy shuold be applied to achieve best performance*/ - if (outer_rd_axis_size < run_threads) - forced_reorder_axis = true; - } else if (op->isa()) { - reo_op_set.insert(op); - } - } - if (forced_reorder_axis) { - std::for_each(reo_op_set.begin(), reo_op_set.end(), - [&graph2orig_ops](const sc_op_ptr &op) { - op->attrs_[op_attr_key::no_fuse] = true; - // sync origin op in partition - auto orig_iter = graph2orig_ops.find(op); - COMPILE_ASSERT(orig_iter != graph2orig_ops.end(), - "Could not find original op in partition") - orig_iter->second->attrs_[op_attr_key::no_fuse] = true; - }); - } - } - - // optimize loop order - return parti->can_optimize_outer_loop(); -} - -static bool try_optimize_fusion_anchor( - mixed_parti_t *parti, sc_graph_t &sub_graph, - const std::unordered_map &graph2orig_ops) { - // auto skip - if (parti->committed_ops_.size() < 2) - return false; - auto &ctx = parti->ctx_; - // check the op whether is the elementwise op with last dim undividable - auto elem_op_with_last_dim_undividable = - [&ctx, &parti](const sc_op_ptr &op) -> bool { - if (!is_elementwise_op(op.get())) - return false; - auto committed_anchor = parti->lookup_anchor_map(op.get(), false); - if (!committed_anchor) - return false; - auto gt = op->get_outputs()[0]; - COMPILE_ASSERT(committed_anchor->fsmap_.hasvalue(gt), - "Unexpected case for elementwise op: " << op->op_name_ << "_" - << op->logical_op_id_) - auto &slice_info = committed_anchor->fsmap_.get(gt); - if (slice_info.size() != 1) - return false; - auto compute_shape = get_slice_shape(slice_info[0]); - if (!compute_shape.back().isa()) - return false; - auto last_dim = get_expr_as_int(compute_shape.back()); - // get max lanes - auto dtype = gt->details_.dtype_; - auto lanes = vectorize_step(ctx, dtype.type_code_); - return ((last_dim > lanes) && (last_dim % lanes != 0)); - }; - // query partition op on graph - auto query_op_on_graph = [&graph2orig_ops](const sc_op_ptr &op) { - auto iter = std::find_if(graph2orig_ops.begin(), graph2orig_ops.end(), - [&op](const std::pair &kv) { - return kv.second == op; - }); - COMPILE_ASSERT(iter != graph2orig_ops.end(), "Could not find mapping op") - return iter->first; - }; - // set hint about fusion anchor - auto set_hint = [](std::vector &ops) -> bool { - bool flag = false; - if (ops.size() > 1) { - for (auto &op : ops) { - op->attrs_.set(mixed_partition_hint::split_anchor_op, true); - } - flag = true; - } - ops.clear(); - return flag; - }; - - auto dep_m = op_dep_matrix_t(sub_graph); - bool redo = false; - std::vector target_ops; - // visit sorted commit ops - for (auto &op : parti->committed_ops_) { - // get mapping op on sub graph - auto sub_graph_op = query_op_on_graph(op); - if (target_ops.empty() && elem_op_with_last_dim_undividable(op)) { - // lookup commited anchor - auto &committed_anchor = *parti->lookup_anchor_map(op.get()); - if (typeid(committed_anchor) == typeid(fusion_anchor_t)) { - target_ops.emplace_back(sub_graph_op); - } - } else if (!target_ops.empty()) { - auto first_op = graph2orig_ops.find(target_ops.front())->second.get(); - // successive elementwise op and same anchor with first op and have - // dependency with last target op - if (elem_op_with_last_dim_undividable(op) && - (parti->lookup_anchor_map(first_op) == - parti->lookup_anchor_map(op.get())) && - dep_m.lookup(target_ops.back(), sub_graph_op) == 1) { - target_ops.emplace_back(sub_graph_op); - } else if (parti->lookup_anchor_map(first_op) == - parti->lookup_anchor_map(op.get(), false) && - !target_ops.empty()) { - target_ops.clear(); - // replace first op - if (elem_op_with_last_dim_undividable(op)) { - target_ops.emplace_back(sub_graph_op); - } - } else { - // check redo flag - redo |= set_hint(target_ops); - } - } - } - // if the first op still exists utils loop ends - redo |= set_hint(target_ops); - - return redo; -} - -bool try_optimize_parti( - mixed_parti_t *parti, sc_graph_t &sub_graph, - const std::unordered_map &graph2orig_ops) { - // if `opt-level == 0`, auto skip - if (parti->ctx_->flags_.opt_level_ == sc_opt_level::lv0) - return false; - if (sub_graph.is_dynamic()) { - return false; - } - // skip already optimized parti - if (parti->is_optimized()) - return false; - bool need_optimize = false; - // optimize reduce - need_optimize |= try_optimize_reduce(parti, sub_graph, graph2orig_ops); - // optimize concat - need_optimize |= try_optimize_concat(parti, sub_graph); - // optimize loop order - need_optimize |= try_optimize_outer_loop(parti, sub_graph, graph2orig_ops); - // optimize fusion anchor - need_optimize |= try_optimize_fusion_anchor(parti, sub_graph, graph2orig_ops); - - if (need_optimize) { - sub_graph.attrs_.set(mixed_partition_hint::optimized_sub_graph, true); - } - return need_optimize; -} - -static std::string get_graph_name(const sc_graph_t &graph) { - std::string ret; - for (auto &op : graph.ops_) { - if (op->isa() || op->isa() || op->isa()) - continue; - if (!ret.empty()) - ret += "_"; - ret += op->op_name_; - } - return ret; -} - -static std::string get_parti_prefix(const mixed_parti_t &parti) { - std::string prefix; - if (parti.ops.size() > 1) { - prefix = "partition_"; - auto outer_loops = parti.get_outer_loops(); - if (!outer_loops.empty()) { - prefix = "outerloop_" + print_loops_range(outer_loops) + "_" + prefix; - } - } - return prefix; -} - -static void search_first_prefetch_op(mixed_parti_t &parti) { - for (const auto &op : parti.committed_ops_) { - if (op->isa() && op->isa()) { - op->attrs_[mixed_partition_hint::first_prefetch_op] = true; - break; - } - } -} - -std::shared_ptr mixed_parti_t::transform_to_mixed_op() { - COMPILE_ASSERT(!empty(), "Could not transform empty partition") - // Get original graph - auto &g = get_host_graph(); - // Make sub graph - sc_graph_t sub_graph; - sub_graph.sync_dynamic_info_with_graph(g); - std::vector fused_op_in, fused_op_out; - std::vector arg_ins, arg_out; - std::string op_name; - // the mapping for original op to op in sub graph - std::unordered_map graph2orig_ops; - // the mapping for original LT in original ops to fuse => the LT in the - // sub_graph. - std::unordered_map orig_2_graph; - auto get_or_create_graph_tsr = [&](const graph_tensor_ptr &orig_lr) { - auto itr = orig_2_graph.find(orig_lr); - if (itr != orig_2_graph.end()) { - return itr->second; - } - auto ret = std::make_shared(nullptr, orig_lr->details_); - orig_2_graph.insert(std::make_pair(orig_lr, ret)); - return ret; - }; - auto visitor = op_visitor_t::dfs_topology_sort(g.ops_.size()); - std::unordered_set input_tsr_set; - // search first prefetch op of original graph and set attr on it - search_first_prefetch_op(*this); - // visit original graph - visitor.visit_graph(g, [&](op_visitor_t *visitor, const sc_op_ptr &op) { - if (ops.find(op) == ops.end()) { - return; - } - std::vector new_graph_in, new_graph_ou; - for (auto &in : op->get_inputs()) { - new_graph_in.emplace_back(get_or_create_graph_tsr(in)); - if (is_parti_inp(in) && input_tsr_set.find(in) == input_tsr_set.end()) { - // if the input is not included in the parti, make an input - // node - auto new_input_op = sub_graph.make_input({new_graph_in.back()}); - // inherit constant attr for input if necessary - if (in->producer_owner_->attrs_.has_key("constant")) { - new_input_op->attrs_.set( - "constant", in->producer_owner_->attrs_.get("constant")); - } - // add the input in the args of the fused op in orig - // sub_graph - fused_op_in.emplace_back(in); - input_tsr_set.insert(in); - COMPILE_ASSERT(buf_alloc_.g2b_map_.haskey(in), - "No buffer allocated for " << op->op_name_ << "_" - << op->logical_op_id_ - << " inputs") - arg_ins.emplace_back(buf_alloc_.g2b_map_.get(in)); - } - } - for (auto &out : op->get_outputs()) { - new_graph_ou.emplace_back(get_or_create_graph_tsr(out)); - // if the output is a "cut" - an edge across the parti and - // outside of the parti; or if concat optimization is enabled: - if (is_parti_out(out) || - (ctx_->flags_.concat_optimization_ && op->isa() && - op->attrs_.get_or_else(concat_optim_attr_keys::is_final_concat, - false))) { - // if there is a use outside of the parti, the tensor should - // be marked "output" - const auto &outtsr = new_graph_ou.back(); - auto new_out_op = sub_graph.make_output({outtsr}); - // make a new output tensor for the fused_op_t in the - // original sub_graph - fused_op_out.emplace_back( - std::make_shared(nullptr, out->details_)); - // save the mapping of the tensor to be replaced => new - // tensor - output_replace_map[out] = fused_op_out.back(); - COMPILE_ASSERT(buf_alloc_.g2b_map_.haskey(out), - "No buffer allocated for " << op->op_name_ << "_" - << op->logical_op_id_ - << " outputs") - auto out_buffer = buf_alloc_.g2b_map_.get(out); - // if outbuffer is already reused, set attr on output op - if (out_buffer->attr().has_key(attr_keys::tensor_inplace_hint)) { - new_out_op->attrs_.set("buffer_already_reused", true); - } - arg_out.emplace_back(out_buffer); - } - } - auto copyable = op->dyn_cast(); - assert(copyable); - auto copied = copyable->copy(new_graph_in, new_graph_ou, sub_graph); - copied->copy_dispatch_key_set_from_op(op); - graph2orig_ops.insert(std::make_pair(copied, op)); - // build the fused op name - if (!op_name.empty()) - op_name += '_'; - op_name += copied->op_name_; - }); - - // copy graph in avoid of redo fall-back case - auto copied_grpah = copy_graph(sub_graph); - - if (try_optimize_parti(this, sub_graph, graph2orig_ops)) { - SC_MODULE_INFO << "Optimizing mixed partition for current pattern: " - << func_->name_; - // copy optimized sub graph - auto copied_opt_grpah = copy_graph(sub_graph); - // redo mixed partition with setting hint - do_mixed_partition(ctx_, sub_graph); - bool fall_back = false; - std::vector parti_list; - std::string mx_op_name; - bool non_mixed_op_exist = false; - // redo validation stage - for (auto &op : sub_graph.ops_) { - if (op->isa() || op->isa() || - op->isa()) - continue; - if (!mx_op_name.empty()) - mx_op_name += "_"; - if (auto mx_op = op->dyn_cast()) { - COMPILE_ASSERT( - mx_op->parti_list_.size() == 1, - "Unexpected partition size found: " << mx_op->parti_list_.size()) - if (!mx_op->parti_list_[0]->validate_optimization()) { - // reset - fall_back = true; - SC_MODULE_INFO << "invalid optimized reduce detected, " - "fall-back " - "to original pattern"; - break; - } - parti_list.emplace_back(mx_op->parti_list_[0]); - mx_op_name += get_graph_name(mx_op->sub_graph_); - } else { - if (op->isa()) { - fall_back = true; - SC_MODULE_INFO << "reduce collect op must be fused with " - "reduce compute op, fall-back " - "to original pattern"; - break; - } - if (!is_reshape_op(op.get())) - non_mixed_op_exist = true; - mx_op_name += op->op_name_; - } - } - if (!fall_back) { - if (parti_list.size() == 1 && !non_mixed_op_exist) { - mx_op_name = get_parti_prefix(*parti_list[0]) + mx_op_name; - } else if (!is_single_op_graph(g)) { - mx_op_name = "multi_partitions_" + mx_op_name; - } - std::vector lower_args(sub_graph.get_output_ops()); - auto input_ops = sub_graph.get_input_ops(); - lower_args.insert(lower_args.end(), input_ops.begin(), input_ops.end()); - auto modu = lower_graph(ctx_, sub_graph, lower_args, false); - auto main_func = modu->get_entry_func(); - main_func->name_ = mx_op_name; - main_func->decl_->name_ = mx_op_name; - return std::make_shared(mx_op_name, parti_list, modu, - copied_opt_grpah, - /*ins*/ fused_op_in, - /*outs*/ - fused_op_out, any_map_t{}); - } else { - // fall-back - parti_list.clear(); - mx_op_name.clear(); - } - } - - // mark read/write buffer - graph::mark_read_or_write_buffers(arg_ins, true); - graph::mark_read_or_write_buffers(arg_out, false); - // build up final func name and param - std::vector args = arg_out, buffer_args; - args.insert(args.end(), arg_ins.begin(), arg_ins.end()); - std::for_each(args.begin(), args.end(), [&g](const expr &arg) { - COMPILE_ASSERT(arg.isa(), - "Only tensor node is expected for function argument, but " - "got " - << arg) - arg->attr()[mixed_partition_hint::cut_buffer] = true; - if (g.is_dynamic() || - g.attrs_.get_or_else("temp.parent_graph_dynamic", false)) { - arg->attr()[attr_keys::always_trans] = true; - } - }); - if (dyn_inter_) { - buffer_args = args; - args.emplace_back(dyn_inter_->inter_funcs_param_); - } - func_->params_ = args; - func_->decl_->params_ = args; - if (dyn_inter_) { - assert(dyn_inter_->inter_call_.defined()); - // replace output buffer of single core/internal func. - auto reset_args = [&buffer_args](std::vector &target_args, - const std::vector &extra_args) { - target_args = buffer_args; - target_args.insert(target_args.end(), extra_args.begin(), - extra_args.end()); - }; - reset_args(dyn_inter_->inter_call_->args_, - dyn_inter_->inter_call_extra_args_); - reset_args(dyn_inter_->inter_func_->params_, - dyn_inter_->inter_func_extra_args_); - dyn_inter_->inter_func_->decl_->params_ = dyn_inter_->inter_func_->params_; - reset_args(dyn_inter_->single_core_func_->params_, - dyn_inter_->single_core_func_extra_args_); - dyn_inter_->single_core_func_->decl_->params_ = - dyn_inter_->single_core_func_->params_; - } - // buffer schedule: declare, set shrink info, tensor initilize and query - // inplace - buffer_schedule(); - - // clear unused fanchor, in avoid of loop fuse break - clear_fanchors(); - - // remove all parallel flag - remove_parallel(func_, true); - - // auto push return to the end of body - auto &seq = func_->body_.checked_as()->seq_; - if (func_->ret_type_ == sc_data_type_t::boolean() && !seq.empty() && - !seq.back().isa()) { - seq.emplace_back(builder::make_returns_unattached(true)); - } - - // set function name - func_->name_ = get_parti_prefix(*this) + op_name; - func_->decl_->name_ = func_->name_; - // link op name - op_name = func_->name_; - - SC_MODULE_INFO << "mixed partition result:"; - SC_MODULE_INFO << func_; - - auto fused_op = std::make_shared( - op_name, - std::vector{ - std::static_pointer_cast(shared_from_this())}, - nullptr, copied_grpah, - /*ins*/ fused_op_in, - /*outs*/ - fused_op_out, any_map_t{}); - - return fused_op; -} - -using crossover_alg = - std::function &parti_vec)>; - -static void -crossover_dispatcher(const std::vector &parti_vec, - parti_merge_kind merge_kind) { - // select merger by merge kind - bool (*merger)(mixed_parti_t *A, mixed_parti_t *B); - switch (merge_kind) { - case parti_merge_kind::vertical: { - merger = try_merge_mixed_parti_vertically; - break; - } - case parti_merge_kind::horizontal: { - merger = try_merge_mixed_parti_horizontally; - break; - } - case parti_merge_kind::parallel: { - merger = try_merge_mixed_parti_parallel; - break; - } - default: - COMPILE_ASSERT(0, "Unknown partition merge kind found") - } - auto op_size = parti_vec.size(); - for (size_t i = 0; i < op_size; i++) { - auto parti_A = parti_vec[i]; - if (!parti_A) - continue; - for (size_t j = i; j < op_size; j++) { - auto parti_B = parti_vec[j]; - if (!parti_B) - continue; - merger(parti_A.get(), parti_B.get()); - } - } -} - -static void -horizontal_crossover(const std::vector &parti_vec) { - SC_MODULE_INFO << "Applying horizontal merge crossover algorithm..."; - crossover_dispatcher(parti_vec, parti_merge_kind::horizontal); -} - -static void -parallel_crossover(const std::vector &parti_vec) { - SC_MODULE_INFO << "Applying parallel merge crossover algorithm..."; - crossover_dispatcher(parti_vec, parti_merge_kind::parallel); -} - -static void -vertical_crossover(const std::vector &parti_vec) { - SC_MODULE_INFO << "Applying vertical merge crossover algorithm..."; - crossover_dispatcher(parti_vec, parti_merge_kind::vertical); -} - -static void crossover_partition(std::vector &op_2_partition, - const std::vector &algs) { - std::vector parti_vec = collect_parti_set(op_2_partition); - for (auto &al : algs) { - al(parti_vec); - } - for (auto &parti : op_2_partition) { - if (parti) { - parti = std::static_pointer_cast( - parti->get_root()->shared_from_this()); - } - } -} - -static expr merge_fusion_condition_by_parti_list( - const std::vector &partis) { - auto parti_set = collect_parti_set(partis); - expr ret = false; - for (auto &parti : parti_set) { - if (parti) { - ret = ret || parti->get_fusion_policy_condition(); - } - } - return ret; -} - -void do_mixed_partition(const context_ptr &ctx, sc_graph_t &graph) { - auto op_size = graph.ops_.size(); - // mapping from op id => partition - std::vector op_2_partition; - // set max iter times - constexpr int maxiter = 5; - // dynamic policy condition - expr fusion_policy_condition = false; - // make dependency matrix from graph here in avoid of repeated construction - // later when graph is extremely large - auto dep_m = std::make_shared(graph); - for (int i = 0; i < maxiter; i++) { - op_2_partition.clear(); - op_2_partition.resize(op_size); - bool ret = do_partition(ctx, graph, op_2_partition, dep_m); - auto cur_cond = merge_fusion_condition_by_parti_list(op_2_partition); - fusion_policy_condition = fusion_policy_condition || cur_cond; - if (ret) - break; - else if (i == maxiter - 1) { - SC_MODULE_INFO << "mixed partition exceed max iteration times, " - "please enlarge limitation and try again"; - return; - } - } - graph.attrs_.set("temp.fusion_policy_condition", fusion_policy_condition); - if (require_fusion_level(ctx, fusion_opt_level::lv3)) { - std::vector algs = {horizontal_crossover, parallel_crossover, - vertical_crossover}; - crossover_partition(op_2_partition, algs); - } - std::vector fused_ops; - for (auto &parti : op_2_partition) { - if (!parti || !parti->output_replace_map.empty() || parti->empty() || - parti->ops.size() == 1) { - // if a partition has been processed or it is empty or single op, - // skip it. - continue; - } - - auto fused_op = parti->transform_to_mixed_op(); - - fused_op->attrs_[mixed_partition_hint::parti] = - std::weak_ptr(parti); - fused_ops.emplace_back(fused_op); - } - - std::unordered_map tsr_replace_map; - for (auto &fused_op : fused_ops) { - auto partition = fused_op->attrs_[mixed_partition_hint::parti] - .get>() - .lock(); - assert(partition); - fused_op->attrs_.remove(mixed_partition_hint::parti); - for (auto &old_new : partition->output_replace_map) { - auto &old = old_new.first; - auto &newv = old_new.second; - old->replace_with(newv); - assert(tsr_replace_map.find(old) == tsr_replace_map.end()); - tsr_replace_map.insert(old_new); - } - for (auto &in : fused_op->info_.inputs_) { - // if an input is replaced by other fused_op node, update it - auto itr = tsr_replace_map.find(in); - if (itr != tsr_replace_map.end()) { - in = itr->second; - } - } - graph.add(fused_op); - - for (auto &op : partition->ops) { - op->remove(); - } - - // no main op is expected - assert(!partition->main_tunable_op); - } - graph.reset_op_ids(); -} - -void mixed_partition(sc_graph_t &graph, const context_ptr &ctx) { - if (!require_fusion_level(ctx, fusion_opt_level::lv1) || - !graph.attrs_.get_or_else("temp.fuse", 1)) { - return; - } - SC_MODULE_INFO << "Starting Mixed Partition..."; - do_mixed_partition(ctx, graph); -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/mixed_partition.hpp b/legacy/core/src/compiler/ir/graph/mixed_partition.hpp deleted file mode 100644 index 7b57cb4b3..000000000 --- a/legacy/core/src/compiler/ir/graph/mixed_partition.hpp +++ /dev/null @@ -1,536 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_MIXED_PARTITION_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_MIXED_PARTITION_HPP -#include "fused_op.hpp" -#include "fusion_anchor.hpp" -#include "fusion_cost_model.hpp" -#include "visitor.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -struct mixed_parti_t; - -namespace mixed_partition_hint { -// pointer: partition owner -constexpr const char *parti = "partition"; -// Boolean: is cut buffer hint -constexpr const char *cut_buffer = "cut_buffer"; -// Boolean: dont inplace hint -constexpr const char *no_inplace = "no_inplace"; -// Boolean: is optimized sub graph -constexpr const char *optimized_sub_graph = "optimized_sub_graph"; -// Boolean: is single op graph -constexpr const char *single_op_graph = "single_op_graph"; -// Boolean: is begining op of pre-op fuse -constexpr const char *pre_fuse_begin_op = "pre_fuse_begin_op"; -// Boolean: whether directly inplaced and would not call `compute_block` -constexpr const char *inplace_optimized_op = "inplace_optimized_op"; -// Boolean: is the first op for prefetching -constexpr const char *first_prefetch_op = "first_prefetch_op"; -// size_t: used to judge op whether is small workload. If less than threshold, -// ignore parallelism check of cost model. (Measured by 3D fp32 InstanceNorm) -constexpr size_t small_op_workload_threshold = 1688UL; -// Boolean: is the op which could not gather input partitions -constexpr const char *no_gather_op = "no_gather_op"; -// Boolean: is the op which need to split common anchor into grouped anchor -constexpr const char *split_anchor_op = "split_anchor_op"; -// Boolean: By default false. When marked as true, we will try to do fusion -// break. However, if it becomes standalone op after fuse break, it will be -// marked as false and we will remove the fuse break attribute -constexpr const char *trial_break = "trial_break"; -// Boolean: whether the tensor has been initialized -constexpr const char *tensor_already_initialized = "tensor_already_initialized"; -} // namespace mixed_partition_hint - -// different fusion policies prepared for dynamic shape, policies will be JIT -// and dispatched by conditions generated by dynamic cost model,/ -enum class dynamic_fusion_policy_t : int { - max_fusion = 0, - max_loop_parallelism -}; - -class mxp_replacer_t : public ir_inplace_visitor_t { -private: - node_ptr_map node_remap_; - -public: - mxp_replacer_t(node_ptr_map &node_remap) : node_remap_(node_remap) {} - - using ir_inplace_visitor_t::dispatch_impl; - using ir_inplace_visitor_t::visit_impl; - expr visit_impl(var v) override { - auto itr = node_remap_.find(v.impl); - if (itr != node_remap_.end()) { - changed_ = true; - return static_cast(itr->second.get())->node_ptr_from_this(); - } - return v; - } - - expr visit_impl(tensor v) override { - // visit and update user-defined tsr - if (v->attr_ && v->attr_->has_key(tensor_shrinker_attrs::should_shrink)) { - auto &shrink_info = v->attr_->get( - tensor_shrinker_attrs::should_shrink); - std::vector _new_expr; - ir_inplace_visitor_t::dispatch_expr_vector(shrink_info.base_, _new_expr); - ir_inplace_visitor_t::dispatch_expr_vector(shrink_info.shape_, _new_expr); - } - - auto itr = node_remap_.find(v.impl); - if (itr != node_remap_.end()) { - changed_ = true; - return static_cast(itr->second.get())->node_ptr_from_this(); - } - return v; - } - - stmt visit_impl(for_loop v) override { - // redirect reduce_root_loop if necessary - if (v->attr_) { - if (auto praw = v->attr_->get_or_null>( - stmt_attr_key::reduce_root_loop)) { - auto raw = praw->lock(); - COMPILE_ASSERT(raw, "reduce_root_loop weak ptr invalidated"); - auto itr = node_remap_.find(raw); - if (itr != node_remap_.end()) { - *praw = std::static_pointer_cast(itr->second); - } - } - } - return ir_inplace_visitor_t::visit_impl(v); - } - - void replace_func(func_t &func) { - if (func) { - dispatch_impl(func); - } - } - void replace_anchor(const std::vector &fanchors); -}; - -struct mxp_buffer_allocator { -private: - mixed_parti_t *binded_mxp_; - -public: - gt2buf_map g2b_map_; // record graph tensor to ir tensor/tensorptr - // mapping(maybe n-to-one) - std::unordered_map - tsr2anch_map_; // real tensor-to-anchor mapping - std::unordered_map - b2g_map_; // buffer-to-gt mapping(one-to-one) - - mxp_buffer_allocator(mixed_parti_t *parti) { binded_mxp_ = parti; } - - std::vector - mem_trace_; // memory allocation trace - - memory_optim::inplace_info_map inplace_map_; // inplace map - - mixed_parti_t *const get_binded_mxp() const { return binded_mxp_; } - // support inplace logic, allocate buffer including either tensor or - // tensorptr - void allocate_buffer(sc_op *op); - // set inplace hint - void set_buffer_inplace_hint(const expr &target_buf, const expr &inplace_buf); - // get inplaced buffer - expr get_inplaced_buffer(const expr &buf) const; - // get allocated buffer - std::tuple, std::vector> get_buffer(sc_op *op) const; - // update input buffer info - void update_input_buffer_info(sc_op *op); - // update output buffer info - void update_output_buffer_info(sc_op *op); - // define tensor - void declare_tensor() const; - // set shrink info - void set_shrink_info() const; - /** merge two buffer allocator - * @param common_anchor_pair: the common anchor overlapped when two - * partition merged. `first` comes from this partition, and `second` comes - * from other one. - * */ - void merge(mxp_buffer_allocator &other, - std::unordered_map &buffer_map, - const std::pair - &common_buffer_anchor_pair); - // clear buffer allocator - void clear(); - // check buffer_allocator whether empty - const bool empty() const { return g2b_map_.empty(); }; - // initilize tensor - void tensor_initialize(); - // replace the specific buffer - void replace_buffer(const expr &old_buffer, const expr &new_buffer); - // get real mem trace, taking consider of tensor shrink and ignore cut - // buffer except for those in `keep_cut_set` - std::vector get_real_mem_trace( - const std::unordered_set &keep_cut_set = {}) const; - // calculate real buffer usage size, taking consider of buffer schedule - size_t get_real_buffer_usage() const; - // get real anchor for the specfic buffer - fusion_anchor_ptr get_real_anchor_for_buffer(const expr &buffer) const; - // get shrinked info for buffer - slice_range get_shrinked_info(const expr &buffer) const; - // query buffer inplace and set hint for IR pass - void query_inplace(); - // calibrate buffer information about inplace and shrink - void calibrate_info(); - // validate tensor2var buffer whether meet the requirement - bool validate_tsr2var() const; - // count of buffer usage - int use_count(const expr &buffer) const; - // concat memory planning related - void copy_concat_memory_attrs_tsr2buf(); -}; - -struct mixed_dyn_internal_info_t { - // The module records internal functions usually contains repeat - // calculations which could be reused like single core brgemm. One partition - // could hold multiple ops who have internal functions. - ir_module_ptr mod_; - // extra parameter for internal func dispatch, a tensor of pointer. - expr inter_funcs_param_; - // internal call node of internal func. - call inter_call_; - // internal func node. - func_t inter_func_; - // single core func. - func_t single_core_func_; - // extra args for inter call - std::vector inter_call_extra_args_; - // extra args for inter func - std::vector inter_func_extra_args_; - // extra args of single core func - std::vector single_core_func_extra_args_; - // number of functions in partition. - int num_func_ = 0; - mixed_dyn_internal_info_t(const context_ptr &ctx) - : mod_(std::make_shared(ctx)) {} -}; -using mixed_dyn_internal_info_ptr = std::shared_ptr; -struct mixed_parti_t : fusion_partition_t { - /* related to Graph */ - // different from ops_ in base class, it records the sequence of committed - // ops in current partition - std::vector committed_ops_; - // dep matrix - dep_mat_ptr dep_m_; - - /* related to IR */ - context_ptr ctx_; - // the body of func_ will be updated once the new op is committed into - // current partition, but the name and argument maybe not confirmed until - // final mixed_fused_op created. - func_t func_; - // the fanchor only manage the shared pointer of fusion_anchor struct, - // during the whole lifetime of mixed_parti_t, it will not copy any - // fusion_anchor struct. - std::vector fanchors_; - // manage graph tensor to real tensor mapping - mxp_buffer_allocator buf_alloc_ = mxp_buffer_allocator(this); - // record the anchor to op mapping - std::unordered_map op_anchor_map_; - - // Cost Model - fusion_cost_model_ptr cost_; - // mixed fusion dyn internal info - mixed_dyn_internal_info_ptr dyn_inter_; - using ptr = std::shared_ptr; - - // append fusion anchor - void append_fusion_anchor(const fusion_anchor_ptr &fanchor); - - void append_fusion_anchor(const std::vector &fanchors) { - for (auto &fanchor : fanchors) { - append_fusion_anchor(fanchor); - } - } - - /** - * The mixed partition merge will override base merge method, including - * following several steps: - * 1. It will firstly check two partition dependency and decide which one is - * `to_merge` and another is `be_merged`. - * 2. extract `outer_loops` from each one and compute greatest common outer - * loops. - * 3. commit inner loop body from `be_merged` to the largest used fanchor of - * `to_merged`. - * 4. update outer and inner fusion anchor map. - * 5. replace expr iter/tensor/tensorptr in `func_`, `buf_alloc_` and - * `fanchors_`. - * 6. call base class `merge` method to do disjoint-set merge. - * void merge(const ptr &other); - * */ - - mixed_parti_t(const context_ptr &ctx, const sc_op_ptr &op, - const dep_mat_ptr &dep_m = nullptr); - - mixed_parti_t(const context_ptr &ctx, const func_t &func, - const fusion_anchor_mgr_t &fmgr, const sc_graph_t &graph); - - bool is_ok_to_add(sc_op *op); - - bool add(const sc_op_ptr &op); - - void remove(const sc_op_ptr &op) { - throw std::runtime_error("remove method is not implemented"); - } - - // if current partition contains no op or those ops generating no - // codes(like tensorview op), return True. - bool empty() const { - if (merged_to) { - return get_root()->empty(); - } - return (ops.empty() || !func_); - } - - mixed_parti_t *get_root() const { - return static_cast(fusion_partition_t::get_root()); - } - - size_t get_ops_size() const { return get_root()->ops.size(); } - - sc_op_ptr get_ith_op(size_t ith) const { - COMPILE_ASSERT(ith < get_root()->committed_ops_.size(), - "Could not get " << ith << "-th op") - return get_root()->committed_ops_[ith]; - } - - // get outer loops of which body(stmts) contains only one stmt or two with - // the second one is empty fanchor - std::vector get_outer_loops(fusion_anchor_ptr fanchor = nullptr, - bool skip_def_node = false) const; - - void try_split_outermost_loop(int64_t block) const; - void try_split_outermost_loop_on_num_threads(int64_t num_groups); - - // query if partition can optimize its loop order - bool can_optimize_outer_loop(bool allow_tensorview = false) const; - - // return op whether in op_anchor_map_ - bool ready_for_op(sc_op *op) const; - - // look up fanchor by op - fusion_anchor_ptr lookup_anchor_map(sc_op *op, - bool throw_assert = true) const; - - // look up fanchor by stmts - fusion_anchor_ptr lookup_anchor_map(const stmts &ss) const; - - // look up sub fanchor by parent fanchor - std::vector - lookup_sub_anchor_map(const fusion_anchor_ptr &parent_fanchor) const; - - // get anchor inside given loop - fusion_anchor_ptr get_anchor_inside_loop(const for_loop &loop, - bool input_anchor = false) const; - - /// get next inner loop including anchor - for_loop get_next_inner_loop_with_anchor( - const for_loop &cur_loop, - const fusion_anchor_ptr &target_fanchor = nullptr) const; - - // clear all contents of given fanchor, but not erase it from - // fanchor list - void clear_fanchor(fusion_anchor_ptr &fanchor); - - // clear all unused fanchor, and erase them from fanchor list - void clear_fanchors(); - - // try to bind given op with given fanchor, if suitable fanchor exists, it - // will compare two fanchor and select smaller one - void set_anchor_for_op(sc_op *op, const fusion_anchor_ptr &fanchor_map); - - // schedule buffer - void buffer_schedule(); - - // judge whether the given graph tensor node is the input of the whole - // partition - bool is_parti_inp(const graph_tensor_ptr >) const; - bool is_parti_inp(const graph_tensor *gt) const; - - // judge whether the given graph tensor node is the output of the whole - // partition - bool is_parti_out(const graph_tensor_ptr >) const; - bool is_parti_out(const graph_tensor *gt) const; - - bool is_parti_cut(const graph_tensor_ptr >) const { - return is_parti_inp(gt) || is_parti_out(gt); - } - bool is_parti_cut(const graph_tensor *gt) const { - return is_parti_inp(gt) || is_parti_out(gt); - } - - // count op number with given type - template size_t count_op_with_type() const { - if (merged_to) { - return get_root()->count_op_with_type(); - } - size_t cnt = 0; - for (auto &op : ops) { - if (op->isa()) - cnt++; - } - return cnt; - } - - // query partition whether contains input fusion anchor - bool contain_input_anchor() const; - - // query partition whether is constant partition - bool is_const_parti() const; - - // query partition whether contains op with given type - template bool contain_op_with_type() const { - if (merged_to) { - return get_root()->contain_op_with_type(); - } - return (count_op_with_type() != 0); - } - - // query partition whether contains op with conv type - bool contain_convolution() const; - - // query partition whether contains nested parallel for - bool contain_nested_parallel_for() const; - - // query partition whether contains tunable op - bool contain_tunable_op() const; - - // query partition whether contains only elementwise op - bool contain_elemwise_op_only() const; - - // query partition whether contains only one op - bool is_single_op_parti() const; - - // query partition whether contains op from optimized sub graph - bool is_optimized() const; - - // check optimization whether legal or not - bool validate_optimization() const; - - // clear all contents of partition object - void clear(); - - // call inner-build cost model to evaluate current partition, return the - // scores - float evaluate_perf(); - - // query partition whether is small workload - bool is_small_workload() const; - - // Get the condition expr generated by dynamic cost model. - expr get_fusion_policy_condition() const { - return cost_->get_fusion_policy_condition(); - } - - // get host graph - sc_graph_t &get_host_graph() const { - COMPILE_ASSERT(!committed_ops_.empty(), - "No op contained in current partition") - return committed_ops_[0]->get_owner_graph(); - } - - // transform partition to mixed fuse op - std::shared_ptr transform_to_mixed_op(); -}; - -enum class parti_merge_kind : int { - vertical = 0, - horizontal = 1, - parallel = 2, -}; - -void search_op_anchor_in_parti(sc_op *op, mixed_parti_t *parti); - -using mxp_mem_info = std::pair, - memory_optim::inplace_info_map>; - -mxp_mem_info merge_real_mem_info(const mxp_buffer_allocator &alloc1, - const mxp_buffer_allocator &alloc2); - -std::vector merge_mem_trace( - const std::vector &mem_trace1, - const std::vector &mem_trace2, - const std::unordered_map &buffer_map); - -memory_optim::inplace_info_map -merge_inplace_map(const memory_optim::inplace_info_map &inplace_map1, - const memory_optim::inplace_info_map &inplace_map2, - const std::unordered_map &buffer_map); - -size_t get_buffer_usage( - const context_ptr &ctx, - const std::vector &mem_trace, - const memory_optim::inplace_info_map &inplace_map); - -// collect unrepeated partition set, and optionally ignore pure const partition -std::vector -collect_parti_set(const std::vector &op_2_partition, - bool ignore_const = true); - -// do mixed partition -bool do_partition(const context_ptr &ctx, sc_graph_t &g, - std::vector &op_2_partition, - const dep_mat_ptr &dep_m); - -// judge the given graph whether is second time retried graph -inline bool is_optimized_sub_graph(sc_graph_t &g) { - return g.attrs_.get_or_else(mixed_partition_hint::optimized_sub_graph, false); -} - -// judge the given graph whether is the graph containing only one op -inline bool is_single_op_graph(sc_graph_t &g) { - return g.attrs_.get_or_else(mixed_partition_hint::single_op_graph, false); -} - -bool concat_memory_planning_on_graph(sc_graph_t &graph); - -// try optimize partition, such as reduce_op optimization -bool try_optimize_parti( - mixed_parti_t *parti, sc_graph_t &sub_graph, - const std::unordered_map &graph2orig_ops = {}); - -// get single mixed op from the graph -mixed_fuse_op_t *get_mixed_op_from_graph(sc_graph_t &graph); - -void do_mixed_partition(const context_ptr &ctx, sc_graph_t &graph); - -// commit graph to TIR, usually used in UT -void commit_graph_to_func(sc_graph_t &g, const func_t &func, - const fusion_anchor_mgr_t &fmgr); -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl -#endif diff --git a/legacy/core/src/compiler/ir/graph/pass/check.cpp b/legacy/core/src/compiler/ir/graph/pass/check.cpp deleted file mode 100644 index 8d7929b6e..000000000 --- a/legacy/core/src/compiler/ir/graph/pass/check.cpp +++ /dev/null @@ -1,69 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include - -#include "../graph.hpp" -#include "../visitor.hpp" -#include "pass.hpp" - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -// using pre-order and post-order visit graph, if any throw error and -// connection error, this function return false -bool check_graph_connection(sc_graph_t &graph) { - try { - op_visitor_t post_vis{ - op_visitor_t::pop_back_selector, - op_visitor_t::create_DAG_updater_post(graph.ops_.size()), true}; - // {logical op id, visited num} map - std::unordered_map visited_ops; - post_vis.post_visit_graph( - graph, [&](op_visitor_t *post_vis, const sc_op_ptr &aop) { - visited_ops[aop->logical_op_id_]++; - }); - - op_visitor_t::dfs_topology_sort(graph.ops_.size()) - .visit_graph(graph, [&](op_visitor_t *vis, const sc_op_ptr &aop) { - visited_ops[aop->logical_op_id_]--; - }); - for (auto op : visited_ops) { - if (op.second) { - return false; - } - } - } catch (...) { - return false; - } - return true; -} - -// check config in tunable op is valid or not. -bool check_graph_config(sc_graph_t &graph, const context_ptr &ctx) { - for (auto &node : graph.ops_) { - if (!node->is_removed_ && !node->is_valid(ctx)) { - return false; - } - } - return true; -} -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/pass/comparer.cpp b/legacy/core/src/compiler/ir/graph/pass/comparer.cpp deleted file mode 100644 index 5cb1c76e1..000000000 --- a/legacy/core/src/compiler/ir/graph/pass/comparer.cpp +++ /dev/null @@ -1,170 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include - -#include "../graph.hpp" -#include "../visitor.hpp" -#include "pass.hpp" - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -bool compare_graph( - sc_op_ptr &first_diff_lhs, sc_op_ptr &first_diff_rhs, const sc_graph_t &lhs, - const sc_graph_t &rhs, - const std::unordered_map &lhs_rhs_input_mapping, - const std::function &filter) { - std::vector op_id_map(lhs.ops_.size(), -1); - std::unordered_map tsr_map; - if (lhs_rhs_input_mapping.empty()) { - auto inops_lhs = lhs.get_input_or_const_ops(); - auto inops_rhs = rhs.get_input_or_const_ops(); - if (inops_lhs.size() != inops_rhs.size()) { - return false; - } - for (size_t i = 0; i < inops_lhs.size(); i++) { - auto &l = inops_lhs[i]; - auto &r = inops_rhs[i]; - op_id_map.at(l->logical_op_id_) = r->logical_op_id_; - } - } else { - for (auto &lhs_rhs_id_map : lhs_rhs_input_mapping) { - op_id_map.at(lhs_rhs_id_map.first) = lhs_rhs_id_map.second; - } - } - bool not_same = false; - op_visitor_t visitor{/*selector with early stopping*/ - [¬_same](op_visitor_t *vis) { - auto ret = op_visitor_t::pop_back_selector(vis); - return not_same ? sc_op_ptr() : ret; - }, - op_visitor_t::create_DAG_updater(lhs.ops_.size()), - false}; - - // a utility macro to return from the visit function and tell the visitor to - // stop early -#define DO_RETURN \ - { \ - not_same = true; \ - first_diff_lhs = lnode; \ - first_diff_rhs = rnode; \ - return; \ - } - - visitor.visit_graph(lhs, [&](op_visitor_t *visitor, const sc_op_ptr &lnode) { - sc_op_ptr rnode; - int rhs_node_idx = op_id_map[lnode->logical_op_id_]; - // if cannot find RHS - if (rhs_node_idx == -1) { - DO_RETURN; - } - rnode = rhs.ops_.at(rhs_node_idx); - // check the name and attrs - if (!lnode->compare_contents(rnode.get(), filter)) { - DO_RETURN; - } - // check the mapping of input tensors. because the input nodes - // are already visited (DAG_updater), we just need to check if - // the LHS tensor is correctly mapped to RHS tensor - auto &lhs_inputs = lnode->get_inputs(); - auto &rhs_inputs = rnode->get_inputs(); - if (lhs_inputs.size() != rhs_inputs.size()) { - DO_RETURN; - } - for (size_t i = 0; i < lhs_inputs.size(); i++) { - auto lhs_tsr = lhs_inputs[i].get(); - auto rhs_tsr = rhs_inputs[i].get(); - auto itr = tsr_map.find(lhs_tsr); - if (itr == tsr_map.end() || itr->second != rhs_tsr) { - DO_RETURN; - } - } - - // the output tensors are owned by the current node, so they - // should have not been visited. For each output tensor, we need - // to: - // 1. check if the logical tensor contents are exactly the same - // (e.g. shape, dtype, etc.) - // 2. set the lhs_tsr => rhs_tsr mapping (the mapping will be - // checked when the tsr is an input of other ops) - // 3. check if the sizes of uses_ matches for LHS and RHS - // 4. for each consumer Op of the tensor (which we call "use"), - // set the op mapping from LHS to RHS - - auto &lhs_outputs = lnode->get_outputs(); - auto &rhs_outputs = rnode->get_outputs(); - if (lhs_outputs.size() != rhs_outputs.size()) { - DO_RETURN; - } - for (size_t i = 0; i < lhs_outputs.size(); i++) { - auto lhs_tsr = lhs_outputs[i].get(); - auto rhs_tsr = rhs_outputs[i].get(); - // 1. contents - if (!(lhs_tsr->details_ == rhs_tsr->details_)) { - DO_RETURN; - } - // 2. tsr mapping - auto itr = tsr_map.find(lhs_tsr); - // if lhs_tsr is already mapped to a rhs, return false - if (itr != tsr_map.end()) { - DO_RETURN; - } - tsr_map.insert(std::make_pair(lhs_tsr, rhs_tsr)); - // 3. size of uses - auto &lhs_uses = lhs_tsr->uses_; - auto &rhs_uses = rhs_tsr->uses_; - if (lhs_uses.size() != rhs_uses.size()) { - DO_RETURN; - } - // 4. op mapping - for (size_t u = 0; u < lhs_uses.size(); u++) { - auto &lhs_use = lhs_uses[u]; - auto &rhs_use = rhs_uses[u]; - if (lhs_use.first != rhs_use.first) { - DO_RETURN; - } - int &mapped_id = op_id_map.at(lhs_use.second->logical_op_id_); - // if there is already a mapping in the op map, and the - // mapped RHS does not match rhs_use's Op id, return - // false - if (mapped_id != -1 && mapped_id != rhs_use.second->logical_op_id_) { - DO_RETURN; - } - // set the mapping. When visiting the node, we need this - // mapping to find the RHS - mapped_id = rhs_use.second->logical_op_id_; - } - } - }); - return !not_same; -} - -bool compare_graph( - const sc_graph_t &lhs, const sc_graph_t &rhs, - const std::unordered_map &lhs_rhs_input_mapping, - const std::function &filter) { - sc_op_ptr first_diff_lhs, first_diff_rhs; - return compare_graph(first_diff_lhs, first_diff_rhs, lhs, rhs, - lhs_rhs_input_mapping, filter); -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/pass/dynamic_infer_shape.cpp b/legacy/core/src/compiler/ir/graph/pass/dynamic_infer_shape.cpp deleted file mode 100644 index a09aa55cb..000000000 --- a/legacy/core/src/compiler/ir/graph/pass/dynamic_infer_shape.cpp +++ /dev/null @@ -1,229 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "../transform/transform.hpp" -#include "../visitor.hpp" -#include "pass.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -using namespace ops; -SC_MODULE(graph.pass.dynamic_infer_shape) -static void print_shapes(const std::string &name, - runtime::dynamic_tensor_t *in) { - std::stringstream ss; - ss << "Shape after op " << name << " is :["; - for (int i = 0; i < in->ndims_; i++) { - if (i) { - ss << ","; - } - ss << std::to_string(in->dims_[i]); - } - ss << "]\n"; - SC_MODULE_INFO << ss.str(); -} -SC_API void dynamic_infer_shape_by_graph(sc_graph_t &graph, - runtime::dynamic_tensor_t **ins, - runtime::dynamic_tensor_t **outs, - size_t num_ins, size_t num_outs) { - if (graph.empty() || !graph.is_dynamic()) { - return; - } - COMPILE_ASSERT(num_ins == graph.get_input_ops().size() && - num_outs == graph.get_output_ops().size(), - "Given input/output number does not match the graph."); - std::unordered_map ltsr_dtsr; - std::vector inner_dyn_tsr; - std::vector> inner_shape_tsr; - - auto inp_ops = graph.get_input_ops(); - auto out_ops = graph.get_output_ops(); - for (size_t i = 0; i < num_ins; i++) { - ltsr_dtsr[inp_ops[i]->get_outputs()[0]] = ins[i]; - } - for (size_t i = 0; i < num_outs; i++) { - ltsr_dtsr[out_ops[i]->get_inputs()[0]] = outs[i]; - } - // avoid realloc - size_t ltsr_num = 0; - for (auto &op : graph.ops_) { - ltsr_num += op->get_inputs().size() + op->get_outputs().size(); - } - ltsr_num -= inp_ops.size() + out_ops.size(); - inner_dyn_tsr.reserve(ltsr_num); - inner_shape_tsr.reserve(ltsr_num); - auto get_or_create_dyn_tsr = [<sr_dtsr, &inner_dyn_tsr, - &inner_shape_tsr](const graph_tensor_ptr &in) { - auto it = ltsr_dtsr.find(in); - if (it != ltsr_dtsr.end()) { - return it->second; - } - auto plain_dims = in->details_.get_plain_dims(); - inner_shape_tsr.emplace_back(plain_dims); - inner_dyn_tsr.emplace_back(nullptr, inner_shape_tsr.back().data(), - static_cast(plain_dims.size()), 0, 0); - ltsr_dtsr.insert(std::make_pair(in, &inner_dyn_tsr.back())); - return &inner_dyn_tsr.back(); - }; - op_visitor_t vis = op_visitor_t::dfs_topology_sort(graph.ops_.size()); - vis.visit_graph(graph, [&get_or_create_dyn_tsr](op_visitor_t *vis, - const sc_op_ptr &node) { - if (node->isa() || node->isa()) { - return; - } else if (node->isa()) { - // don't need to infer. - get_or_create_dyn_tsr(node->get_outputs()[0]); - } else if (node->isa() || - node->isa()) { - auto *data = get_or_create_dyn_tsr(node->get_inputs()[0]); - auto *weight = get_or_create_dyn_tsr(node->get_inputs()[1]); - auto *out = get_or_create_dyn_tsr(node->get_outputs()[0]); - infer_shape_matmul_op(out, data, weight); - print_shapes(node->op_name_, out); - } else if (node->isa()) { - auto *data = get_or_create_dyn_tsr(node->get_inputs()[0]); - auto *weight = get_or_create_dyn_tsr(node->get_inputs()[1]); - auto *out = get_or_create_dyn_tsr(node->get_outputs()[0]); - auto &stride = node->attrs_.get("strides"); - auto dilations = get_dilations(node->attrs_); - sc_dims dilation_dims(data->ndims_ - 2, dilations[0]); - if (dilations.size() > 1) { - dilation_dims = dilations; - } - auto &pads_begin = node->attrs_.has_key("pads_begin") - ? node->attrs_.get("pads_begin") - : node->attrs_.get("paddings"); - auto &pads_end = node->attrs_.has_key("pads_end") - ? node->attrs_.get("pads_end") - : node->attrs_.get("paddings"); - auto dyn_conv_info = - data->ndims_ == 4 - ? dyn_conv_fwd_runtime_info_t(stride[0], stride[1], pads_begin[0], - pads_begin[1], pads_end[0], - pads_end[1], dilation_dims[0], - dilation_dims[1]) - : dyn_conv_fwd_runtime_info_t( - stride[0], stride[1], stride[2], pads_begin[0], - pads_begin[1], pads_begin[2], pads_end[0], pads_end[1], - pads_end[2], dilation_dims[0], dilation_dims[1], - dilation_dims[2]); - infer_shape_conv_fwd_op(out, data, weight, dyn_conv_info); - print_shapes(node->op_name_, out); - } else if (node->isa() || - node->isa() || - node->isa() || - node->isa()) { - auto *in = get_or_create_dyn_tsr(node->get_inputs()[0]); - auto *out = get_or_create_dyn_tsr(node->get_outputs()[0]); - infer_shape_unary_fusible_op(out, in); - print_shapes(node->op_name_, out); - } else if (node->isa()) { - auto *in0 = get_or_create_dyn_tsr(node->get_inputs()[0]); - auto *in1 = get_or_create_dyn_tsr(node->get_inputs()[1]); - auto *out = get_or_create_dyn_tsr(node->get_outputs()[0]); - infer_shape_binary_fusible_op(out, in0, in1); - print_shapes(node->op_name_, out); - } else if (node->isa() || node->isa()) { - auto *in = get_or_create_dyn_tsr(node->get_inputs()[0]); - auto *out = get_or_create_dyn_tsr(node->get_outputs()[0]); - auto rd_axis = node->attrs_.get>("rd_axis"); - infer_shape_reduce_op(out, in, rd_axis.data(), - static_cast(rd_axis.size())); - print_shapes(node->op_name_, out); - } else if (node->isa()) { - auto *in = get_or_create_dyn_tsr(node->get_inputs()[0]); - auto *out = get_or_create_dyn_tsr(node->get_outputs()[0]); - auto order = node->attrs_.get>("order"); - infer_shape_transpose_op(out, in, order.data(), - static_cast(order.size())); - print_shapes(node->op_name_, out); - } else if (node->isa()) { - auto *in = get_or_create_dyn_tsr(node->get_inputs()[0]); - auto *out = get_or_create_dyn_tsr(node->get_outputs()[0]); - auto new_shape = node->attrs_.get>("shape"); - auto input_plain_dims = node->get_inputs()[0]->details_.get_plain_dims(); - infer_shape_tensor_view_op(out, in, input_plain_dims.data(), - static_cast(input_plain_dims.size()), - new_shape.data(), - static_cast(new_shape.size())); - print_shapes(node->op_name_, out); - } else if (node->isa()) { - auto *in0 = get_or_create_dyn_tsr(node->get_inputs()[0]); - auto *in1 = get_or_create_dyn_tsr(node->get_inputs()[1]); - auto *in2 = get_or_create_dyn_tsr(node->get_inputs()[2]); - auto *out = get_or_create_dyn_tsr(node->get_outputs()[0]); - infer_shape_select_op(out, in0, in1, in2); - print_shapes(node->op_name_, out); - } else if (node->isa()) { - auto *data = get_or_create_dyn_tsr(node->get_inputs()[0]); - auto *out = get_or_create_dyn_tsr(node->get_outputs()[0]); - auto pooling_op = node->stc_cast(); - std::string auto_pad = pooling_op->attrs_.get_or_else( - pooling_attr_key::auto_pad, auto_pad_options::none); - auto auto_pads_same = auto_pad == auto_pad_options::same_upper || - auto_pad == auto_pad_options::same_lower; - auto rounding_type_floor = - pooling_op->attrs_.get_or_else( - pooling_attr_key::rounding_type, rounding_type_options::floor) == - rounding_type_options::floor; - auto dyn_info = - data->ndims_ == 5 - ? dyn_pooling_runtime_info_t( - pooling_op->stride_[0], pooling_op->stride_[1], - pooling_op->stride_[2], pooling_op->pads_begin_[0], - pooling_op->pads_begin_[1], pooling_op->pads_begin_[2], - pooling_op->pads_end_[0], pooling_op->pads_end_[1], - pooling_op->pads_end_[2], pooling_op->kernel_[0], - pooling_op->kernel_[1], pooling_op->kernel_[2], - rounding_type_floor, auto_pads_same) - : dyn_pooling_runtime_info_t( - pooling_op->stride_[0], pooling_op->stride_[1], - pooling_op->pads_begin_[0], pooling_op->pads_begin_[1], - pooling_op->pads_end_[0], pooling_op->pads_end_[1], - pooling_op->kernel_[0], pooling_op->kernel_[1], - rounding_type_floor, auto_pads_same); - infer_shape_pooling_fusible_op(out, data, dyn_info); - print_shapes(node->op_name_, out); - } - - else { - COMPILE_ASSERT(false, - "Unsupported op for shape inference: " << node->op_name_); - } - }); -} -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/pass/graph_code_cache.cpp b/legacy/core/src/compiler/ir/graph/pass/graph_code_cache.cpp deleted file mode 100644 index eeeb4677c..000000000 --- a/legacy/core/src/compiler/ir/graph/pass/graph_code_cache.cpp +++ /dev/null @@ -1,343 +0,0 @@ -/******************************************************************************* - * Copyright 2023-2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "pass.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -SC_MODULE(graph.pass.code_cache); - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -struct prehashed_graph_for_code_share_t { - context_ptr ctx_; - variant> g_; - // the ids for the base tensors for each shared const tensor - std::vector base_tensor_id_; - // the offset in the base tensors for each shared const tensor - std::vector offset_; - size_t hash_; - static bool attr_filter(const sc_op *op, const std::string &attr) { - if (attr == op_attr_key::const_input_cache) { - return false; - } - if (op->isa()) { - if (attr == "values") { - return false; - } - } - static std::unordered_set ignorable_keys{ - attr_keys::scales, attr_keys::zero_points, - attr_keys::data_scales, attr_keys::data_zero_points, - attr_keys::weight_scales, attr_keys::weight_zero_points}; - if (ignorable_keys.find(attr) != ignorable_keys.end()) { - return false; - } - return true; - } - - prehashed_graph_for_code_share_t(const context_ptr &ctx, sc_graph_t &&g, - const std::vector &base_tensor_id, - const std::vector &offset, - size_t ghash) - : ctx_(ctx), g_(std::move(g)), base_tensor_id_(base_tensor_id), - offset_(offset) { - hash_ = ghash; - hash_combine(hash_, base_tensor_id_); - hash_combine(hash_, offset_); - hash_combine(hash_, ctx_); - } - - prehashed_graph_for_code_share_t(const context_ptr &ctx, - std::reference_wrapper g, - const std::vector &base_tensor_id, - const std::vector &offset, - size_t ghash) - : ctx_(ctx), g_(g), base_tensor_id_(base_tensor_id), offset_(offset) { - hash_ = ghash; - hash_combine(hash_, base_tensor_id_); - hash_combine(hash_, offset_); - hash_combine(hash_, ctx_); - } - bool operator==(const prehashed_graph_for_code_share_t &other) const { - if (hash_ != other.hash_) { - return false; - } - if (ctx_ != other.ctx_) { - return false; - } - if (base_tensor_id_ != other.base_tensor_id_) { - return false; - } - if (offset_ != other.offset_) { - return false; - } - return compare_graph(g_.cast(), - g_.cast(), {}, attr_filter); - } -}; - -std::ostream &operator<<(std::ostream &os, - const prehashed_graph_for_code_share_t &v) { - os << "hash=" << v.hash_ - << ", base tsr=" << utils::print_vector(v.base_tensor_id_) - << ", offsets=" << utils::print_vector(v.offset_); - return os; -} - -struct graph_shared_ptr_hasher_t { - size_t operator()(const prehashed_graph_for_code_share_t &v) const { - return v.hash_; - } -}; - -struct graph_code_cache_handle; -struct graph_code_cache_manager; - -struct graph_code_cache_value_t { - std::weak_ptr v_; - std::shared_ptr deletion_flag_; -}; - -using graph_code_cache_map = - std::unordered_map; - -struct graph_code_cache_handle { - std::weak_ptr code_; - statics_table_t module_data_template_; - graph_code_cache_map::iterator iter_; - std::shared_ptr deletion_flag_; - std::shared_ptr mgr_; - graph_code_cache_handle(const std::weak_ptr &code, - statics_table_t &&module_data, - const std::shared_ptr &mgr) - : code_(code), module_data_template_(std::move(module_data)), - deletion_flag_(std::make_shared(false)), mgr_(mgr) {} - ~graph_code_cache_handle(); -}; - -static const char *shared_const_handle_name = "__shared_const_handle"; -statics_table_t -prepare_static_table_for_cached_code(graph_code_cache_handle &v, - const sc_graph_t &orig_graph) { - statics_table_t ret = v.module_data_template_.copy(); - if (auto bases = orig_graph.attrs_.get_or_null< - std::vector>>( - "shared_const_bases")) { - if (!bases->empty()) { - void **phandles = (void **)ret.get_or_null(shared_const_handle_name); - COMPILE_ASSERT(phandles, - "Cannot find shared_const_handle_name in module data"); - for (size_t i = 0; i < bases->size(); i++) { - phandles[i] = (*bases)[i]->is_lazy_ - ? (*bases)[i].get() - : (*bases)[i]->get_buffer_if_not_lazy(); - } - ret.shared_tensors_ = - orig_graph.attrs_ - .get>>( - "shared_const_tensors"); - } - } - return ret; -} - -static std::shared_ptr get_cache_mgr(); - -struct graph_code_cache_manager { - std::mutex lock_; - graph_code_cache_map map_; - - std::shared_ptr - internal_query(const prehashed_graph_for_code_share_t &v) { - auto itr = map_.find(v); - if (itr != map_.end()) { - // find the key in the map, and make sure the weakptr are alive. If - // the weakptr expired, remove the key from the map - auto ret = - some_opt(itr->second.v_.lock()) - .map([](const std::shared_ptr &v) { - return v->code_.lock(); - }) - .get_or_else(nullptr); - if (!ret) { - // notify the dtor of graph_code_cache_handle that the key is - // already removed - *itr->second.deletion_flag_ = false; - map_.erase(itr); - return nullptr; - } - return ret; - } - return nullptr; - } - - std::shared_ptr - query(const prehashed_graph_for_code_share_t &v) { - std::lock_guard guard{lock_}; - return internal_query(v); - } - - std::shared_ptr - insert(prehashed_graph_for_code_share_t &&v, const jit_module &m) { - std::lock_guard guard{lock_}; - if (internal_query(v)) { - SC_MODULE_INFO << "The graph is already in the cache"; - return nullptr; - } - - auto value = std::make_shared( - m.code_, m.globals_.copy(), get_cache_mgr()); - auto graph_iter = map_.insert(std::make_pair( - std::move(v), graph_code_cache_value_t{value, value->deletion_flag_})); - assert(graph_iter.second); - value->iter_ = graph_iter.first; - m.code_->graph_cache_handle_ = value; - SC_MODULE_INFO << "Putting into code cache, " << graph_iter.first->first; - if (auto sc_stream_temp = - ::dnnl::impl::graph::gc::runtime::get_info_logging_stream( - "graph.pass.verbose.code_cache")) { - print_graph(graph_iter.first->first.g_.cast(), - *sc_stream_temp.stream_, true, true); - } - return value; - } - - void remove(graph_code_cache_handle &v) { - std::lock_guard guard{lock_}; - /* - see why the use of deletion_flag_ in const_graph_tensor_cache::remove - */ - bool already_deleted = *v.deletion_flag_; - if (already_deleted) { - return; - } - if (v.iter_ != map_.end()) - map_.erase(v.iter_); - } -}; - -graph_code_cache_handle::~graph_code_cache_handle() { mgr_->remove(*this); } - -static std::shared_ptr get_cache_mgr() { - static std::shared_ptr ret = - std::make_shared(); - return ret; -} - -size_t query_cached_code_of_context(const context_ptr &ctx) { - auto mgr = get_cache_mgr(); - size_t ret = 0; - std::lock_guard guard{mgr->lock_}; - for (auto &kv : mgr->map_) { - if (kv.first.ctx_ == ctx) - ret++; - } - return ret; -} - -std::shared_ptr register_code_in_graph_cache( - const jit_module &m, - std::shared_ptr &&key) { - return get_cache_mgr()->insert(std::move(*key), m); -} - -SC_INTERNAL_API void graph_code_cache(sc_graph_t &mgr, const context_ptr &ctx) { - if (!ctx->flags_.const_share_) { - return; - } - if (ctx->engine_->vtable_->get_tensor_cache_cap(ctx->engine_) == 0) { - // if constant cache is off - return; - } - // collect the base tensors and offsets - std::vector> bases; - std::vector base_ids; - std::vector offsets; - std::vector> shared_tsr; - op_visitor_t::dfs_topology_sort(mgr.ops_.size()) - .visit_graph(mgr, [&](op_visitor_t *, const sc_op_ptr &op) { - if (auto buffer = - op->attrs_.get_or_null< - std::vector>>( - op_attr_key::const_input_cache)) { - for (auto &buf : *buffer) { - shared_tsr.emplace_back(buf); - auto itr = std::find(bases.begin(), bases.end(), buf->buf_base_); - if (itr != bases.end()) { - base_ids.push_back(static_cast(itr - bases.begin())); - } else { - base_ids.push_back(bases.size()); - bases.emplace_back(buf->buf_base_); - } - offsets.push_back(buf->offset_); - } - } - }); - mgr.attrs_["shared_const_bases"] = bases; - mgr.attrs_["shared_const_tensors"] = shared_tsr; - auto cache_mgr = get_cache_mgr(); - auto ghash = mgr.hash_contents(prehashed_graph_for_code_share_t::attr_filter); - prehashed_graph_for_code_share_t key{ - ctx, std::reference_wrapper(mgr), base_ids, offsets, - ghash}; - - if (auto cached = cache_mgr->query(key)) { - mgr.attrs_["graph_code_cache"] = cached; - SC_MODULE_INFO << "Found cached code for the graph: " << key; - return; - } - auto new_graph = copy_graph(mgr); - // cleanup attrs - for (auto &op : new_graph.ops_) { - auto &attrs = op->attrs_.as_map(); - for (auto itr = attrs.begin(); itr != attrs.end();) { - if (utils::string_startswith(itr->first, "temp.") || - !prehashed_graph_for_code_share_t::attr_filter(op.get(), - itr->first)) { - itr = attrs.erase(itr); - continue; - } - ++itr; - } - } - auto cache_key = std::make_shared( - ctx, std::move(new_graph), base_ids, offsets, ghash); - mgr.attrs_["graph_code_cache_key"] = cache_key; -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/pass/graph_code_cache.hpp b/legacy/core/src/compiler/ir/graph/pass/graph_code_cache.hpp deleted file mode 100644 index 94ac5f0ad..000000000 --- a/legacy/core/src/compiler/ir/graph/pass/graph_code_cache.hpp +++ /dev/null @@ -1,72 +0,0 @@ -/******************************************************************************* - * Copyright 2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_PASS_GRAPH_CODE_CACHE_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_PASS_GRAPH_CODE_CACHE_HPP - -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -/** - * The graph code cache uses prehashed_graph_for_code_share_t as the key and - * graph_code_cache_handle as the value. It contains a weakptr to the - * jit_module_code. The jit_module_code should hold a sharedptr to this handle. - * When a jit_module_code is destructed, it will destroy its - * graph_code_cache_handle as well. This will automatically unregister the graph - * in graph code cache - */ -struct graph_code_cache_handle; - -/** - * The key in graph code cache - */ -struct prehashed_graph_for_code_share_t; -class sc_graph_t; -struct jit_module; - -/** - * Create and prepare the JIT module data of a cached graph. - */ -statics_table_t -prepare_static_table_for_cached_code(graph_code_cache_handle &v, - const sc_graph_t &orig_graph); - -/** - * Register the compilation result and the query key (graph) into the graph code - * cache. - * - * @returns the pointer to the cache item handle. This can be null if the key - * already exists in the cache - */ -std::shared_ptr register_code_in_graph_cache( - const jit_module &m, - std::shared_ptr &&key); - -// get the number of cache code for a given context -size_t query_cached_code_of_context(const context_ptr &ctx); - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/graph/pass/graph_constant_cache.hpp b/legacy/core/src/compiler/ir/graph/pass/graph_constant_cache.hpp deleted file mode 100644 index e55a502a3..000000000 --- a/legacy/core/src/compiler/ir/graph/pass/graph_constant_cache.hpp +++ /dev/null @@ -1,91 +0,0 @@ -/******************************************************************************* - * Copyright 2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_PASS_GRAPH_CONSTANT_CACHE_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_PASS_GRAPH_CONSTANT_CACHE_HPP - -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -namespace runtime { -struct const_cache_proxy; -} - -struct cached_const_graph_tensor; -struct const_graph_tensor_cache; -class sc_graph_t; - -// the value for the graph-cached_const_graph_tensor map. Holds an deletion_flag -// indicating that the key is already removed or not -struct flaged_cached_const_graph_tensor_t { - std::weak_ptr v_; - std::shared_ptr deletion_flag_; -}; -struct graph_weak_ptr_hasher { - size_t operator()(const std::weak_ptr &v) const; -}; -struct graph_weak_ptr_cmper { - bool operator()(const std::weak_ptr &v1, - const std::weak_ptr &v2) const; -}; - -using graph_weak_ptr_map = - std::unordered_map, - flaged_cached_const_graph_tensor_t, - graph_weak_ptr_hasher, graph_weak_ptr_cmper>; - -using tensor_id_map = - std::unordered_map>; - -struct cached_const_graph_tensor { - std::shared_ptr dependency_; - // the data pointer buf_base_ is initialized null. It will be later updated - // when finishing visiting the whole graph in graph_constant_input_folding - // pass. The update of this pointer is protected by - // shared_global_data_allocator_t::lock_ - size_t size_; - graph_weak_ptr_map::iterator graph_iter_; - tensor_id_map::iterator id_iter_; - std::shared_ptr cache_owner_; - const std::shared_ptr deletion_flag_; - // the base pointer of buf_. buf_ may be cut from a larger buffer buf_base_. - std::shared_ptr buf_base_; - // the offset of buf_ on the base buffer of buf_base_ - size_t offset_ = 0; - cached_const_graph_tensor( - const std::shared_ptr &dep, size_t buf_size, - const std::shared_ptr &owner) - : dependency_{dep}, size_{buf_size}, cache_owner_{owner}, - deletion_flag_{std::make_shared(false)} {} - ~cached_const_graph_tensor(); -}; - -namespace op_attr_key { -constexpr const char *const_input_cache = "temp.const_input_cache"; -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/graph/pass/graph_constant_input_folding.cpp b/legacy/core/src/compiler/ir/graph/pass/graph_constant_input_folding.cpp deleted file mode 100644 index 211588e06..000000000 --- a/legacy/core/src/compiler/ir/graph/pass/graph_constant_input_folding.cpp +++ /dev/null @@ -1,481 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "../fused_op.hpp" -#include "../fusible_op.hpp" -#include "../visitor.hpp" -#include "graph_constant_cache.hpp" -#include "pass.hpp" -#include -#include -#include -#include -#include -#include -#include -#include - -SC_MODULE(graph.pass.const_input_fold); - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -size_t -graph_weak_ptr_hasher::operator()(const std::weak_ptr &v) const { - auto g = v.lock(); - if (!g) { - return 0; - } - return g->hash_contents(); -} -bool graph_weak_ptr_cmper::operator()( - const std::weak_ptr &v1, - const std::weak_ptr &v2) const { - auto g1 = v1.lock(); - if (!g1) { - return false; - } - auto g2 = v2.lock(); - if (!g2) { - return false; - } - if (g1 == g2) { - return true; - } - return compare_graph(*g1, *g2); -} - -struct shared_global_data_allocator_t { - std::mutex lock_; - size_t needed_allocation_size( - const std::vector> &cache, - const std::vector &existing_data, size_t &out_real_const) { - size_t ret_lazy = 0, ret_compile_time_const = 0; - for (size_t i = 0; i < cache.size(); i++) { - auto &v = cache[i]; - size_t &val = existing_data[i] ? ret_compile_time_const : ret_lazy; - if (!v->buf_base_) { - val += utils::divide_and_ceil(v->size_, 64) * 64; - } - } - out_real_const = ret_compile_time_const; - return ret_lazy; - } - void - alloc(const std::vector> &cache, - const std::vector &existing_data, runtime::engine_t *engine) { - size_t ret_lazy = 0, ret_compile_time_const = 0; - ret_lazy = - needed_allocation_size(cache, existing_data, ret_compile_time_const); - if (ret_lazy + ret_compile_time_const) { - std::lock_guard guard{lock_}; - // do double-check locking - ret_lazy = - needed_allocation_size(cache, existing_data, ret_compile_time_const); - if (ret_lazy) { - // need to register into const cache manager for - // lazy-initialized buffers - auto base = - engine->vtable_->alloc_and_register_tensor_cache(engine, ret_lazy); - size_t offset = 0; - for (size_t idx = 0; idx < cache.size(); idx++) { - auto &v = cache[idx]; - if (existing_data[idx]) - continue; - // the update on buf_ is protected by lock_ - if (!v->buf_base_) { - v->buf_base_ = base; - v->offset_ = offset; - offset += utils::divide_and_ceil(v->size_, 64) * 64; - SC_MODULE_INFO << "Alloc buffer for " << v - << ", offset=" << v->offset_; - } - } - } - if (ret_compile_time_const) { - // no need to register into const cache manager for - // compile-time constants because they are never evicted - std::shared_ptr baseptr = std::shared_ptr{ - engine->vtable_->persistent_alloc(engine, ret_compile_time_const), - [engine](void *p) { - engine->vtable_->persistent_dealloc(engine, p); - }}; - auto base = std::make_shared( - baseptr, baseptr.get(), ret_compile_time_const, - /*lazy*/ false); - size_t offset = 0; - for (size_t idx = 0; idx < cache.size(); idx++) { - auto &v = cache[idx]; - if (!existing_data[idx]) - continue; - // the update on buf_ is protected by lock_ - if (!v->buf_base_) { - v->buf_base_ = base; - v->offset_ = offset; - memcpy((char *)baseptr.get() + offset, existing_data[idx], - v->size_); - offset += utils::divide_and_ceil(v->size_, 64) * 64; - SC_MODULE_INFO << "Alloc buffer for " << v - << ", offset=" << v->offset_; - } - } - } - } - } -}; - -static std::shared_ptr get_cache(); -struct const_graph_tensor_cache { - std::mutex lock_; - tensor_id_map from_tensor_id_; - graph_weak_ptr_map from_dep_graph_; - shared_global_data_allocator_t alloca_; - - std::shared_ptr - add_tensor(const std::shared_ptr &dep_graph, size_t buf_size, - runtime::engine_t *engine) { - std::lock_guard guard{lock_}; - auto itr = from_dep_graph_.find(dep_graph); - if (itr != from_dep_graph_.end()) { - auto ret = itr->second.v_.lock(); - if (ret) { - return ret; - } - // the weakptr expired. mark the key as deleted. - *itr->second.deletion_flag_ = true; - from_dep_graph_.erase(itr); - } - auto ret = std::make_shared(dep_graph, buf_size, - get_cache()); - // insert into graph map - auto graph_iter = from_dep_graph_.insert(std::make_pair( - dep_graph, - flaged_cached_const_graph_tensor_t{ret, ret->deletion_flag_})); - assert(graph_iter.second); - // allocate a unique ID for the tensor - auto hash = dep_graph->hash_contents(); - bool found = false; - for (int retries = 0; retries < 3000; retries++) { - auto id_iter = from_tensor_id_.find(hash); - if (id_iter == from_tensor_id_.end()) { - found = true; - break; - } - hash++; - } - COMPILE_ASSERT(found, "Cannot insert unique cached tensor id"); - auto id_iter = from_tensor_id_.insert(std::make_pair(hash, ret)); - ret->graph_iter_ = graph_iter.first; - ret->id_iter_ = id_iter.first; - return ret; - } - - void remove(cached_const_graph_tensor &v) { - std::lock_guard guard{lock_}; - /* need to check if the key in the map is already deleted. Consider the - * following case if we don't use the deletion_flag_. - * 1. Thread 1: The last reference to a cached_const_graph_tensor is - * destroyed, calling the destructor of cached_const_graph_tensor - * 2. Thread 2: Before thread 1 enters - * const_graph_tensor_cache::remove(), thread2 enters add_tensor. The - * graph_key is the same as but a different instance of the graph_key of - * thread1's. Thread2 will find that the graph is already in - * from_dep_graph_ as the key, but the value (weakptr to - * cached_const_graph_tensor) already expired. So thread2 cannot reuse - * the key-value already in from_dep_graph_. It removes the key from - * from_dep_graph_ and adds a new key to it. - * 3. Thread 1 enters const_graph_tensor_cache::remove() and calls - * from_dep_graph_.erase(v.graph_iter_) - however, the iterator is - * already invalidated because in step 2, thread2 removed it. - * - */ - bool already_deleted = *v.deletion_flag_; - if (already_deleted) { - return; - } - if (v.graph_iter_ != from_dep_graph_.end()) - from_dep_graph_.erase(v.graph_iter_); - if (v.id_iter_ != from_tensor_id_.end()) - from_tensor_id_.erase(v.id_iter_); - } -}; - -static std::shared_ptr get_cache() { - static std::shared_ptr c = - std::make_shared(); - return c; -} - -cached_const_graph_tensor::~cached_const_graph_tensor() { - // if dependency_, the cached tensor should be created by unit tests - if (dependency_) - cache_owner_->remove(*this); -} - -static std::atomic internal_tensor_id = {0xfffff000}; - -SC_INTERNAL_API void graph_constant_input_folding_impl(sc_graph_t &mgr, - const context_ptr &ctx, - bool share_constants) { - if (ctx->engine_->vtable_->get_tensor_cache_cap(ctx->engine_) == 0) { - // if constant cache is off, remove all constant attr - for (auto &op : mgr.ops_) { - if (op->attrs_.get_or_else("constant", const_kind::not_const) != - const_kind::not_const) { - op->attrs_["constant"] = const_kind::not_const; - } - } - return; - } - op_visitor_t vis = op_visitor_t::dfs_topology_sort(mgr.ops_.size()); - std::vector edge_ops; - vis.visit_graph(mgr, [&edge_ops](op_visitor_t *vis, const sc_op_ptr &node) { - if (node->isa()) { - if (!node->attrs_.get_or_null("temp.tensor_id")) { - size_t tsr_id = internal_tensor_id++; - node->attrs_["temp.tensor_id"] = tsr_id; - } - } - auto cur_const_state = - node->attrs_.get_or_else("constant", const_kind::not_const); - if (node->isa() || cur_const_state) { - if (node->isa()) { - edge_ops.emplace_back(node.get()); - node->attrs_.set("constant", const_kind::local_const); - } - bool all_constant_inputs = true; - for (const auto &input : node->get_inputs()) { - auto parent_node = input->producer_owner_; - if (parent_node->attrs_.get_or_else( - "constant", const_kind::not_const) == const_kind::not_const && - !parent_node->isa()) { - all_constant_inputs = false; - break; - } - } - if (!all_constant_inputs) { - for (const auto &input : node->get_inputs()) { - auto parent_node = input->producer_owner_; - if (parent_node->attrs_.get_or_else( - "constant", const_kind::not_const) != const_kind::not_const) { - parent_node->attrs_.set("constant", const_kind::global_const); - edge_ops.emplace_back(parent_node); - if (parent_node->isa()) { - auto tv_in = parent_node; - while (tv_in->isa()) { - tv_in = tv_in->get_inputs()[0]->producer_owner_; - } - tv_in->attrs_.set("constant", const_kind::global_const); - edge_ops.emplace_back(tv_in); - } - } - } - node->attrs_.set("constant", const_kind::not_const); - } else { - if (!node->isa()) { - // Setting attrs here is intermediary status. - // Current node is constant node, its uses may - // also be constant, so we set `local_const` - // here temporarily meaning `may_constant`. - // Later when visiting its uses, we check their - // all inputs and decide whether we reserve this - // attr. - for (auto &out : node->get_outputs()) { - for (auto &cld_node : out->uses_) { - cld_node.second->attrs_.set("constant", const_kind::local_const); - } - } - } - } - } - }); - if (share_constants && !edge_ops.empty() && ctx->flags_.const_share_) { - op_dep_matrix_t dependency{mgr}; - std::vector hash_cache(mgr.ops_.size()); - std::vector existing_data_vec; - // the list of cached tensors in the whole graph - std::vector> caches; - for (auto op : edge_ops) { - if (op->attrs_.has_key(op_attr_key::const_input_cache)) { - continue; - } - if (op->isa()) { - continue; - } - // the list of cached tensors in this op - std::vector> results; - // the op-ids of inputs and constants that current op depends on - std::list depending_inputs; - // if all of the dependent ops are tensor_view, skip - bool is_all_tensor_view = true; - std::vector op_mask(mgr.ops_.size()); - for (size_t i = 0; i < mgr.ops_.size(); i++) { - if (i != (size_t)op->logical_op_id_ && - dependency.lookup(i, op->logical_op_id_) == 1) { - // if is input/constant - if (mgr.ops_[i]->get_inputs().empty()) { - depending_inputs.emplace_back(mgr.ops_[i]); - } - op_mask[i] = true; - if (!mgr.ops_[i]->isa() && - !mgr.ops_[i]->isa() && - !mgr.ops_[i]->isa()) { - is_all_tensor_view = false; - } - } - } - if (is_all_tensor_view && op->isa()) { - continue; - } - // the selector for graph visitor to visit sub graph for the - // current op. Try to normalize the visiting order by hash of - // the ops - auto selector = [&op_mask, &hash_cache](op_visitor_t *v) -> sc_op_ptr { - std::list::iterator theitr; - size_t max_hash = 0; - bool found = false; - for (auto itr = v->to_visit_.begin(); itr != v->to_visit_.end();) { - auto &cur = *itr; - // if the cur is not depended by op, skip - if (v->has_visited(cur->logical_op_id_) || - !op_mask[cur->logical_op_id_]) { - itr = v->to_visit_.erase(itr); - continue; - } - auto &cached_hash = hash_cache[cur->logical_op_id_]; - size_t cur_hash = 0; - if (!cached_hash) { - cur_hash = cur->hash_contents(); - cached_hash = cur_hash; - } else { - cur_hash = cached_hash; - } - // find the op with max hash value - if (cur_hash >= max_hash) { - max_hash = cur_hash; - found = true; - theitr = itr; - } - ++itr; - } - if (found) { - auto ret = *theitr; - v->to_visit_.erase(theitr); - return ret; - } - return nullptr; - }; - op_visitor_t visitor{ - selector, op_visitor_t::create_DAG_updater(mgr.ops_.size()), false}; - std::vector depending_ops; - visitor.to_visit_ = std::move(depending_inputs); - visitor.visit([&depending_ops](op_visitor_t *v, const sc_op_ptr &p) { - depending_ops.emplace_back(p->logical_op_id_); - }); - depending_ops.emplace_back(op->logical_op_id_); - for (size_t i = 0; i < op->get_outputs().size(); i++) { - void *existing_data = nullptr; - if (auto c_op = op->dyn_cast()) { - auto &src = *(c_op->get_constant_values()); - existing_data = src.data_; - } - existing_data_vec.emplace_back(existing_data); - auto buf_size = op->get_outputs()[i]->details_.get_blocking_byte_size(); - if (buf_size <= 8UL) { - // skip scalars, don't register it in the cache manager - auto cached_tsr = std::make_shared( - nullptr, buf_size, get_cache()); - // mark as deleted from the cache manager - *cached_tsr->deletion_flag_ = true; - results.emplace_back(cached_tsr); - caches.emplace_back(cached_tsr); - continue; - } - // for each output tensor of the edge op, rebuild the - // dependency graph - auto g = std::make_shared(); - std::vector newops; - newops.resize(mgr.ops_.size()); - for (auto &depop_id : depending_ops) { - const auto &theop = mgr.ops_[depop_id]; - if (theop->isa()) { - newops[depop_id] = g->make_input( - copy_logical_tsr(theop->get_outputs()), theop->attrs_); - newops[depop_id]->attrs_["constant_input_id"] = - theop->attrs_["temp.tensor_id"]; - } else { - auto copyable = theop->dyn_cast(); - COMPILE_ASSERT(copyable, "The const cache ops must be copyable"); - std::vector newins; - auto &oldins = theop->get_inputs(); - for (size_t in_idx = 0; in_idx < oldins.size(); in_idx++) { - auto oldowner = oldins[in_idx]->producer_owner_; - auto &mapped_op = newops[oldowner->logical_op_id_]; - auto idx = - std::find(oldowner->get_outputs().begin(), - oldowner->get_outputs().end(), oldins[in_idx]) - - oldowner->get_outputs().begin(); - newins.emplace_back(mapped_op->get_outputs()[idx]); - } - newops[depop_id] = copyable->copy( - newins, copy_logical_tsr(theop->get_outputs()), *g); - } - } - auto &lastop = newops[op->logical_op_id_]; - g->make_output({lastop->get_outputs()[i]}); - results.emplace_back(get_cache()->add_tensor( - g, op->get_outputs()[i]->details_.get_blocking_byte_size(), - ctx->engine_)); - caches.emplace_back(results.back()); - if (auto sc_stream_temp = - ::dnnl::impl::graph::gc::runtime::get_info_logging_stream( - __sc_module_name)) { - (*sc_stream_temp.stream_) - << "Putting into shared cache: " << results.back() - << ", uid=" << (void *)results.back()->id_iter_->first - << ", size=" << results.back()->size_ << ", dep =\n"; - print_graph(*g, *sc_stream_temp.stream_, true, true); - } - } - // mark the op with const_input_cache attr. We need to skip the - // tensorview ops, because we don't generate tensor buffer for these - // ops - auto cached_op = op; - while (cached_op->isa()) { - cached_op = cached_op->get_inputs()[0]->producer_owner_; - } - cached_op->attrs_[op_attr_key::const_input_cache] = std::move(results); - } - get_cache()->alloca_.alloc(caches, existing_data_vec, ctx->engine_); - } -} - -SC_INTERNAL_API void graph_constant_input_folding(sc_graph_t &mgr, - const context_ptr &ctx) { - graph_constant_input_folding_impl(mgr, ctx, false); -} -SC_INTERNAL_API void -graph_constant_input_folding_and_share_constants(sc_graph_t &mgr, - const context_ptr &ctx) { - graph_constant_input_folding_impl(mgr, ctx, true); -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/pass/graph_copy.cpp b/legacy/core/src/compiler/ir/graph/pass/graph_copy.cpp deleted file mode 100644 index 92aacdf56..000000000 --- a/legacy/core/src/compiler/ir/graph/pass/graph_copy.cpp +++ /dev/null @@ -1,98 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "../dynamic_lower_info.hpp" -#include "../fusible_op.hpp" -#include "../traits.hpp" -#include "../visitor.hpp" - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -using namespace op_traits; -SC_INTERNAL_API sc_graph_t copy_graph(const sc_graph_t &graph) { - for (auto &op : graph.ops_) { - if (!op->is_removed_ && !op->dyn_cast() && - !op->dyn_cast() && !op->dyn_cast()) { - return sc_graph_t(); - } - } - sc_graph_t copied_graph; - op_visitor_t vis = op_visitor_t::bfs_topology_sort(graph.ops_.size()); - std::unordered_map old_new_lt_map; - std::unordered_map op_id_map; - // the map from old op id to new op - std::vector old_id_2_new_op; - old_id_2_new_op.resize(graph.ops_.size()); - vis.visit_graph(graph, [&](op_visitor_t *vis, const sc_op_ptr &node) { - sc_op_ptr new_node; - if (node->dyn_cast()) { - new_node = copied_graph.make_input(copy_logical_tsr(node->get_outputs())); - - // "unique_id" for integration - new_node->attrs_ = node->attrs_; - } else { - std::vector ins; - ins.reserve(node->get_inputs().size()); - for (auto &t : node->get_inputs()) { - ins.emplace_back(old_new_lt_map.at(t)); - } - if (node->dyn_cast()) { - new_node = copied_graph.make_output(ins); - // "unique_id" for integration - new_node->attrs_ = node->attrs_; - } else { - new_node = node->dyn_cast()->copy( - ins, copy_logical_tsr(node->get_outputs()), copied_graph); - new_node->info_.cur_impl_ = node->info_.cur_impl_; - } - } - // recording old graph_tensor->new graph_tensor - for (size_t i = 0; i < new_node->get_outputs().size(); ++i) { - old_new_lt_map[node->get_outputs()[i]] = new_node->get_outputs()[i]; - } - op_id_map[new_node] = node->logical_op_id_; - old_id_2_new_op[node->logical_op_id_] = new_node; - }); - // update the uses order, it is important for checking equality of the - // copied graph - for (auto &newop : copied_graph.ops_) { - auto &mapped_old = graph.ops_[op_id_map[newop]]; - auto &newouts = newop->get_outputs(); - auto &oldouts = mapped_old->get_outputs(); - for (size_t i = 0; i < newouts.size(); i++) { - // copy the old uses, and re-map to new ops - newouts[i]->uses_ = oldouts.at(i)->uses_; - for (auto &use : newouts[i]->uses_) { - auto old_id = use.second.lock()->logical_op_id_; - use.second = old_id_2_new_op[old_id]; - } - } - } - copied_graph.attrs_ = graph.attrs_; - // deep copy here. - if (graph.dyn_info_) { - copied_graph.dyn_info_ = - std::make_shared(*graph.dyn_info_); - } - copied_graph.resort_op_ids(op_id_map); - return copied_graph; -} -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/pass/intrusive_opt_level.cpp b/legacy/core/src/compiler/ir/graph/pass/intrusive_opt_level.cpp deleted file mode 100644 index fc26b710b..000000000 --- a/legacy/core/src/compiler/ir/graph/pass/intrusive_opt_level.cpp +++ /dev/null @@ -1,40 +0,0 @@ -/******************************************************************************* - * Copyright 2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "pass.hpp" -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -SC_INTERNAL_API void intrusive_opt_level(sc_graph_t &graph, - const context_ptr &ctx) { - for (auto &op : graph.ops_) { - if (ctx->flags_.opt_level_ < sc_opt_level::lv3) { - if (op->isa()) { - op->attrs_.set("image_affinity", false); - } - } - } -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/pass/padded_mask_mark.cpp b/legacy/core/src/compiler/ir/graph/pass/padded_mask_mark.cpp deleted file mode 100644 index 57295b826..000000000 --- a/legacy/core/src/compiler/ir/graph/pass/padded_mask_mark.cpp +++ /dev/null @@ -1,118 +0,0 @@ -/******************************************************************************* - * Copyright 2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "pass.hpp" - -#include -#include -#include -#include -#include -#include -#include -SC_MODULE(graph.pass.const_input_fold); - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -enum mask_etype : int { - no_need_mask = 0, - not_nan, // not produce nan/inf(div) - has_nan -}; -const std::unordered_set ops_need_mask = {"exp", "select", "add", - "sub", "div", "max"}; -static int need_op_mask(const sc_op_ptr &op) { - int pre_mask_type = - op->attrs_.get_or_else("temp.mask_type", (int)mask_etype::no_need_mask); - if (ops_need_mask.find(op->op_name_) != ops_need_mask.end()) { - // div may produce nan/inf, and nan/inf * 0 is nan - if (op->op_name_ == "div") { - return std::max(pre_mask_type, (int)mask_etype::has_nan); - } - // f(0, 0) = 0 - if (utils::is_one_of(op->op_name_, std::string("add"), std::string("sub"), - std::string("max"))) { - if (op->get_inputs()[0]->details_.get_plain_dims() != - op->get_inputs()[1]->details_.get_plain_dims()) { - return std::max(pre_mask_type, (int)mask_etype::not_nan); - } - return std::max(pre_mask_type, (int)mask_etype::no_need_mask); - } - if (op->op_name_ == "select") { - if (op->get_inputs()[1]->details_.get_plain_dims() != - op->get_inputs()[2]->details_.get_plain_dims()) { - return std::max(pre_mask_type, (int)mask_etype::not_nan); - } - return std::max(pre_mask_type, (int)mask_etype::no_need_mask); - } - - return std::max(pre_mask_type, (int)mask_etype::not_nan); - } - return pre_mask_type; -} -static void mark_mask_attr(const sc_op_ptr &op) { - if (!op->isa() && - !op->isa() && !op->isa()) { - return; - } - int mask_type = need_op_mask(op); - auto &uses = op->get_outputs()[0]->uses_; - if (op->get_outputs()[0]->details_.get_format().is_blocking() && mask_type) { - for (auto &use : uses) { - if (use.second->isa() || - use.second->isa()) { - op->attrs_.set(op_attr_key::use_padded_mask, true); - return; - } - if (use.second->isa() && - mask_type == (int)mask_etype::has_nan) { - op->attrs_.set(op_attr_key::use_padded_mask, true); - return; - } - if (use.second->isa()) { - if (use.second->isa() && - use.second->get_inputs()[0]->details_.get_format().is_blocking() && - use.second->get_outputs()[0]->details_.get_format().is_plain()) { - continue; - } - if (use.second->isa()) { - continue; - } - op->attrs_.set(op_attr_key::use_padded_mask, true); - return; - } - } - // pass down to its uses. - for (auto &use : uses) { - use.second->attrs_.set("temp.mask_type", mask_type); - } - } - op->attrs_.set(op_attr_key::use_padded_mask, false); -} - -SC_INTERNAL_API void padded_mask_mark(sc_graph_t &graph, - const context_ptr &ctx) { - op_visitor_t vis = op_visitor_t::dfs_topology_sort(graph.ops_.size()); - vis.visit_graph(graph, [](op_visitor_t *vis, const sc_op_ptr &node) { - mark_mask_attr(node); - }); -} -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/pass/pass.hpp b/legacy/core/src/compiler/ir/graph/pass/pass.hpp deleted file mode 100644 index 4438bbccd..000000000 --- a/legacy/core/src/compiler/ir/graph/pass/pass.hpp +++ /dev/null @@ -1,152 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_PASS_PASS_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_PASS_PASS_HPP - -#include "../graph.hpp" -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -// Used in `graph_constant_input_folding`, stores as an attribute -// "constant" in nodes which describe current node's outputs status. -namespace const_kind { -// means non constant node -constexpr int not_const = 0; -// means intermediate constant node, with constant inputs and its -// outputs will be used to calculate local/global constant node in init stage. -constexpr int local_const = 1; -// means final constant node, with constant inputs and its outputs will be -// lowerred to global variables and be used as inputs in non-const nodes. -constexpr int global_const = 2; -} // namespace const_kind - -SC_INTERNAL_API void print_graph(const sc_graph_t &mgr, std::ostream &os, - bool print_shape = false, - bool print_attr = false, - bool print_name = false, - bool print_stride = false); - -SC_INTERNAL_API sc_graph_t copy_graph(const sc_graph_t &graph); - -SC_API bool check_graph_connection(sc_graph_t &graph); -bool check_graph_config(sc_graph_t &graph, - const context_ptr &ctx = get_default_context()); -/** - * Folding nodes with constant attribute from inputs in graph. - * In this pass, we only propagate "constant" states from constant op and input - * op which marked `constant`. Actual folding will occur in lowering. - * */ -SC_INTERNAL_API void -graph_constant_input_folding(sc_graph_t &graph, - const context_ptr &ctx = get_default_context()); -/** - * Do the same as graph_constant_input_folding, except that it also try to share - * the constant buffer with other graphs. This pass should be put after all - * other graph_constant_input_folding - * */ -SC_INTERNAL_API void -graph_constant_input_folding_and_share_constants(sc_graph_t &mgr, - const context_ptr &ctx); -/** - * Mark the elementwise op with padded input/output could use output mask(not - * mask load/store) or not. The op could use mask when its direct uses have - * reduce or memory movement semantics. - */ -SC_INTERNAL_API void padded_mask_mark(sc_graph_t &graph, - const context_ptr &ctx); -/** - * Enable/Disable some latest optimizations like image-affinity according to the - * compiler opt level. - */ -SC_INTERNAL_API void intrusive_opt_level(sc_graph_t &graph, - const context_ptr &ctx); - -// find the graph in cached code. If a matched graph is found, the -// compiler_driver/graph_driver can skip the compilation and reuse the code -SC_INTERNAL_API void graph_code_cache(sc_graph_t &mgr, const context_ptr &ctx); - -/** - * Compares the graphs. - * @param lhs the left hand side graph - * @param rhs the right hand side graph - * @param first_diff_lhs outputs the first different LHS Op - * @param first_diff_rhs outputs the first different RHS Op - * @param lhs_rhs_input_mapping {the left graph input op id, the right graph - * input op id} mapping - * @param filter the filter function for op attr, @see - * sc_graph_t::compare_contents - * @return true if the the graphs are the same - * */ -SC_INTERNAL_API bool compare_graph( - sc_op_ptr &first_diff_lhs, sc_op_ptr &first_diff_rhs, const sc_graph_t &lhs, - const sc_graph_t &rhs, - const std::unordered_map &lhs_rhs_input_mapping = {}, - const std::function &filter = - nullptr); - -/** - * Compares the graphs. - * @param lhs the left hand side graph - * @param rhs the right hand side graph - * @param lhs_rhs_input_mapping {the left graph input op id, the right graph - * input op id} mapping - * @param filter the filter function for op attr, @see - * sc_graph_t::compare_contents - * @return true if the the graphs are the same - * */ -SC_INTERNAL_API bool compare_graph( - const sc_graph_t &lhs, const sc_graph_t &rhs, - const std::unordered_map &lhs_rhs_input_mapping = {}, - const std::function &filter = - nullptr); - -namespace runtime { -struct dynamic_tensor_t; -} -/** - * The api for dynamic infer shape. - * At runtime, given input real shapes of partition, infer the output shapes. - * After calling this api, the sizes of output buffers are known for allocation. - * @param graph cached raw graph after translation(after graph inline and - * constant optimization) but not graph driver. We need original graph cache to - * preserve all semantics of ops - * @param ins a pointer to dynamic tensor vector of inputs, contains real shape - * info. - * @param outs a pointer to dynamic tensor vector of outputs, the real shape - * need to be infered. - * @param num_ins, the length of input dynamic tensor vector, should be equal to - * number of inputs in graph. - * @param num_outs, the length of output dynamic tensor vector, should be equal - * to number of outputs in graph. - * */ -SC_API void dynamic_infer_shape_by_graph(sc_graph_t &graph, - runtime::dynamic_tensor_t **ins, - runtime::dynamic_tensor_t **outs, - size_t num_ins, size_t num_outs); - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/graph/pass/printer.cpp b/legacy/core/src/compiler/ir/graph/pass/printer.cpp deleted file mode 100644 index 91b23ce08..000000000 --- a/legacy/core/src/compiler/ir/graph/pass/printer.cpp +++ /dev/null @@ -1,138 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "../fusible_op.hpp" -#include "../graph_op.hpp" -#include "../visitor.hpp" -#include "pass.hpp" -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -SC_INTERNAL_API void print_graph(const sc_graph_t &mgr, std::ostream &os, - bool print_shape, bool print_attr, - bool print_name, bool print_stride) { - std::unordered_map tsr_idx; - auto get_tensor_id = [&](const graph_tensor_ptr &t) { - auto itr = tsr_idx.find(t); - if (itr != tsr_idx.end()) { - return itr->second; - } - int ret = tsr_idx.size(); - tsr_idx[t] = ret; - return ret; - }; - auto print_tensor_name = [&](const graph_tensor_ptr &t) { - if (print_name) { - sc_op *linked_output = nullptr; - for (auto &use : t->uses_) { - if (use.second->isa()) { - linked_output = use.second.get(); - break; - } - } - auto name = graph::get_tensor_name(t.get(), linked_output); - if (!name.empty()) { - os << name; - return; - } - } - os << 'v' << get_tensor_id(t); - }; - auto print_tensor_list = [&](const std::vector &list, - bool p_shape, bool p_stride) { - bool is_first_input = true; - for (const auto &tsr : list) { - if (!is_first_input) { - os << ", "; - } else { - is_first_input = false; - } - print_tensor_name(tsr); - if (p_shape) { - os << ": " << tsr->details_.dtype_ << '['; - bool is_first_shape = true; - for (auto dim : tsr->details_.get_blocking_dims()) { - if (!is_first_shape) { - os << ", "; - } else { - is_first_shape = false; - } - os << dim; - } - os << ']'; - } - if (p_stride) { - os << ", " << (tsr->details_.is_dense() ? "dense:" : "strided:"); - for (auto stride : tsr->details_.get_strides()) { - os << "s"; - os << stride; - } - } - } - }; - std::vector inputs; - std::vector outputs; - for (auto &v : mgr.ops_) { - if (v->isa()) { - inputs.insert(inputs.end(), v->get_outputs().begin(), - v->get_outputs().end()); - } else if (v->isa()) { - outputs.insert(outputs.end(), v->get_inputs().begin(), - v->get_inputs().end()); - } - } - std::string graph_name; - if (print_name) { - graph_name = mgr.attrs_.get_or_else("temp.name", "graph"); - } else { - graph_name = "graph"; - } - os << graph_name << '('; - print_tensor_list(inputs, print_shape, print_stride); - os << ") -> ["; - print_tensor_list(outputs, print_shape, print_stride); - os << "] {\n"; - - op_visitor_t::dfs_topology_sort().visit_graph( - mgr, [&](op_visitor_t *vis, const sc_op_ptr &node) { - if (!node->isa() && !node->isa()) { - print_indents(os, 1); - - os << '['; - print_tensor_list(node->get_outputs(), print_shape, print_stride); - os << "] = " << node->op_name_; - os << '('; - if (auto con_node = node->dyn_cast()) { - os << utils::print_vector(con_node->get_constant_blocking_dims()); - } else { - print_tensor_list(node->get_inputs(), false, false); - } - os << ')' << '\n'; - } - }); - os << '}' << '\n'; -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/quantization/compensation.cpp b/legacy/core/src/compiler/ir/graph/quantization/compensation.cpp deleted file mode 100644 index fcf99aeb2..000000000 --- a/legacy/core/src/compiler/ir/graph/quantization/compensation.cpp +++ /dev/null @@ -1,43 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "quantize_info.hpp" -#include -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -static std::vector -get_padded_union_val_vector(const std::vector &input, unsigned padded) { - if (input.empty() || padded <= 0) { - return std::vector(); - } - std::vector results(padded); - for (unsigned i = 0; i < padded; i++) { - if (i < input.size()) { - results[i] = (int64_t)input[i]; - } else { - results[i] = (int64_t)0; - } - } - return results; -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/quantization/quantize_info.hpp b/legacy/core/src/compiler/ir/graph/quantization/quantize_info.hpp deleted file mode 100644 index 7989488e4..000000000 --- a/legacy/core/src/compiler/ir/graph/quantization/quantize_info.hpp +++ /dev/null @@ -1,96 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_QUANTIZATION_QUANTIZE_INFO_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_QUANTIZATION_QUANTIZE_INFO_HPP - -#include -#include -#include -#include -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace attr_keys { -// static quantize attribute keys, values are concrete numbers. -constexpr const char *scales = "scales"; -constexpr const char *zero_points = "zero_points"; -constexpr const char *data_scales = "data_scales"; -constexpr const char *data_zero_points = "data_zero_points"; -constexpr const char *weight_scales = "weight_scales"; -constexpr const char *weight_zero_points = "weight_zero_points"; -// dynamic quantize attribute keys, values are std::shared_ptr. -// need "temp.". e.g. "temp.data_dyn_scales" -constexpr const char *dyn_scales = "temp.dyn_scales"; -constexpr const char *dyn_zero_points = "temp.dyn_zero_points"; -constexpr const char *dyn_data_scales = "temp.dyn_data_scales"; -constexpr const char *dyn_data_zero_points = "temp.dyn_data_zero_points"; -constexpr const char *dyn_weight_scales = "temp.dyn_weight_scales"; -constexpr const char *dyn_weight_zero_points = "temp.dyn_weight_zero_points"; -// common attributes -constexpr const char *quan_dtype = "dtype"; -constexpr const char *per_channel = "per_channel"; -constexpr const char *channel_axis = "channel_axis"; -constexpr const char *weight_channel_axis = "weight_channel_axis"; -constexpr const char *output_channel_axis = - "output_channel_axis"; // output of tuanble ops -constexpr const char *asymmetric = "asymmetric"; -// pass attributes -constexpr const char *mixed_dtype = "mixed_dtype"; -constexpr const char *may_quantize = "may_quantize"; -} // namespace attr_keys - -namespace quantize { - -enum class tensor_type { input_tensor, weight_tensor, bias_tensor }; - -/** information for int8 quantization, include quantize/re-quantize/de-quantize. - * @param dtype_ decide clip range, u8 [0,255], s8 [-128, 127] - * @param scales_ scales for current quantize op output. - * @param zero_points_ output zero points for asymmetric quantize, no need - * input zero points. - * @param per_channel_ a param for conv op, if true, each channel has a scale - * and zero point. - * @param channel_axis_ an option param available when per_channel_ is true, - * points which axis is channel. - * @param asymmetric_ whether use symmetric or asymmetric quantization.(not used - * yet) - * */ - -struct quantize_infos_t { - sc_data_type_t dtype_ = sc_data_type_t::u8(1); - std::vector scales_ = {1.f}; - std::vector zero_points_ = {0}; - bool per_channel_ = false; - int channel_axis_ = 0; - bool asymmetric_ = false; - quantize_infos_t() = default; - quantize_infos_t(sc_data_type_t dtype, const std::vector &scales, - const std::vector &zero_points, - bool per_channel = false, int channel_axis = 0, - bool asymmetric = false) - : dtype_(dtype), scales_(scales), zero_points_(zero_points), - per_channel_(per_channel), channel_axis_(channel_axis), - asymmetric_(asymmetric) {} -}; -quantize_infos_t get_quantize_info_from_attrs(const any_map_t &attrs); - -} // namespace quantize -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl -#endif diff --git a/legacy/core/src/compiler/ir/graph/quantization/quantize_op.cpp b/legacy/core/src/compiler/ir/graph/quantization/quantize_op.cpp deleted file mode 100644 index e8ff1337b..000000000 --- a/legacy/core/src/compiler/ir/graph/quantization/quantize_op.cpp +++ /dev/null @@ -1,429 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "quantize_op.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace quantize { - -static void common_query_function( - sc_op *node, const sc_data_format_t &out_format, - std::vector> &supported_ins, - std::vector> &supported_outs) { - std::vector> in_formats, out_formats; - out_formats.push_back({out_format}); - node->format_to_dense_format_stride_pair(in_formats, out_formats, - supported_ins, supported_outs); -} - -quantize_infos_t get_quantize_info_from_attrs(const any_map_t &attrs) { - quantize_infos_t infos; - infos.dtype_ = - attrs.get_or_else(attr_keys::quan_dtype, sc_data_type_t::u8(1)); - infos.scales_ = attrs.get_or_else(attr_keys::scales, std::vector{1.f}); - infos.zero_points_ = - attrs.get_or_else(attr_keys::zero_points, std::vector{0}); - infos.per_channel_ = attrs.get_or_else(attr_keys::per_channel, false) || - infos.scales_.size() > 1; - infos.channel_axis_ = attrs.get_or_else(attr_keys::channel_axis, 0); - infos.asymmetric_ = attrs.get_or_else(attr_keys::asymmetric, true); - assert(utils::is_one_of(infos.dtype_, datatypes::f32, datatypes::bf16, - datatypes::u8, datatypes::s8) && - ((infos.per_channel_ && !infos.scales_.empty()) || - (!infos.per_channel_ && infos.scales_.size() == 1 && - infos.zero_points_.size() <= 1)) && - (infos.asymmetric_ || - (!infos.asymmetric_ && - (infos.zero_points_.empty() || - (infos.zero_points_.size() == 1 && infos.zero_points_[0] == 0))))); - return infos; -} - -static void -check_dynamic_quantization_schema(const quantize_infos_t &qinfos, - const std::vector &inputs) { - const auto &input = inputs[0]; - const auto &scales = inputs[1]; - const auto &zps = inputs.size() == 3 ? inputs[2] : nullptr; - COMPILE_ASSERT(scales->details_.dtype_.type_code_ == sc_data_etype::F32, - "dynamic quantization's scales shall be f32 dtype."); - if (zps) { - COMPILE_ASSERT( - utils::is_one_of(zps->details_.dtype_.type_code_, sc_data_etype::U8, - sc_data_etype::S8, sc_data_etype::S32), - "dynamic quantization's zps shall be one of u8/s8/s32 dtype."); - } - if (qinfos.per_channel_) { - const auto &channel_size = - input->details_.get_plain_dims()[qinfos.channel_axis_]; - COMPILE_ASSERT( - gc::graph::check_shape_equal(scales->details_.get_plain_dims(), - sc_dims{channel_size}), - "dynamic quantization op's scale shall confirm with channel " - "size in per_channel case."); - if (zps) { - COMPILE_ASSERT(gc::graph::check_shape_equal( - zps->details_.get_plain_dims(), sc_dims{channel_size}), - "dynamic quantization op's zps size shall confirm with " - "channel size in per_channel case."); - } - } else { - COMPILE_ASSERT(gc::graph::check_shape_equal( - scales->details_.get_plain_dims(), sc_dims{1}), - "dynamic quantization op's scale shall be {1} in " - "per_tensor case."); - if (zps) { - COMPILE_ASSERT(gc::graph::check_shape_equal( - zps->details_.get_plain_dims(), sc_dims{1}), - "dynamic quantization op's zps shall be {1} in " - "per_tensor case."); - } - } -} - -quantize_op_t::quantize_op_t(const std::vector &ins, - const std::vector &outs, - const any_map_t &attrs) { - assert(ins.size() == 1); - assert(ins[0]->details_.dtype_.type_code_ == sc_data_etype::F32 || - ins[0]->details_.dtype_.type_code_ == sc_data_etype::S32); - info_.inputs_ = ins; - if (outs.empty()) { - // fixme: correctly infer the shape for broadcast - info_.outputs_.emplace_back(std::make_shared(this)); - info_.outputs_[0]->details_ = ins[0]->details_; - assert(attrs.has_key(attr_keys::quan_dtype)); - info_.outputs_[0]->details_.dtype_ = - attrs.get(attr_keys::quan_dtype); - } else { - info_.outputs_ = outs; - } - attrs_ = attrs; - op_name_ = "quantize"; -} - -quantize_op_t::quantize_op_t(const std::vector &ins, - const any_map_t &attrs) - : quantize_op_t(ins, std::vector(), attrs) {} - -void quantize_op_t::get_graph_impl(std::shared_ptr &graph) { - // create new input logical tensors - std::vector inputs, outputs; - inputs = remake_logical_tensors(info_.inputs_); - outputs = remake_logical_tensors(info_.outputs_); - const auto qinfos = get_quantize_info_from_attrs(attrs_); - assert(utils::is_one_of(qinfos.dtype_, datatypes::u8, datatypes::s8)); - auto scales = qinfos.scales_; - scales = math_utils::vector_rcp(scales); - std::shared_ptr scales_ptr = - std::make_shared(scales); - - sc_dims plain_dims = {1}; - if (scales.size() > 1) { - plain_dims.resize(inputs[0]->details_.get_plain_dims().size(), 1); - plain_dims[qinfos.channel_axis_] = static_cast(scales.size()); - } - auto quantize_const_scales = graph->make("constant", {}, {}, - {{"values", scales_ptr}, - {"dtype", datatypes::f32}, - {"plain_dims", plain_dims}, - {"format", sc_data_format_t()}, - {"all_positive", true}}); - auto div_scale = graph->make( - "mul", {inputs[0], quantize_const_scales->get_outputs()[0]}, {}, {}); - - auto zeropoints = qinfos.zero_points_; - if (!zeropoints.empty()) { - int zp_all_zero = std::all_of(zeropoints.begin(), zeropoints.end(), - [](int i) { return i == 0; }); - if (!zp_all_zero) { - std::vector zeropoints_f32(zeropoints.begin(), zeropoints.end()); - std::shared_ptr zeropoints_ptr = - std::make_shared(zeropoints_f32); - auto quantize_const_zeropoints = graph->make( - "constant", {}, {}, - {{"values", zeropoints_ptr}, - {"dtype", datatypes::f32}, - {"plain_dims", sc_dims{static_cast(zeropoints_f32.size())}}, - {"format", sc_data_format_t()}}); - - div_scale = graph->make("add", - {div_scale->get_outputs()[0], - quantize_const_zeropoints->get_outputs()[0]}, - {}, {}); - } - } - // maybe we need clip op in future -#if 0 - auto clip = graph->make("clip", sub_zp->get_outputs(), {}, - {{"clip_min", - qinfos.dtype_.is_etype(sc_data_etype::U8) - ? 0.f - : -128.f}, - {"clip_max", - qinfos.dtype_.is_etype( - sc_data_etype::U8) - ? 255.f - : 127.f}}); -#endif - auto int8_cast = graph->make("cast", div_scale->get_outputs(), {}, - {{"dtype", qinfos.dtype_}, {"saturated", true}}); - graph->make_output(int8_cast->get_outputs()); -} - -void quantize_op_t::query_format( - context_ptr ctx, - std::vector> &supported_ins, - std::vector> &supported_outs) { - common_query_function(this, info_.inputs_[0]->details_.get_format(), - supported_ins, supported_outs); -} - -dequantize_op_t::dequantize_op_t(const std::vector &ins, - const std::vector &outs, - const any_map_t &attrs) { - assert(ins.size() == 1); - assert(ins[0]->details_.dtype_.type_code_ == sc_data_etype::U8 || - ins[0]->details_.dtype_.type_code_ == sc_data_etype::S8 || - ins[0]->details_.dtype_.type_code_ == sc_data_etype::S32 || - ins[0]->details_.dtype_.type_code_ == sc_data_etype::F32); - info_.inputs_ = ins; - if (outs.empty()) { - // fixme: correctly infer the shape for broadcast - info_.outputs_.emplace_back(std::make_shared(this)); - info_.outputs_[0]->details_ = ins[0]->details_; - info_.outputs_[0]->details_.dtype_.type_code_ = sc_data_etype::F32; - } else { - info_.outputs_ = outs; - } - attrs_ = attrs; - op_name_ = "dequantize"; -} - -dequantize_op_t::dequantize_op_t(const std::vector &ins, - const any_map_t &attrs) - : dequantize_op_t(ins, std::vector(), attrs) {} - -void dequantize_op_t::get_graph_impl(std::shared_ptr &graph) { - // create new input logical tensors - std::vector inputs, outputs; - inputs = remake_logical_tensors(info_.inputs_); - outputs = remake_logical_tensors(info_.outputs_); - auto qinfos = get_quantize_info_from_attrs(attrs_); - qinfos.dtype_ = datatypes::f32; - std::vector scales = qinfos.scales_; - std::shared_ptr scales_ptr = - std::make_shared(scales); - sc_dims scales_plain_dims = {1}; - if (scales.size() > 1) { - scales_plain_dims.resize(inputs[0]->details_.get_plain_dims().size(), 1); - scales_plain_dims[qinfos.channel_axis_] = static_cast(scales.size()); - } - auto ins = graph->make_input(inputs); - auto const_scales = graph->make("constant", {}, {}, - {{"values", scales_ptr}, - {"dtype", datatypes::f32}, - {"plain_dims", scales_plain_dims}, - {"format", sc_data_format_t()}, - {"all_positive", true}}); - auto f32_cast = ins; - if (inputs[0]->details_.dtype_.type_code_ != sc_data_etype::F32) { - f32_cast = - graph->make("cast", ins->get_outputs(), {}, {{"dtype", qinfos.dtype_}}); - } - - bool all_zero = - std::all_of(qinfos.zero_points_.begin(), qinfos.zero_points_.end(), - [](int x) { return x == 0; }); - if (!all_zero) { - std::vector zero_points(qinfos.zero_points_.begin(), - qinfos.zero_points_.end()); - auto const_zero_points = graph->make( - "constant", {}, {}, - {{"values", std::make_shared(zero_points)}, - {"dtype", datatypes::f32}, - {"plain_dims", sc_dims{static_cast(zero_points.size())}}, - {"format", sc_data_format_t()}}); - f32_cast = graph->make( - "sub", - {f32_cast->get_outputs()[0], const_zero_points->get_outputs()[0]}, {}, - {}); - } - auto mul_scale = graph->make( - "mul", {f32_cast->get_outputs()[0], const_scales->get_outputs()[0]}, {}, - {}); - graph->make_output(mul_scale->get_outputs()); -} - -void dequantize_op_t::query_format( - context_ptr ctx, - std::vector> &supported_ins, - std::vector> &supported_outs) { - common_query_function(this, info_.inputs_[0]->details_.get_format(), - supported_ins, supported_outs); -} - -dynamic_quantize_op_t::dynamic_quantize_op_t( - const std::vector &ins, - const std::vector &outs, const any_map_t &attrs) { - COMPILE_ASSERT(ins.size() == 2 || ins.size() == 3, - "dynamic quantize op shall have 2 or 3 inputs."); - assert(ins[0]->details_.dtype_.type_code_ == sc_data_etype::F32 || - ins[0]->details_.dtype_.type_code_ == sc_data_etype::S32); - info_.inputs_ = ins; - const auto qinfos = get_quantize_info_from_attrs(attrs); - check_dynamic_quantization_schema(qinfos, info_.inputs_); - if (outs.empty()) { - info_.outputs_.emplace_back(std::make_shared(this)); - info_.outputs_[0]->details_ = ins[0]->details_; - assert(attrs.has_key(attr_keys::quan_dtype)); - info_.outputs_[0]->details_.dtype_ = - attrs.get(attr_keys::quan_dtype); - } else { - info_.outputs_ = outs; - COMPILE_ASSERT(gc::graph::check_shape_equal( - info_.inputs_[0]->details_.get_plain_dims(), - info_.outputs_[0]->details_.get_plain_dims()), - "dynamic quantize's output shall have the same shape as " - "input."); - COMPILE_ASSERT( - utils::is_one_of(info_.outputs_[0]->details_.dtype_.type_code_, - sc_data_etype::U8, sc_data_etype::S8), - "dynamic quantize's output shall be s8 or u8 dtype."); - } - attrs_ = attrs; - op_name_ = "dynamic_quantize"; -} - -void dynamic_quantize_op_t::get_graph_impl(std::shared_ptr &graph) { - // create new input logical tensors - std::vector inputs, outputs; - inputs = remake_logical_tensors(info_.inputs_); - outputs = remake_logical_tensors(info_.outputs_); - const auto qinfos = get_quantize_info_from_attrs(attrs_); - assert(utils::is_one_of(qinfos.dtype_, datatypes::u8, datatypes::s8)); - auto &inp = inputs[0]; - auto &scales = inputs[1]; - auto inp_op = graph->make_input(inputs); - auto bc_axis = scales->details_.get_plain_dims() != sc_dims{1} - ? std::vector{qinfos.channel_axis_} - : std::vector{}; - auto div_scale = - graph->make("div", {inp, scales}, {}, {{"bc_axis", bc_axis}}); - if (inputs.size() == 3) { - auto zp_cast = - graph->make("cast", {inputs[2]}, {}, {{"dtype", datatypes::f32}}); - div_scale = graph->make( - "add", {div_scale->get_outputs()[0], zp_cast->get_outputs()[0]}, {}, - {{"bc_axis", bc_axis}}); - } - auto int8_cast = graph->make("cast", div_scale->get_outputs(), {}, - {{"dtype", qinfos.dtype_}, {"saturated", true}}); - graph->make_output(int8_cast->get_outputs()); -} - -void dynamic_quantize_op_t::query_format( - context_ptr ctx, - std::vector> &supported_ins, - std::vector> &supported_outs) { - common_query_function(this, info_.inputs_[0]->details_.get_format(), - supported_ins, supported_outs); -} - -dynamic_dequantize_op_t::dynamic_dequantize_op_t( - const std::vector &ins, - const std::vector &outs, const any_map_t &attrs) { - assert(ins.size() == 2 || ins.size() == 3); - assert(ins[0]->details_.dtype_.type_code_ == sc_data_etype::U8 || - ins[0]->details_.dtype_.type_code_ == sc_data_etype::S8 || - ins[0]->details_.dtype_.type_code_ == sc_data_etype::S32); - info_.inputs_ = ins; - const auto qinfos = get_quantize_info_from_attrs(attrs); - check_dynamic_quantization_schema(qinfos, info_.inputs_); - if (outs.empty()) { - info_.outputs_.emplace_back(std::make_shared(this)); - info_.outputs_[0]->details_ = ins[0]->details_; - info_.outputs_[0]->details_.dtype_ = datatypes::f32; - } else { - info_.outputs_ = outs; - COMPILE_ASSERT(gc::graph::check_shape_equal( - info_.inputs_[0]->details_.get_plain_dims(), - info_.outputs_[0]->details_.get_plain_dims()), - "dynamic dequantize's output shall have the same shape as " - "input."); - COMPILE_ASSERT(info_.outputs_[0]->details_.dtype_.type_code_ == - sc_data_etype::F32, - "dynamic dequantize's output shall be fp32 dtype."); - } - attrs_ = attrs; - op_name_ = "dynamic_dequantize"; -} - -void dynamic_dequantize_op_t::get_graph_impl( - std::shared_ptr &graph) { - // create new input logical tensors - std::vector inputs, outputs; - inputs = remake_logical_tensors(info_.inputs_); - outputs = remake_logical_tensors(info_.outputs_); - auto qinfos = get_quantize_info_from_attrs(attrs_); - qinfos.dtype_ = datatypes::f32; - auto &inp = inputs[0]; - auto &scales = inputs[1]; - auto inp_op = graph->make_input(inputs); - auto f32_cast = graph->make("cast", {inp}, {}, {{"dtype", datatypes::f32}}); - auto bc_axis = scales->details_.get_plain_dims() != sc_dims{1} - ? std::vector{qinfos.channel_axis_} - : std::vector{}; - if (inputs.size() == 3) { - auto zp_cast = - graph->make("cast", {inputs[2]}, {}, {{"dtype", datatypes::f32}}); - f32_cast = graph->make( - "sub", {f32_cast->get_outputs()[0], zp_cast->get_outputs()[0]}, {}, - {{"bc_axis", bc_axis}}); - } - auto mul_scale = graph->make("mul", {f32_cast->get_outputs()[0], scales}, {}, - {{"bc_axis", bc_axis}}); - graph->make_output(mul_scale->get_outputs()); -} - -void dynamic_dequantize_op_t::query_format( - context_ptr ctx, - std::vector> &supported_ins, - std::vector> &supported_outs) { - common_query_function(this, info_.inputs_[0]->details_.get_format(), - supported_ins, supported_outs); -} - -} // namespace quantize - -OP_REGISTER(quantize::quantize_op_t, quantize) -OP_REGISTER(quantize::dequantize_op_t, dequantize) -OP_REGISTER(quantize::dynamic_quantize_op_t, dynamic_quantize) -OP_REGISTER(quantize::dynamic_dequantize_op_t, dynamic_dequantize) -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/quantization/quantize_op.hpp b/legacy/core/src/compiler/ir/graph/quantization/quantize_op.hpp deleted file mode 100644 index 79e5723a0..000000000 --- a/legacy/core/src/compiler/ir/graph/quantization/quantize_op.hpp +++ /dev/null @@ -1,104 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_QUANTIZATION_QUANTIZE_OP_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_QUANTIZATION_QUANTIZE_OP_HPP - -#include "quantize_info.hpp" -#include -#include -#include -#include -#include -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace quantize { -/** - * A temporary quantize and requantize op before calculation for conversion from - * frontend. Support bf16 and int8/uint8 quantization. - * General quantize op, scales and zero points are stored in attribute. - * */ -class quantize_op_t : public graph_op_t, public op_traits::auto_copyable_t { -public: - quantize_op_t(const std::vector &ins, - const std::vector &outs, - const any_map_t &attrs); - quantize_op_t(const std::vector &ins, - const any_map_t &attrs = any_map_t()); - void get_graph_impl(std::shared_ptr &graph) override; - void query_format( - context_ptr ctx, - std::vector> &supported_ins, - std::vector> &supported_outs) override; -}; - -// dequantize op -class dequantize_op_t : public graph_op_t, public op_traits::auto_copyable_t { -public: - dequantize_op_t(const std::vector &ins, - const std::vector &outs, - const any_map_t &attrs); - dequantize_op_t(const std::vector &ins, - const any_map_t &attrs = any_map_t()); - - void get_graph_impl(std::shared_ptr &graph) override; - void query_format( - context_ptr ctx, - std::vector> &supported_ins, - std::vector> &supported_outs) override; -}; - -/** - * Dynamic quantize op, with scales and zero points as extra inputs. - * */ -class dynamic_quantize_op_t : public graph_op_t, - public op_traits::auto_copyable_t { -public: - dynamic_quantize_op_t(const std::vector &ins, - const std::vector &outs, - const any_map_t &attrs); - dynamic_quantize_op_t(const std::vector &ins, - const any_map_t &attrs = any_map_t()); - void get_graph_impl(std::shared_ptr &graph) override; - void query_format( - context_ptr ctx, - std::vector> &supported_ins, - std::vector> &supported_outs) override; -}; - -// dynamic dequantize op -class dynamic_dequantize_op_t : public graph_op_t, - public op_traits::auto_copyable_t { -public: - dynamic_dequantize_op_t(const std::vector &ins, - const std::vector &outs, - const any_map_t &attrs); - dynamic_dequantize_op_t(const std::vector &ins, - const any_map_t &attrs = any_map_t()); - - void get_graph_impl(std::shared_ptr &graph) override; - void query_format( - context_ptr ctx, - std::vector> &supported_ins, - std::vector> &supported_outs) override; -}; -} // namespace quantize -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl -#endif diff --git a/legacy/core/src/compiler/ir/graph/tensor_detail.hpp b/legacy/core/src/compiler/ir/graph/tensor_detail.hpp deleted file mode 100644 index 1e353e453..000000000 --- a/legacy/core/src/compiler/ir/graph/tensor_detail.hpp +++ /dev/null @@ -1,140 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_TENSOR_DETAIL_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_TENSOR_DETAIL_HPP - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -namespace reflection { -template struct type_registry; -} -class sc_graph_t; -struct SC_INTERNAL_API logical_tensor_t { -public: - sc_data_type_t dtype_; - logical_tensor_t() = default; - - bool operator==(const logical_tensor_t &other) const { - return format_ == other.format_ && dtype_ == other.dtype_ && - plain_dims_ == other.plain_dims_ && strides_ == other.strides_; - } - - bool operator!=(const logical_tensor_t &other) const { - return !(*this == other); - } - - logical_tensor_t(const sc_data_format_t &format, const sc_dims &plain_dims, - const sc_data_type_t &type, const sc_dims &strides = {}) - : dtype_(type), format_(format), plain_dims_(plain_dims), - strides_(strides) { - internal_update(); - } - - // gets the dims, taking blocking into consideration, using cache - // If the shape is dynamic, this function should not be used in lowering. - // Use `sc_data_format_t::get_blocking_shapes_expr(plain_shape, format)` - // instead. - const sc_dims &get_blocking_dims() const; - // gets the blocking dims exprs, we could not just use - // graph.dims_to_expr(dims_) because of may generate block and num of block - // of dynamic var, we should calculate by - // sc_data_format_t::get_blocking_dims_expr(); - std::vector get_blocking_dims_expr(sc_graph_t &g) const; - // sets the logical dims in blocking format - void set_blocking_dims(const sc_dims &blocking_dims); - // gets the logical dims in plain format - const sc_dims &get_plain_dims() const { return plain_dims_; } - // gets strides corresponding to each blocking dim - const sc_dims &get_strides() const { return strides_; } - // In dynamic, strides should be compute by compute_dense_stride_expr() - std::vector get_strides_expr(sc_graph_t &) const; - // sets the logical dims in plain format - void set_plain_dims(const sc_dims &plain_dims); - // gets the data format - const sc_data_format_t &get_format() const { return format_; } - // sets the data format and invalidate the cached blocking_dims - void set_format(const sc_data_format_t &newv); - const std::unordered_set &get_format_candidates() const { - return format_candidates_; - } - // add a format to candidates, if exists in candidates, do nothing. - void add_format_candidate(const sc_data_format_t &newv); - // remove format candidate by format. - void remove_format_candidate(const sc_data_format_t &v); - // sets the format candidates and update the format_ if number of candidates - // == 1 - void set_format_candidates(const std::vector &newf); - // gets the size of the tensor in bytes - size_t get_blocking_byte_size() const; - // if the tensor is dynamic - bool is_dynamic() const; - // sets the strides - void set_strides(const sc_dims &strides); - // sets the data format and stride - void set_format_and_stride(const sc_data_format_t &newv, - const sc_dims &strides); - // judge whether the current logical tensor is dense - bool is_dense(); - // print the tensor detail to string - void to_string(std::ostream &os); - // used to compute dense stride based on dims - static sc_dims compute_dense_stride(const sc_dims &dims); - static std::vector - compute_dense_stride_expr(sc_graph_t &graph, const std::vector &dims); - size_t hash() const; - -private: - template friend struct reflection::type_registry; - friend struct std::hash; - // definite format for internal sync. - sc_data_format_t format_; - // The real dims, which may be blocking. - sc_dims dims_; - // the logical dims in plain format - sc_dims plain_dims_; - // strides corresponding to each dim of `dims_`, always dense in dynamic - sc_dims strides_; - // dynamic shape may has several format/stride pair in graph. - std::unordered_set format_candidates_; - // sync real dims based on plain dims and format - void internal_update(); -}; -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -namespace std { -template <> struct hash { - std::size_t - operator()(const dnnl::impl::graph::gc::logical_tensor_t &k) const; -}; -} // namespace std -#endif diff --git a/legacy/core/src/compiler/ir/graph/tensor_slice.hpp b/legacy/core/src/compiler/ir/graph/tensor_slice.hpp deleted file mode 100644 index e3c611ba7..000000000 --- a/legacy/core/src/compiler/ir/graph/tensor_slice.hpp +++ /dev/null @@ -1,84 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_TENSOR_SLICE_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_TENSOR_SLICE_HPP - -#include -#include //for std::pair -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -using slice_range = std::vector>; -/** - * A slice of the tensor. - * @param tptr_ the base tensor_ptr - * @param shape_ the slice shape - * */ -struct tensor_slice { - tensorptr tptr_; - std::vector shape_; - tensor_slice() = default; - - tensor_slice(const expr &tsr); - - tensor_slice(const expr &tsr, slice_range &&ranges); - - // Gets the start address of the tensor slice - expr get_tensor_ptr() const { return tptr_; } - - // Gets the shape of the sliced tensor - const std::vector &get_shape() const { return shape_; } - - int64_t nslice_dims() const { return static_cast(shape_.size()); } - int64_t nbase_dims() const { - return static_cast(get_base_dims().size()); - } - - // Gets the offset of the sliced tensor - const std::vector &get_offset() const { return tptr_->base_->idx_; } - - // Gets the ranges of the sliced tensor - slice_range get_ranges() const; - - // Gets the real shape of base tensor (const version) - const std::vector &get_base_dims() const; - - // Gets the dtype of base tensor - sc_data_type_t get_base_dtype() const; - - // Gets the real tensor of tensor slice, not the tensor_ptr - tensor get_real_tensor() const; - - // check whether slice is full on specific axis - bool full_on_axis(const std::vector &axis) const; - - // is_full - bool is_full() const; - - // is_const - bool is_const() const; -}; - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/graph/trait/configurable.hpp b/legacy/core/src/compiler/ir/graph/trait/configurable.hpp deleted file mode 100644 index 93ae19a3f..000000000 --- a/legacy/core/src/compiler/ir/graph/trait/configurable.hpp +++ /dev/null @@ -1,60 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_TRAIT_CONFIGURABLE_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_TRAIT_CONFIGURABLE_HPP - -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -using config_ptr_vec = std::vector; -using impl_kind_map = std::unordered_map, int>; -namespace op_traits { -struct configurable_t : public virtual op_base_trait_t { - virtual reflection::shared_general_object_t get_config() = 0; - - virtual void - set_config(const reflection::shared_general_object_t &config) = 0; - virtual reflection::shared_general_object_t - get_default_config(context_ptr ctx) = 0; - // Get config space in dynamic shape case, similar with get_config_space but - // due to the number limit of kernels, the space is shrinked from static - // config space. Return the vector of config ptr, index is the impl kind. - virtual config_ptr_vec - get_dynamic_config_candidates(const context_ptr &ctx) = 0; - // Convert the input config space vector to a unordered map for query in - // runtime. This function accepts the configs from output of - // `get_dynamic_config_candidates`. We need the intermediate - // result(config_ptr_vec) so we split the implement into this two functions. - virtual impl_kind_map - convert_config_candidates_to_impl_map(const config_ptr_vec &configs) = 0; -}; - -} // namespace op_traits -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/graph/trait/may_broadcast.cpp b/legacy/core/src/compiler/ir/graph/trait/may_broadcast.cpp deleted file mode 100644 index b6e555f1a..000000000 --- a/legacy/core/src/compiler/ir/graph/trait/may_broadcast.cpp +++ /dev/null @@ -1,155 +0,0 @@ -/******************************************************************************* - * Copyright 2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include -#include - -#include "may_broadcast.hpp" - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace op_traits { - -std::vector -may_broadcast_t::get_auto_broadcast_bc_axis(const sc_dims &input_shape, - const sc_dims &output_shape) { - if (input_shape.size() == 1 && input_shape[0] == 1) { - return {-1}; - } - // following auto_broadcast semantics - const size_t input_rank = input_shape.size(); - const size_t output_rank = output_shape.size(); - COMPILE_ASSERT(output_rank >= input_rank, - "Incorrect input or output shape for broadcast op."); - const size_t offset = output_rank - input_rank; - std::vector bc_axis; - for (size_t i = 0; i < input_rank; ++i) { - // TODO(yifei): consider whether input_shape[i] != 1 is - // necessary here - if (input_shape[i] == output_shape[i + offset] || - (is_dynamic_dim(input_shape[i]) && - is_dynamic_dim(output_shape[i + offset]))) { - bc_axis.emplace_back(i + offset); - } - } - if (bc_axis.empty()) { - bc_axis.emplace_back(-1); - } - return bc_axis; -} - -bool may_broadcast_t::broadcastable_shape_equal(const sc_dims &shape1, - const sc_dims &shape2) { - COMPILE_ASSERT(shape1.size() <= shape2.size(), - "broadcastable shape equal function shall have input shape1 " - "smaller than input shape2."); - if (shape1.size() != shape2.size()) - return false; - for (size_t i = 0; i < shape1.size(); ++i) { - if (!is_dynamic_dim(shape1[i]) && !is_dynamic_dim(shape2[i]) && - shape1[i] != shape2[i]) { - return false; - } else if (shape1[i] == 1 && is_dynamic_dim(shape2[i])) { - return false; - } - } - return true; -} - -sc_dims may_broadcast_t::infer_auto_broadcast_output_shape(const sc_dims &lhs, - const sc_dims &rhs) { - const size_t lhs_rank = lhs.size(); - const size_t rhs_rank = rhs.size(); - const size_t max_rank = std::max(lhs_rank, rhs_rank); - - const size_t lhs_offset = max_rank - lhs_rank; - const size_t rhs_offset = max_rank - rhs_rank; - bool auto_broadcastable = true; - sc_dims ret(max_rank, 1); - for (size_t i = 0; i < max_rank; ++i) { - sc_dim l = 1, r = 1; - if (i >= lhs_offset) - l = lhs[i - lhs_offset]; - if (i >= rhs_offset) - r = rhs[i - rhs_offset]; - if (l == 1 || r == 1) { - ret[i] = (l == 1 ? r : l); - } else if (is_dynamic_dim(l) && is_dynamic_dim(r)) { - // TODO(xxx): correctly handle dynamic case - ret[i] = l; - } else if (is_dynamic_dim(l) || is_dynamic_dim(r)) { - ret[i] = (is_dynamic_dim(l) ? r : l); - } else { - if (l != r) { - auto_broadcastable = false; - break; - } else { - ret[i] = l; - } - } - } - COMPILE_ASSERT(auto_broadcastable, - "The given input shapes do not follow auto_broadcast " - "semantics. " - "Please recheck the input shapes, or consider specifying " - "detailed " - "bc_axis."); - return ret; -} - -sc_data_format_t -may_broadcast_t::infer_broadcast_format(const logical_tensor_t &target_lt, - const logical_tensor_t &bc_lt) { - COMPILE_ASSERT(bc_lt.get_plain_dims().size() == - target_lt.get_plain_dims().size(), - "infer_blocking_format only support plain dimension aligned " - "cases"); - sc_data_format_kind_t target_lt_format_code = - target_lt.get_format().format_code_; - sc_data_format_t::blocking_t blocks = target_lt.get_format().blocks_; - sc_data_format_kind_t bc_lt_format_code = bc_lt.get_format().format_code_; - // start infer the blocks - sc_dims bc_plain_dim = bc_lt.get_plain_dims(); - sc_dims target_plain_dim = target_lt.get_plain_dims(); - int target_batch_dim = - target_lt.get_plain_dims().size() - target_lt_format_code.norig_dims(); - for (int i = 0; i < target_lt_format_code.norig_dims(); ++i) { - if (bc_plain_dim[target_batch_dim + i] == 1 && - target_plain_dim[target_batch_dim + i] != 1) { - // if bc_plain_dim is 1 and this axis is with broadcast - // semantics - auto axis = target_lt_format_code.collect_blocking_index(i); - for (auto ax : axis) { - blocks[ax] = 1; - } - } - } - // start infer the format code - // if both batch OR both non-batch - // smaller side's format code == larger side's format code - COMPILE_ASSERT(target_lt_format_code.norig_dims() == - bc_lt_format_code.norig_dims(), - "Unsupported case for broadcastable op query format."); - return sc_data_format_t(target_lt.get_format().format_code_, blocks); -} - -} // namespace op_traits -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/trait/may_broadcast.hpp b/legacy/core/src/compiler/ir/graph/trait/may_broadcast.hpp deleted file mode 100644 index ea21ccfeb..000000000 --- a/legacy/core/src/compiler/ir/graph/trait/may_broadcast.hpp +++ /dev/null @@ -1,81 +0,0 @@ -/******************************************************************************* - * Copyright 2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_TRAIT_MAY_BROADCAST_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_TRAIT_MAY_BROADCAST_HPP - -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace op_traits { -struct may_broadcast_t : public virtual op_base_trait_t { - static constexpr int NOT_DETERMINED = -2; - /** - * Returns the index of all the input logical tensors that do - * not need to be broadcasted (whose shape same as the output) - * */ - virtual std::vector - get_non_broadcast_input_index(bool assert_non_empty) const = 0; - - /** - * Returns the input index serving as the layout reference, which tends to - * be the index of the largest input - * */ - virtual int get_ref_input_index(bool assert_determined) const = 0; - - const std::vector> &get_plain_bc_axis() const { - return plain_bc_axis_; - } - - /** - * Returns a vector of input_shape's axis whose shape matches with the - * corresponding dimension of the output (so no need to be broadcasted) - * */ - static std::vector - get_auto_broadcast_bc_axis(const sc_dims &input_shape, - const sc_dims &output_shape); - - /** - * Returns whether shape1 and shape2 are equal (means shape1 to shape 2 does - * not involve broadcast). - * Assuming that shape1 is broadcastable to shape2. - * */ - static bool broadcastable_shape_equal(const sc_dims &shape1, - const sc_dims &shape2); - - static sc_dims infer_auto_broadcast_output_shape(const sc_dims &lhs, - const sc_dims &rhs); - - static sc_data_format_t - infer_broadcast_format(const logical_tensor_t &target_lt, - const logical_tensor_t &bc_lt); - -protected: - std::vector> plain_bc_axis_; -}; - -} // namespace op_traits -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/graph/trait/may_inplace.hpp b/legacy/core/src/compiler/ir/graph/trait/may_inplace.hpp deleted file mode 100644 index c4f79463e..000000000 --- a/legacy/core/src/compiler/ir/graph/trait/may_inplace.hpp +++ /dev/null @@ -1,51 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_TRAIT_MAY_INPLACE_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_TRAIT_MAY_INPLACE_HPP - -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace op_traits { -struct may_inplace_t : public virtual op_base_trait_t { - /** - * @brief Get the inplace mapping - * - * @return the list of >. - * output_tensor_index is the index of an output tensor of this op staring - * from 0. input_tensor_index is the index of an input tensor of this op - * staring from 0. returning empty vector means that this op cannot inplace - * reuse any input - */ - virtual std::vector>> - get_inplace_map() = 0; -}; - -} // namespace op_traits -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/graph/trait/may_prefetch.cpp b/legacy/core/src/compiler/ir/graph/trait/may_prefetch.cpp deleted file mode 100644 index de8b8d004..000000000 --- a/legacy/core/src/compiler/ir/graph/trait/may_prefetch.cpp +++ /dev/null @@ -1,170 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "may_prefetch.hpp" -#include -#include -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace op_traits { - -// replace all var/tensors in the tensor with remade ones, because we are -// creating a new function -class remake_args_visitor_t : public ir_visitor_t { -public: - using ir_visitor_t::visit; - std::vector args_; - std::vector old_args_; - std::unordered_map mapping_; - - expr get_mapped(const expr_c &v) { - auto itr = mapping_.find(v); - if (itr != mapping_.end()) { - return itr->second; - } - auto ret = v->remake(); - mapping_[v] = ret; - args_.emplace_back(ret); - old_args_.emplace_back(v.remove_const()); - return ret; - } - - expr_c visit(var_c v) override { return get_mapped(v); } - expr_c visit(tensor_c v) override { return get_mapped(v); } -}; - -void may_prefetch_t::generate_prefetcher_body_for_slice( - const context_ptr &ctx, const std::vector &func_args, - const std::vector &ins, const std::vector &indices) { - COMPILE_ASSERT(false, "not yet implemented"); -} - -func_t may_prefetch_t::generate_prefetcher_and_set_idle( - const context_ptr &ctx, bool is_global, - const std::vector &ins, const std::vector &indices, - std::vector &out_set_idle_code) { - std::vector new_ins; - new_ins.reserve(ins.size()); - // remake the vars and tensors - remake_args_visitor_t vis; - sc_data_type_t type_trigger = datatypes::s32; - vis.args_.emplace_back(builder::make_tensor("state", {1}, type_trigger)); - vis.args_.emplace_back(builder::make_var(type_trigger, "expected")); - // the tid in func arg. It may not be valid (-1), we need to check it and - // reassign the real tid if necessary - auto tid_in_arg = builder::make_var(datatypes::s32, "tid"); - vis.args_.emplace_back(tid_in_arg); - size_t tid_in_arg_idx = vis.args_.size() - 1; - - size_t num_standard_args = vis.args_.size(); - vis.old_args_.emplace_back(); - - std::vector generator_args = vis.args_; - - vis.args_.front()->attr()["volatile"] = true; - // remake the tensor slice - for (auto &idx : indices) { - auto &v = ins.at(idx); - auto base = vis.visit(v.tptr_).checked_as(); - std::vector shapes; - for (auto &s : v.shape_) { - shapes.emplace_back(vis.dispatch(s).remove_const()); - } - new_ins.emplace_back(); - new_ins.back().tptr_ = std::move(base); - new_ins.back().shape_ = std::move(shapes); - } - - auto func_body = make_stmt(std::vector()); - std::vector func_args; - for (size_t i = 0; i < num_standard_args; i++) { - func_args.emplace_back(vis.args_[i]); - } - - expr general_args; - if (num_standard_args + 1 == vis.args_.size()) { - general_args = vis.args_[num_standard_args]; - } else { - general_args = builder::make_tensor("__args", {vis.args_.size() - 1}, - datatypes::generic); - // extract the arguments in new prefetch function from general arg array - for (size_t i = num_standard_args; i < vis.args_.size(); i++) { - auto &arg = vis.args_[i]; - func_body->seq_.emplace_back(builder::make_var_tensor_def_unattached( - arg, linkage::local, - builder::make_cast(arg->dtype_, - general_args[i - num_standard_args]))); - } - } - - func_args.emplace_back(general_args); - - // call the user func to generate the func body - builder::ir_builder_t builder; - builder.push_scope(); - - // the real tid - auto realtid = builder::make_var(datatypes::s32, "realtid"); - builder.push_var_tensor_def( - realtid, linkage::local, - builder::make_select(tid_in_arg < 0, builtin::get_thread_id_func()(), - tid_in_arg)); - generator_args[tid_in_arg_idx] = realtid; - - std::vector args_ins; - for (auto &t : new_ins) { - COMPILE_ASSERT(t.is_full(), - "Cannot generate prefetcher for tensor slice yet."); - args_ins.emplace_back(t.get_real_tensor()); - } - generate_prefetcher_body_for_tensor(ctx, generator_args, args_ins, indices); - - auto body = builder.pop_scope(); - for (auto &s : body.checked_as()->seq_) { - func_body->seq_.emplace_back(std::move(s)); - } - - auto op = dynamic_cast(this); - std::string func_name = op->op_name_; - func_name += "_"; - func_name += std::to_string(op->logical_op_id_); - func_name += "_prefetch"; - auto retfunc = - builder::make_func(func_name, func_args, func_body, datatypes::index); - retfunc->attr()[function_attrs::low_level] = true; - retfunc->attr()[function_attrs::private_] = true; - retfunc->decl_->attr()[function_attrs::private_] = true; - - vis.old_args_[0] = builder::make_func_addr(retfunc); - out_set_idle_code.emplace_back( - builder::make_evaluate_unattached(make_expr( - intrin_type::set_thread_idle_func, vis.old_args_, any_map_t{}))); - return retfunc; -} -} // namespace op_traits -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/trait/may_prefetch.hpp b/legacy/core/src/compiler/ir/graph/trait/may_prefetch.hpp deleted file mode 100644 index fe00a374a..000000000 --- a/legacy/core/src/compiler/ir/graph/trait/may_prefetch.hpp +++ /dev/null @@ -1,109 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_TRAIT_MAY_PREFETCH_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_TRAIT_MAY_PREFETCH_HPP - -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace op_traits { -struct may_prefetch_t : public virtual op_base_trait_t { - /** - * @brief Query the the indices of inputs which needs prefetching - * - * @param ctx the context - * @param is_global query for global or local prefetch. @see - * generate_prefetcher_and_set_idle - * @param ins the vector of input tensor slices - * @return std::vector a vector of input tensor indices - */ - virtual std::vector - query_prefetch(const context_ptr &ctx, bool is_global, - const std::vector &ins) = 0; - - /** - * @brief generate prefetcher code for the inputs selected by query_prefetch - * into the current ir_builder. This function is for tensor slice. - * - * @param ctx the context - * @param func_args the arguments for the prefetcher function. The first arg - * is the pointer to the trigger value(as a tensor). The second arg is the - * expected value. The prefetcher should exit when trigger value != - * expected. - * @param ins the inputs, in the new prefetch IR function created by - * generate_prefetcher_func. The tensors and vars are in func_args - * @param indices the input indices selected by query_prefetch - */ - virtual void generate_prefetcher_body_for_slice( - const context_ptr &ctx, const std::vector &func_args, - const std::vector &ins, const std::vector &indices); - - /** - * @brief generate prefetcher code for the inputs selected by query_prefetch - * into the current ir_builder. This function is for whole tensor. - * - * @param ctx the context - * @param func_args the arguments for the prefetcher function. The first - * arg is the pointer to the trigger value(as a tensor). The second arg is - * the expected value. The prefetcher should exit when trigger value == - * expected. The third arg is the tid. - * @param ins the inputs, in the new prefetch IR function created by - * generate_prefetcher_func. The tensors and vars are in func_args - * @param indices the input indices selected by query_prefetch - */ - virtual void generate_prefetcher_body_for_tensor( - const context_ptr &ctx, const std::vector &func_args, - const std::vector &ins, const std::vector &indices) = 0; - - /** - * @brief generate prefetcher function for the inputs selected by - * query_prefetch. The default inplementation calles - * generate_prefetcher_body. Op implementations usually don't need to - * override this function. - * - * @param ctx the context - * @param is_global whether the user wants a global or local prefetcher. A - * global prefetcher will be registered in the main entry function via - * managed thread pool API and the local prefetcher is passed to - * sc_arrive_at_barrier. Local prefetchers are usually used in fused op. The - * prefetcher interfaces are slightly different. - * @param ins the slice for all inputs of the Op, in the Op's computation - * code. Should be the same as the parameter passed to query_prefetch - * @param indices the input indices selected by query_prefetch - * @param out_set_idle_code outputs the statements for calling - * set_thread_idle_func() - * @return the generated prefetcher function - */ - virtual func_t - generate_prefetcher_and_set_idle(const context_ptr &ctx, bool is_global, - const std::vector &ins, - const std::vector &indices, - std::vector &out_set_idle_code); -}; - -} // namespace op_traits -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/graph/traits.hpp b/legacy/core/src/compiler/ir/graph/traits.hpp deleted file mode 100644 index 10e2c43dd..000000000 --- a/legacy/core/src/compiler/ir/graph/traits.hpp +++ /dev/null @@ -1,178 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_TRAITS_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_TRAITS_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -class fusion_anchor_mgr_t; -struct brgemm_fusion_register; -struct mixed_parti_t; -struct fusion_anchor_t; - -namespace op_traits { -struct copyable_t : public virtual op_base_trait_t { - virtual sc_op_ptr copy(const std::vector &ins, - const std::vector &outs, - sc_graph_t &mgr) = 0; -}; - -/** - * A class is auto-copyable if we can construct a valid copy of the node with - * the in/out tensors and the attrs of the node. The op name should be in the op - * registery - * */ -struct auto_copyable_t : public copyable_t { - sc_op_ptr copy(const std::vector &ins, - const std::vector &outs, - sc_graph_t &mgr) override; -}; - -/** - * @brief The util trait template for Ops that is "almost" auto-copyable except - * that they need to copy the data in the trait. - * - * @tparam TArgs op_traits that has copy_from methods - */ -template -struct auto_copyable_with_trait_t : public auto_copyable_t { - template - static void copy_impl(auto_copyable_with_trait_t *from, sc_op *to) { - auto pto = dynamic_cast(to); - assert(pto); - auto pfrom = dynamic_cast(from); - assert(pfrom); - pto->copy_from(pfrom); - } - - template - static void - copy_impl(auto_copyable_with_trait_t *from, - typename std::enable_if<(sizeof...(T) > 0), sc_op *>::type to) { - copy_impl(from, to); - copy_impl(from, to); - } - sc_op_ptr copy(const std::vector &ins, - const std::vector &outs, - sc_graph_t &mgr) override { - auto ret = auto_copyable_t::copy(ins, outs, mgr); - copy_impl(this, ret.get()); - return ret; - } -}; - -// the OP can be optimized if some of the inputs are constants -struct constant_optimizable_t : public virtual op_base_trait_t { - // do optimization and return the new optmized op. If no optimization can be - // applied, return null - virtual sc_op_ptr constant_optimize(sc_graph_t &graph) = 0; -}; - -// the part of OP's workload can be computed, e.g. intrisics(brgemm), tensor -// slice. -struct workload_computable_t : public virtual op_base_trait_t { - using shape_dtype_pair = std::pair; - static const size_t read_weight = parallel_workload::read_weight; - static const size_t write_weight = parallel_workload::write_weight; - static constexpr const char *workload_number = - parallel_workload::attr_workload_number; - // compute workload with given input and output tensor pointers, according - // to read/write times and operator numbers. - virtual size_t - compute_workload(const std::vector &ins, - const std::vector &outs) = 0; -}; - -// the OP can accept a fusion manager to do post fusion -struct post_fusion_acceptable_t : public virtual op_base_trait_t { - virtual ir_module_ptr - get_func(context_ptr ctx, - const std::shared_ptr &fuse_mgr, - const std::string &func_name) = 0; -}; - -// the OP can be fused into brgemm calculation. -struct brgemm_fusion_acceptable_t : public virtual op_base_trait_t { - static constexpr const char *brgemm_fusion = "brgemm_fusion"; - bool fuse_in_brgemm_ = false; - brgemm::alg_kind_t alg_kind_ = brgemm::alg_kind_t::alg_kind_undef; - virtual bool - register_brgemm_fusion(const context_ptr &ctx, - const std::vector &outputs, - const std::vector &inputs, - brgemm_fusion_register &brg_reg) = 0; - void copy_from(brgemm_fusion_acceptable_t *from) { - fuse_in_brgemm_ = from->fuse_in_brgemm_; - alg_kind_ = from->alg_kind_; - } -}; - -// quantize -struct may_quantize_t : public virtual op_base_trait_t { - virtual sc_op_ptr do_compensations(sc_graph_t &mgr, const context_ptr &ctx) { - need_compensation_ = false; - return sc_op_ptr(); - } - bool should_quantized_ = false; - bool is_quantized_ = false; - bool need_compensation_ = true; -}; - -struct mixed_partition_acceptable : public virtual op_base_trait_t { - // create a new partition for current op - virtual void create_mixed_partition(mixed_parti_t *parti) = 0; - - // append current op to the existed partition - virtual void append_mixed_partition(mixed_parti_t *parti) = 0; - - // search fusion anchor for current op in given partition - virtual void search_anchor(mixed_parti_t *parti) = 0; - - // commit current op into given partition - virtual void commit_into_anchor(fusion_anchor_t *committed_anchor) = 0; - - // infer binding axis from inputs to outputs - virtual void infer_binding_axis(binding_axis_map &bdax_map) = 0; - - // infer binding axis from outputs to inputs - virtual void pre_infer_binding_axis(binding_axis_map &bdax_map) = 0; -}; - -struct data_compensation_t : public virtual op_base_trait_t {}; -struct weight_compensation_t : public virtual op_base_trait_t {}; -struct constant_compensation_t : public virtual op_base_trait_t {}; - -} // namespace op_traits - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/graph/transform/annotate_config.cpp b/legacy/core/src/compiler/ir/graph/transform/annotate_config.cpp deleted file mode 100644 index 3dc69b59a..000000000 --- a/legacy/core/src/compiler/ir/graph/transform/annotate_config.cpp +++ /dev/null @@ -1,111 +0,0 @@ -/******************************************************************************* - * Copyright 2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "../fusible_op.hpp" -#include "../graph_op.hpp" -#include "../pass/pass.hpp" -#include "../visitor.hpp" -#include "transform.hpp" -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -// Give annotation to graph_op which contains tunable op. The same tunable op -// with a given default config will perform differently in different graphs, -// considering the potential impact of fusion. Attributes are given in this pass -// to tell tunable op use proper config (or even format) accordingly. Currently -// only managed_matmul_core_op benefits from this pass. -void annotate_config(sc_graph_t &graph, const context_ptr &ctx) { - // annotate graph with matmul+reduce fusion - auto vis0 = op_visitor_t::bfs(); - bool is_int8 = false; - vis0.visit_graph(graph, [&](op_visitor_t *vis0, const sc_op_ptr &node) { - // consider mixed-datatype - if (node->isa()) { - is_int8 = true; - } - if (node->op_name_ == "matmul" || node->op_name_ == "managed_matmul_core") { - // currently only consider static case - if (node->is_dynamic()) { - return; - } - size_t use_size = node->get_outputs()[0]->uses_.size(); - auto next_node = node->get_outputs()[0]->uses_.at(0).second.get_shared(); - while (!next_node->isa() && !next_node->isa() && - next_node->op_name_ != "matmul") { - if (next_node->isa() || - next_node->isa() || - next_node->op_name_ == "layernorm") { - auto data_dims = node->get_inputs()[0]->details_.get_plain_dims(); - auto weight_dims = node->get_inputs()[1]->details_.get_plain_dims(); - auto out_dims_size = std::max(data_dims.size(), weight_dims.size()); - bool transpose_b = node->attrs_.get_or_else("transpose_b", false); - bool transpose_a = node->attrs_.get_or_else("transpose_a", false); - if (next_node->attrs_.has_key("rd_axis") && - !next_node->attrs_.get_or_else(op_attr_key::break_pre_fuse, - false)) { - std::vector rd_axis = - next_node->attrs_.get>("rd_axis"); - if ((weight_dims.size() == 2 || data_dims.size() == 2) && - rd_axis.size() == 1 && - rd_axis.at(0) == static_cast(out_dims_size) - 1) { - auto K = transpose_b ? weight_dims.at(1) : weight_dims.at(0); - auto M = transpose_a ? data_dims.at(1) - : math_utils::get_dims_product(data_dims) / - data_dims.back(); - if (((K >= 640 && K < 4096) || M < 12288) && - utils::is_one_of(node->get_inputs()[0]->details_.dtype_, - datatypes::bf16, datatypes::f16) && - !is_int8) { - next_node->attrs_.set(op_attr_key::break_pre_fuse, true); - break; - } - } - node->attrs_.set("post_rd_axis", rd_axis); - break; - } - } - if (next_node->attrs_.get_or_else(op_attr_key::break_pre_fuse, false)) { - break; - } - if (next_node->get_outputs()[0]->uses_.size() > 1) { - break; - } - if (next_node->isa() && - !(next_node->get_inputs()[1]->producer_owner_->isa()) && - node->get_inputs()[0]->details_.dtype_ == datatypes::f32 && - !is_int8) { - // consider residual cases - break; - } - next_node = - next_node->get_outputs()[0]->uses_.at(0).second.get_shared(); - } - } - }); -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/transform/brgemm_fusion.cpp b/legacy/core/src/compiler/ir/graph/transform/brgemm_fusion.cpp deleted file mode 100644 index 2c39d99c3..000000000 --- a/legacy/core/src/compiler/ir/graph/transform/brgemm_fusion.cpp +++ /dev/null @@ -1,596 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -static const constexpr char *forbidden_brgemm_fusion = - "forbidden_brgemm_fusion"; - -enum class brg_broadcast_type { - non_broadcast = 0, // {N, C, H, W} - non_channel, // {N, 1, H, W} => {N, C, H, W} - only_channel, // {1, C, 1, 1} => {N, C, H, W} - all_broadcast, // {1} -}; - -static bool all_ones_except_channel(const sc_dims &shapes, int channel_axis) { - assert(!shapes.empty()); - for (int i = 0; i < static_cast(shapes.size()); i++) { - if (i != channel_axis && shapes[i] != 1) { - return false; - } - } - return true; -} - -static op_traits::brgemm_fusion_acceptable_t * -to_brg_fusible(const sc_op_ptr &p) { - return p->dyn_cast(); -} - -static brg_broadcast_type get_brgemm_broadcast_type( - const sc_data_format_t &l_format, const sc_data_format_t &r_format, - const sc_dims &l_plain_dims, const sc_dims &r_plain_dims) { - int channel_axis = - l_format.format_code_.get(l_format.format_code_.ndims() - 1); - sc_dim channel_size = l_plain_dims.at(channel_axis); - COMPILE_ASSERT( - (l_format.format_code_ == r_format.format_code_ && - l_plain_dims.size() == r_plain_dims.size()) || - r_plain_dims.size() == 1, - "The tensors on the left and right hands cannot be calculated."); - sc_dims expected_all_broadcast{1}; - if (r_plain_dims == l_plain_dims && - !all_ones_except_channel(l_plain_dims, channel_axis)) { - return brg_broadcast_type::non_broadcast; - } else if (r_plain_dims == expected_all_broadcast) { - return brg_broadcast_type::all_broadcast; - } else if (r_plain_dims.size() == 1 || - r_plain_dims.at(channel_axis) == l_plain_dims.at(channel_axis)) { - return brg_broadcast_type::only_channel; - } else if (r_plain_dims.at(channel_axis) == 1) { - return brg_broadcast_type::non_channel; - } else { - throw std::runtime_error( - "The shapes on left and right sides do not correspond!"); - } -} -struct fusion_state_t { - int max_op_count_ = 1; - int cur_op_count_ = 0; - template bool isa() const { - static_assert(std::is_base_of::value, - "T is not a subclass of fusion_state_t."); - return dynamic_cast(this); - } - virtual void reset() { cur_op_count_ = 0; } - // the cur_op can be accepted or not in this state. - virtual bool is_acceptable(const sc_op_ptr &cur_op) = 0; - virtual std::vector> next_possible_states( - const std::vector> &all_states) = 0; - virtual void do_transform(sc_graph_t &graph, const sc_op_ptr &main_op, - const sc_op_ptr &cur_op) { - to_brg_fusible(cur_op)->fuse_in_brgemm_ = true; - } - virtual ~fusion_state_t() = default; -}; -#define GET_BROADCAST_TYPE() \ - auto l_format = cur_op->get_inputs()[0]->details_.get_format(); \ - auto r_format = cur_op->get_inputs()[1]->details_.get_format(); \ - auto l_plain_dims = cur_op->get_inputs()[0]->details_.get_plain_dims(); \ - auto r_plain_dims = cur_op->get_inputs()[1]->details_.get_plain_dims(); \ - auto bct_type = get_brgemm_broadcast_type(l_format, r_format, l_plain_dims, \ - r_plain_dims); - -struct cast_state_t; -struct scale_state_t; -struct bias_state_t; -struct elem_state_t; -struct c_zp_state_t; -struct ab_zp_state_t : public fusion_state_t { - bool has_a_zp = false; - bool has_b_zp = false; - ab_zp_state_t() : has_a_zp(false), has_b_zp(false) { max_op_count_ = 2; } - void reset() override { - cur_op_count_ = 0; - has_a_zp = false; - has_b_zp = false; - } - bool is_acceptable(const sc_op_ptr &cur_op) override { - if (cur_op_count_ >= max_op_count_) { - return false; - } - if (!cur_op->isa()) { - return false; - } - auto sub_op = cur_op->stc_cast(); - if (sub_op->get_broadcast_input() == 0) { - return false; - } - GET_BROADCAST_TYPE(); - // if non-broadcast or all broadcast, return - if (bct_type == brg_broadcast_type::non_broadcast || - bct_type == brg_broadcast_type::all_broadcast) { - return false; - } - // should be s32 - if (!cur_op->get_inputs()[0]->details_.dtype_.is_etype( - sc_data_etype::S32) || - !cur_op->get_inputs()[1]->details_.dtype_.is_etype( - sc_data_etype::S32)) { - return false; - } - if (!has_a_zp && bct_type == brg_broadcast_type::only_channel) { - has_a_zp = true; - cur_op_count_++; - sub_op->alg_kind_ = brgemm::alg_kind_t::a_zp; - return true; - } - if (!has_b_zp && bct_type == brg_broadcast_type::non_channel) { - has_b_zp = true; - cur_op_count_++; - sub_op->alg_kind_ = brgemm::alg_kind_t::b_zp; - return true; - } - return false; - } - std::vector> next_possible_states( - const std::vector> &all_states) override { - std::vector> ret; - for (auto &st : all_states) { - if (st->isa() || st->isa()) { - ret.emplace_back(st); - } - } - return ret; - } -}; - -struct cast_state_t : public fusion_state_t { - // calculation type cast, memory storage type cast - cast_state_t() { max_op_count_ = 2; } - bool is_acceptable(const sc_op_ptr &cur_op) override { - if (cur_op_count_ >= max_op_count_) { - return false; - } - if (!cur_op->isa()) { - return false; - } - cur_op_count_++; - cur_op->stc_cast()->alg_kind_ = brgemm::alg_kind_t::out_dtype; - return true; - } - std::vector> next_possible_states( - const std::vector> &all_states) override { - std::vector> ret; - for (auto &st : all_states) { - if (st->isa() || st->isa() || - st->isa() || st->isa()) { - ret.emplace_back(st); - } - } - return ret; - } -}; - -struct scale_state_t : public fusion_state_t { - bool is_acceptable(const sc_op_ptr &cur_op) override { - if (cur_op_count_ >= max_op_count_) { - return false; - } - if (!cur_op->isa()) { - return false; - } - auto mul_op = cur_op->stc_cast(); - if (mul_op->get_broadcast_input() == 0) { - return false; - } - - GET_BROADCAST_TYPE(); - // support only_channel, all broadcast - if (bct_type == brg_broadcast_type::non_broadcast || - bct_type == brg_broadcast_type::non_channel) { - return false; - } - if (bct_type == brg_broadcast_type::all_broadcast && - !mul_op->get_inputs()[1]->producer_owner_->isa()) { - return false; - } - // should be f32 - if (!cur_op->get_inputs()[0]->details_.dtype_.is_etype( - sc_data_etype::F32) || - !cur_op->get_inputs()[1]->details_.dtype_.is_etype( - sc_data_etype::F32)) { - return false; - } - cur_op_count_++; - mul_op->alg_kind_ = brgemm::alg_kind_t::out_scales; - return true; - } - std::vector> next_possible_states( - const std::vector> &all_states) override { - std::vector> ret; - for (auto &st : all_states) { - if (st->isa() || st->isa() || - st->isa() || st->isa()) { - ret.emplace_back(st); - } - } - return ret; - } - - // scalar scale to vector - void do_transform(sc_graph_t &graph, const sc_op_ptr &main_op, - const sc_op_ptr &cur_op) override { - if (auto const_op = cur_op->get_inputs()[1] - ->producer_owner_->dyn_cast()) { - if (const_op->get_constant_plain_dims() == sc_dims{1}) { - float scale = reinterpret_cast( - const_op->get_constant_values()->data_)[0]; - auto main_format = main_op->get_outputs()[0]->details_.get_format(); - auto main_plain_shapes = - main_op->get_outputs()[0]->details_.get_plain_dims(); - auto &fcode = main_format.format_code_; - int channel_axis = fcode.get(fcode.ndims() - 1); - sc_dim channel_size = main_plain_shapes.at(channel_axis); - std::vector scales(channel_size, scale); - sc_dims new_plain_dims(main_plain_shapes.size(), 1); - new_plain_dims.at(channel_axis) = main_plain_shapes.at(channel_axis); - auto new_format = main_format; - for (int i = 0; i < new_format.format_code_.norig_dims(); i++) { - auto blk_idxs = new_format.format_code_.collect_blocking_index(i); - if (i != channel_axis) { - for (auto &idx : blk_idxs) { - new_format.blocks_[idx] = 1; - } - } - } - auto new_const_op = - graph.make("constant", {}, {}, - {{"values", std::make_shared(scales)}, - {"dtype", datatypes::f32}, - {"plain_dims", new_plain_dims}, - {"format", new_format}}); - auto new_scale_op = graph.make( - "mul", {cur_op->get_inputs()[0], new_const_op->get_outputs()[0]}, - {}, cur_op->attrs_); - auto new_scale_fu = to_brg_fusible(new_scale_op); - new_scale_fu->alg_kind_ = brgemm::alg_kind_t::out_scales; - new_scale_fu->fuse_in_brgemm_ = true; - cur_op->replace_uses_with_and_remove(new_scale_op); - const_op->remove(); - return; - } - } - auto cur_op_fu = to_brg_fusible(cur_op); - cur_op_fu->alg_kind_ = brgemm::alg_kind_t::out_scales; - cur_op_fu->fuse_in_brgemm_ = true; - } -}; - -struct bias_state_t : public fusion_state_t { - bool is_acceptable(const sc_op_ptr &cur_op) override { - if (cur_op_count_ >= max_op_count_) { - return false; - } - if (!cur_op->isa()) { - return false; - } - auto add_op = cur_op->stc_cast(); - if (add_op->get_broadcast_input() == 0) { - return false; - } - GET_BROADCAST_TYPE(); - // support only_channel - if (bct_type != brg_broadcast_type::only_channel) { - return false; - } - // should be f32, bf16 or fp16 - if (!utils::is_one_of(cur_op->get_inputs()[0]->details_.dtype_, - datatypes::bf16, datatypes::f16, datatypes::f32)) { - return false; - } - cur_op_count_++; - add_op->alg_kind_ = brgemm::alg_kind_t::bias_add; - return true; - } - std::vector> next_possible_states( - const std::vector> &all_states) override { - std::vector> ret; - for (auto &st : all_states) { - if (st->isa() || st->isa()) { - ret.emplace_back(st); - } - } - return ret; - } - - // e.g. conv + cast + scale + bias => conv + bias + cast + scale - void do_transform(sc_graph_t &graph, const sc_op_ptr &main_op, - const sc_op_ptr &cur_op) override { - // s32 + cast + f32 - if (main_op->get_outputs()[0]->details_.dtype_.is_etype( - sc_data_etype::S32) && - cur_op->get_inputs()[0]->details_.dtype_.is_etype(sc_data_etype::F32)) { - COMPILE_ASSERT(main_op->attrs_.has_key("data_scales") && - main_op->attrs_.has_key("weight_scales"), - "Main op is not quantized but output s32."); - auto l_plain_dims = main_op->get_outputs()[0]->details_.get_plain_dims(); - auto r_plain_dims = cur_op->get_inputs()[1]->details_.get_plain_dims(); - auto &r_format = cur_op->get_inputs()[1]->details_.get_format(); - auto &fcode = r_format.format_code_; - int channel_axis = fcode.get(fcode.ndims() - 1); - sc_dim channel_size = r_plain_dims.at(channel_axis); - auto data_scales = main_op->attrs_.get>("data_scales"); - auto weight_scales = - main_op->attrs_.get>("weight_scales"); - auto output_scales = math_utils::vector_mul(data_scales, weight_scales); - auto next_op_by_main = main_op->get_outputs()[0]->uses_[0].second; - assert(main_op->get_outputs()[0]->uses_[0].first == 0); - assert(next_op_by_main.get() != cur_op.get()); - output_scales = math_utils::vector_rcp(output_scales); - if (output_scales.size() == 1) { - output_scales = std::vector(channel_size, output_scales[0]); - } - sc_dims const_plain_dims = r_plain_dims; - auto const_scales = graph.make( - "constant", {}, {}, - {{"values", std::make_shared(output_scales)}, - {"dtype", datatypes::f32}, - {"plain_dims", const_plain_dims}, - {"format", r_format}}); - auto bias_mul_scales = graph.make( - "mul", {cur_op->get_inputs()[1], const_scales->get_outputs()[0]}, {}, - {}); - auto castf32 = graph.make("cast", main_op->get_outputs(), {}, - {{"dtype", datatypes::f32}}); - auto bias_add = graph.make( - "add", {castf32->get_outputs()[0], bias_mul_scales->get_outputs()[0]}, - {}, cur_op->attrs_); - auto casts32 = graph.make("cast", bias_add->get_outputs(), {}, - {{"dtype", datatypes::s32}}); - to_brg_fusible(bias_add)->alg_kind_ = brgemm::alg_kind_t::bias_add; - to_brg_fusible(castf32)->fuse_in_brgemm_ = true; - to_brg_fusible(bias_add)->fuse_in_brgemm_ = true; - to_brg_fusible(casts32)->fuse_in_brgemm_ = true; - next_op_by_main->replace_input(0, casts32->get_outputs()[0]); - auto pre_op_by_cur = cur_op->get_inputs()[0]->producer_owner_; - auto uses = cur_op->get_outputs()[0]->uses_; - for (auto &use : uses) { - use.second->replace_input(use.first, pre_op_by_cur->get_outputs()[0]); - } - cur_op->remove(); - return; - } - auto cur_op_fu = to_brg_fusible(cur_op); - cur_op_fu->alg_kind_ = brgemm::alg_kind_t::bias_add; - cur_op_fu->fuse_in_brgemm_ = true; - } -}; - -struct elem_state_t : public fusion_state_t {}; -struct unary_elem_state_t : public elem_state_t { - unary_elem_state_t() { - max_op_count_ = brgemm::postops_setting_t::max_postops_num; - } - bool is_acceptable(const sc_op_ptr &cur_op) override { - if (cur_op_count_ >= max_op_count_) { - return false; - } - // cast cur_op should be processed by cast_state_t. - if (cur_op->isa()) { - return false; - } - if (!cur_op->isa()) { - return false; - } - cur_op_count_++; - return true; - } - - std::vector> next_possible_states( - const std::vector> &all_states) override { - std::vector> ret; - for (auto &st : all_states) { - if (st->isa() || st->isa()) { - ret.emplace_back(st); - } - } - return ret; - } -}; - -struct binary_elem_state_t : public elem_state_t { - bool is_acceptable(const sc_op_ptr &cur_op) override { - if (cur_op_count_ >= max_op_count_) { - return false; - } - if (!cur_op->isa()) { - return false; - } - auto bin_op = cur_op->stc_cast(); - if (bin_op->get_broadcast_input() == 0) { - return false; - } - GET_BROADCAST_TYPE(); - // support only_channel, non broadcast - if (bct_type == brg_broadcast_type::all_broadcast || - bct_type == brg_broadcast_type::non_channel) { - return false; - } - cur_op_count_++; - return true; - } - std::vector> next_possible_states( - const std::vector> &all_states) override { - std::vector> ret; - for (auto &st : all_states) { - if (st->isa() || st->isa()) { - ret.emplace_back(st); - } - } - return ret; - } -}; - -struct c_zp_state_t : public fusion_state_t { - bool is_acceptable(const sc_op_ptr &cur_op) override { - if (cur_op_count_ >= max_op_count_) { - return false; - } - if (!cur_op->isa()) { - return false; - } - auto add_op = cur_op->stc_cast(); - if (add_op->get_broadcast_input() == 0) { - return false; - } - GET_BROADCAST_TYPE(); - if (bct_type != brg_broadcast_type::all_broadcast) { - return false; - } - // should be f32 - if (!cur_op->get_inputs()[0]->details_.dtype_.is_etype( - sc_data_etype::F32) || - !cur_op->get_inputs()[1]->details_.dtype_.is_etype( - sc_data_etype::F32)) { - return false; - } - cur_op_count_++; - add_op->alg_kind_ = brgemm::alg_kind_t::c_zp; - return true; - } - std::vector> next_possible_states( - const std::vector> &all_states) override { - std::vector> ret; - for (auto &st : all_states) { - if (st->isa() || st->isa() || - st->isa()) { - ret.emplace_back(st); - } - } - return ret; - } -}; - -std::vector> -create_all_possible_states(bool include_zp = false, - bool include_binary = false) { - std::vector> ret; - if (include_zp) { - ret.emplace_back(std::make_shared()); - } - ret.emplace_back(std::make_shared()); - ret.emplace_back(std::make_shared()); - ret.emplace_back(std::make_shared()); - ret.emplace_back(std::make_shared()); - // onednn binary postop injector has redundant latency. - if (include_binary) { - ret.emplace_back(std::make_shared()); - } - if (include_zp) { - ret.emplace_back(std::make_shared()); - } - return ret; -} - -// Mark the alg kind of brgemm fusion ops and do brgemm-friendly -// transformation (like move bias closer to main op; use vector buffer for -// scales. It does not change calculation itself) -// As this transform pass is only used in `fuse_ops` pass, so we introduce two -// auxiliary ops -void brgemm_fusion_transform(sc_graph_t &graph, const context_ptr &ctx) { - if (graph.attrs_.get_or_else("temp.disable_graph_fusion", 0) == 1) { - return; - } - // todo: need interleave optimization on amx - if (ctx->machine_.cpu_flags_.fAVX512AMXTILE) { - return; - } - auto ops = graph.ops_; - for (auto &main_op : ops) { - if (!main_op->isa()) { - continue; - } - if (!main_op->is_single_output_single_use()) { - continue; - } - sc_op_ptr cur_op = main_op; - // todo : enable zp - bool include_zp = - graph.attrs_.get_or_else("temp.brgemm_fusion_include_zp", false); - // currently binary injector has slow implement. - bool include_binary = - graph.attrs_.get_or_else("temp.brgemm_fusion_include_binary", false); - static std::vector> all_possible_states = - create_all_possible_states(false); - static std::vector> - all_possible_states_include_zp = create_all_possible_states(true); - auto all_states = - include_zp ? all_possible_states_include_zp : all_possible_states; - if (include_binary) { - all_states = create_all_possible_states(include_zp, true); - } - // reset because of the static states - for (auto &st : all_states) { - st->reset(); - } - int fused_op_count = 0; - auto next_op = cur_op->get_outputs()[0]->uses_[0].second; - cur_op = next_op; - auto next_states = all_states; - while (fused_op_count < brgemm::postops_setting_t::max_postops_num && - !cur_op->isa()) { - // store next op as cur op may be transformed by new op - next_op = cur_op->get_outputs()[0]->uses_[0].second; - bool has_next = cur_op->is_single_output_single_use(); - bool accept = false; - for (auto &st : next_states) { - if (st->is_acceptable(cur_op)) { - st->do_transform(graph, main_op, cur_op); - next_states = st->next_possible_states(all_states); - fused_op_count++; - accept = true; - break; - } - } - if (!accept || !has_next) { - break; - } - cur_op = next_op; - } - } - graph.reset_op_ids(); -} -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/transform/broadcast_transform.cpp b/legacy/core/src/compiler/ir/graph/transform/broadcast_transform.cpp deleted file mode 100644 index eaf590f1e..000000000 --- a/legacy/core/src/compiler/ir/graph/transform/broadcast_transform.cpp +++ /dev/null @@ -1,103 +0,0 @@ -/******************************************************************************* - * Copyright 2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#include -#include - -#include "../fusible_op.hpp" -#include "../visitor.hpp" -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -/* -Insert broadcast after the input with largest size -*/ -static size_t get_input_idx_to_broadcast(const sc_op_ptr &node) { - // TODO(yifei): upgrade the logic here - auto inputs = node->get_inputs(); - std::vector input_sizes(inputs.size()); - std::transform(inputs.begin(), inputs.end(), input_sizes.begin(), - [](const graph_tensor_ptr >) -> int64_t { - auto lt = gt->details_; - if (lt.is_dynamic()) { - return 0; - } else { - return math_utils::get_dims_product( - lt.get_blocking_dims()) * - static_cast( - utils::get_sizeof_etype(lt.dtype_.type_code_)); - } - }); - return std::max_element(input_sizes.begin(), input_sizes.end()) - - input_sizes.begin(); -} - -/* -Applies to multi-directional broadcast scenario -One side broadcast to the output shape -TODO(yifei): decide broadcast insertion rule - - Which side to choose -(x,1,z,1)[v0] [v1](y,z,w) - \ / - add - | - (x,y,z,w)[v3] -=============== - [v1](y,z,w) - | - broadcast - | -(x,1,z,1)[v0] [v4](x,y,z,w) - \ / - add - | - (x,y,z,w)[v3] -*/ -void broadcast_transform(sc_graph_t &graph, const context_ptr &ctx) { - op_visitor_t vis = op_visitor_t::dfs_topology_sort(); - vis.visit_graph(graph, [&](op_visitor_t *vis, const sc_op_ptr &node) { - if (auto broadcastable_node = - node->dyn_cast()) { - if (broadcastable_node->get_non_broadcast_input_index(false).empty()) { - auto inputs = node->info_.inputs_; - auto &out_shape = node->info_.outputs_[0]->details_.get_plain_dims(); - auto broadcast_input_idx = get_input_idx_to_broadcast(node); - const auto &bc_axis = broadcastable_node->get_plain_bc_axis(); - auto broadcast = - graph.make("broadcast", {inputs[broadcast_input_idx]}, {}, - {{"output_shape", out_shape}, - {"bc_axis", bc_axis[broadcast_input_idx]}}); - std::vector new_inputs = inputs; - new_inputs[broadcast_input_idx] = broadcast->get_outputs()[0]; - auto new_broadcastable = graph.make(node->op_name_, new_inputs, {}, {}); - node->replace_uses_with_and_remove(new_broadcastable); - vis->update_state_for_visited(broadcast); - vis->update_state_for_visited(new_broadcastable); - } - } - }); - graph.reset_op_ids(); -} -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/transform/constant_optimize.cpp b/legacy/core/src/compiler/ir/graph/transform/constant_optimize.cpp deleted file mode 100644 index bfa632833..000000000 --- a/legacy/core/src/compiler/ir/graph/transform/constant_optimize.cpp +++ /dev/null @@ -1,44 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include - -#include "../visitor.hpp" -#include "transform.hpp" -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -void constant_optimization(sc_graph_t &graph, const context_ptr &ctx) { - auto vis = op_visitor_t::dfs_topology_sort(graph.ops_.size()); - vis.visit_graph(graph, [&](op_visitor_t *vis, const sc_op_ptr &node) { - if (auto graph_node = node->dyn_cast()) { - auto ret = graph_node->constant_optimize(graph); - if (ret) { - vis->update_state_for_visited(ret); - } - } - }); - graph.reset_op_ids(); -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/transform/div_bcast_transform.cpp b/legacy/core/src/compiler/ir/graph/transform/div_bcast_transform.cpp deleted file mode 100644 index c46104972..000000000 --- a/legacy/core/src/compiler/ir/graph/transform/div_bcast_transform.cpp +++ /dev/null @@ -1,86 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "../fusible_op.hpp" -#include "../graph.hpp" -#include "../visitor.hpp" -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -/* -[v0] [v1] <--broadcast tensor - \ / - div - | - [b1] -=============== - [v1] - | - reciprocal - | -[v0] [v2] <--broadcast tensor - \ / - mul - | - [b1] -*/ - -sc_op_ptr insert_rcp(sc_graph_t &graph, div_op_t *node) { - auto dtype = node->get_inputs()[1]->details_.dtype_; - if (node->attrs_.get_or_else(op_attr_key::must_div, false)) { - return nullptr; - } - if (dtype.type_code_ == sc_data_etype::F32 || - dtype.type_code_ == sc_data_etype::BF16) { - // next node uses div result - graph_tensor_ptr v0 = node->get_inputs()[0]; - graph_tensor_ptr v1 = node->get_inputs()[1]; - auto v2 = graph.make("reciprocal", {v1}, {}, {})->get_outputs()[0]; - auto new_node = graph.make("mul", {v0, v2}, {}, node->attrs_); - node->replace_uses_with_and_remove(new_node); - return new_node; - } - return nullptr; -} - -void div_bcast_transform(sc_graph_t &graph, const context_ptr &ctx) { - op_visitor_t vis = op_visitor_t::bfs(); - vis.visit_graph(graph, [&graph](op_visitor_t *vis, const sc_op_ptr &node) { - if (auto div_node = node->dyn_cast()) { - if (div_node->attrs_.get_or_else(op_attr_key::must_div, false)) { - return; - } - auto bcast_idx = div_node->get_broadcast_input(); - if (bcast_idx == 1) { - assert(div_node->get_inputs().size() == 2); - auto inserted_op = insert_rcp(graph, div_node); - if (inserted_op) { - vis->update_state_for_visited(inserted_op); - } - } - } - }); - graph.reset_op_ids(); -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/transform/dynamic_graph_transform.cpp b/legacy/core/src/compiler/ir/graph/transform/dynamic_graph_transform.cpp deleted file mode 100644 index 9d09261c4..000000000 --- a/legacy/core/src/compiler/ir/graph/transform/dynamic_graph_transform.cpp +++ /dev/null @@ -1,97 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "../fusible_op.hpp" -#include "../graph_op.hpp" -#include "../pass/pass.hpp" -#include "../quantization/quantize_op.hpp" -#include "../tunable_op.hpp" -#include "../visitor.hpp" -#include "transform.hpp" -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -/// Push tensor_view/transpose op back to quantize/dequantize ops for better -/// fusion in dynamic cases. -void tensor_view_push_back(sc_graph_t &graph, const context_ptr &ctx) { - constexpr const int max_try_times = 10; - for (int i = 0; i < max_try_times; i++) { - bool changed = false; - auto vis = op_visitor_t::bfs_unchecked(); - vis.visit_graph(graph, [&](op_visitor_t *vis, const sc_op_ptr &node) { - if (node->isa() || node->isa() || - node->isa()) { - auto cur_node = node; - auto details = node->get_inputs()[0]->details_; - while (cur_node->is_single_output_single_use()) { - auto next_node = cur_node->get_outputs()[0]->uses_[0].second; - int use_idx = cur_node->get_outputs()[0]->uses_[0].first; - if (!(next_node->isa() || - next_node->isa())) { - break; - } - if (cur_node == node) { - next_node->replace_input(use_idx, cur_node->get_inputs()[0]); - } - cur_node = next_node; - details.dtype_ = cur_node->get_inputs()[0]->details_.dtype_; - cur_node->get_inputs()[0]->details_ = details; - details.dtype_ = cur_node->get_outputs()[0]->details_.dtype_; - cur_node->get_outputs()[0]->details_ = details; - } - // if changed - if (cur_node != node) { - changed = true; - // cache uses. - auto uses = cur_node->get_outputs()[0]->uses_; - node->replace_input(0, cur_node->get_outputs()[0]); - // correct datatype as we could meet a cast op. - auto out_tsr = node->get_outputs()[0]; - out_tsr->details_.dtype_ = - cur_node->get_outputs()[0]->details_.dtype_; - for (auto &use : uses) { - use.second->replace_input(use.first, out_tsr); - } - } - } - vis->update_state_for_visited(node); - }); - if (!changed) { - break; - } - } - graph.reset_op_ids(); -} - -void dynamic_graph_transform(sc_graph_t &graph, const context_ptr &ctx) { - if (!graph.is_dynamic()) { - return; - } - tensor_view_push_back(graph, ctx); -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/transform/elemwise_bcast_swap.cpp b/legacy/core/src/compiler/ir/graph/transform/elemwise_bcast_swap.cpp deleted file mode 100644 index 4f9c01210..000000000 --- a/legacy/core/src/compiler/ir/graph/transform/elemwise_bcast_swap.cpp +++ /dev/null @@ -1,101 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "../fusible_op.hpp" -#include "../visitor.hpp" -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -/* -[v0] [v1] - \ / - add - | - [v3] [v2] <--broadcast tensor - \ / - bcast - | - [b1] -=============== -[v0] [v2] <--broadcast tensor - \ / - bcast - | - [v3] [v1] - \ / - add - | - [b1] -*/ -static bool do_swap(sc_graph_t &mgr, sc_op *bcast, sc_op *other, - int bcast_tsr_idx, op_visitor_t *visitor) { - assert(bcast->get_inputs().size() == 2 && other->get_inputs().size() == 2); - assert(bcast->get_outputs().size() == 1 && other->get_outputs().size() == 1); - graph_tensor_ptr v2 = bcast->get_inputs()[bcast_tsr_idx]; - graph_tensor_ptr v0 = other->get_inputs()[0]; - graph_tensor_ptr v1 = other->get_inputs()[1]; - graph_tensor_ptr b1 = bcast->get_outputs()[0]; - graph_tensor_ptr v3 = other->get_outputs()[0]; - if (v3->uses_.size() > 1) { - // v3 has multiple uses, cannot swap - return false; - } - - int idx_of_other_in_bcast = false ? 1 : (bcast_tsr_idx == 1); - auto new_bcast = mgr.make(bcast->op_name_, {v0, v2}, {}, bcast->attrs_); - auto new_other = mgr.make(other->op_name_, {new_bcast->get_outputs()[0], v1}, - {}, other->attrs_); - bcast->replace_uses_with_and_remove(new_other); - other->remove(); - visitor->update_state_for_visited(new_other); - return true; -} - -static bool check_and_swap(sc_graph_t &mgr, add_op_t *bcast_node, - int index_in_bcast, int bcast_input_idx, - op_visitor_t *visitor) { - auto parent = bcast_node->get_inputs()[index_in_bcast]->producer_owner_; - if (auto parent_add = parent->dyn_cast()) { - if (parent_add->get_broadcast_input() == -1) { - return do_swap(mgr, bcast_node, parent_add, bcast_input_idx, visitor); - } - } - return false; -} - -void elemwise_bcast_swap(sc_graph_t &mgr, const context_ptr &ctx) { - op_visitor_t vis = op_visitor_t::dfs_topology_sort(); - vis.visit_graph(mgr, [&mgr](op_visitor_t *vis, const sc_op_ptr &node) { - if (auto add_node = node->dyn_cast()) { - auto non_bcast_idx = add_node->get_non_broadcast_input_index(false); - if (non_bcast_idx.size() == 1) { - assert(add_node->get_inputs().size() == 2); - int bcast_idx = 1 - non_bcast_idx[0]; - if (!check_and_swap(mgr, add_node, 0, bcast_idx, vis)) { - check_and_swap(mgr, add_node, 1, bcast_idx, vis); - } - } - } - }); -} -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/transform/elemwise_dimension_alignment.cpp b/legacy/core/src/compiler/ir/graph/transform/elemwise_dimension_alignment.cpp deleted file mode 100644 index f77c2037a..000000000 --- a/legacy/core/src/compiler/ir/graph/transform/elemwise_dimension_alignment.cpp +++ /dev/null @@ -1,203 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#include - -#include "../fusible_op.hpp" -#include "../visitor.hpp" -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -static void infer_aligned_shape(const logical_tensor_t &a, - const logical_tensor_t &b, - const std::vector &plain_bc_axis, - sc_dims &aligned_shape, - std::vector &aligned_axis, - sc_data_format_t &new_format) { - assert(a.get_plain_dims().size() > b.get_plain_dims().size()); - // skip the case where b's shape == {1} - if (plain_bc_axis == std::vector{-1}) { - aligned_shape = sc_dims{}; - aligned_axis = {}; - new_format = sc_data_format_t(); - return; - } - - int dim_difference = a.get_plain_dims().size() - b.get_plain_dims().size(); - sc_data_format_t b_format = b.get_format(); - if (b_format.is_any()) { - b_format = sc_data_format_t::get_plain_by_dims(b.get_plain_dims().size()); - } - // get dimension binding relation of b to plain_bc_axis index - std::unordered_map bc_axis_bind; - int bc_axis_idx = 0; - for (size_t i = 0; i < b.get_plain_dims().size(); ++i) { - if (b.get_plain_dims()[i] == 1 && - a.get_plain_dims()[plain_bc_axis[bc_axis_idx]] != 1) { - continue; - } else if (b.get_plain_dims()[i] != 1 && - a.get_plain_dims()[plain_bc_axis[bc_axis_idx]] == 1) { - COMPILE_ASSERT(0, "Invalid bc_axis found for broadcastable ops."); - } else { - COMPILE_ASSERT( - is_dynamic_dim(b.get_plain_dims()[i]) || - is_dynamic_dim(a.get_plain_dims()[plain_bc_axis[bc_axis_idx]]) || - b.get_plain_dims()[i] == - a.get_plain_dims()[plain_bc_axis[bc_axis_idx]], - "Invalid bc_axis found for broadcastable ops."); - bc_axis_bind[i] = bc_axis_idx; - bc_axis_idx++; - if (bc_axis_idx >= static_cast(plain_bc_axis.size())) { - break; - } - } - } - COMPILE_ASSERT(bc_axis_idx == static_cast(plain_bc_axis.size()), - "Invalid bc_axis found for broadcastable ops."); - // b's new shape --> extend to a's number of dimension - aligned_shape.resize(a.get_plain_dims().size(), 1); - // start infer extended shape - for (const auto &bind : bc_axis_bind) { - aligned_shape[plain_bc_axis[bind.second]] = b.get_plain_dims()[bind.first]; - } - for (size_t i = 0; i < a.get_plain_dims().size(); ++i) { - if (std::find(plain_bc_axis.begin(), plain_bc_axis.end(), (int)i) == - plain_bc_axis.end()) { - aligned_axis.push_back(i); - } - } - - // start infer extended format - // the logic below wish to let blocking shape having extended dims - // as its leading dims (having limitation on batch format) - // e.g. - // plain_shape: [3, 5] --> [1, 3, 1, 5] - // blocking_shape: [5, 3] --> [1, 1, 5, 3] - // format: BA --> xxDB --> ACDB - // plain_shape: [3, 1, 5] --> [1, 3, 1, 5] ({1, 3}) - // format: ABC --> xBxD --> ABCD - // format: CAB ([5, 3, 1])--> xDBx --> ADBC --> [1, 5, 3, 1] - std::vector storage_args(b_format.format_code_.ndims() + dim_difference, - -1); - std::unordered_set axis; - for (int i = 0; i < b_format.format_code_.ndims(); ++i) { - int original_axis = b_format.format_code_.get(i); - // original_axis is the axis of the b's plain_dims - // if plain_dims[original_axis] - // the issue is we don't know whether original_axis is - if (bc_axis_bind.find(original_axis) != bc_axis_bind.end()) { - storage_args[i + dim_difference] = - plain_bc_axis[bc_axis_bind[original_axis]]; - axis.insert(storage_args[i + dim_difference]); - } else { - COMPILE_ASSERT(b.get_plain_dims()[original_axis] == 1, - "Axis not found in bc_axis_bind map's corresponding dim " - "must be 1."); - } - } - - int sequential_fill_up = 0; - for (size_t i = 0; i < storage_args.size(); ++i) { - if (storage_args[i] == -1) { - while (axis.find(sequential_fill_up) != axis.end()) { - sequential_fill_up++; - } - storage_args[i] = sequential_fill_up++; - } - } - new_format = sc_data_format_t(storage_args, b_format.blocks_); -} - -/* -(x,y,z,w)[v0] [v1](y,w) <-- shorter side - \ / - add - | - (x,y,z,w)[v3] -=============== - [v1](y,w) <-- shorter side - | - tensor_view - | -(x,y,z,w)[v0] [v4](1,y,1,w) - \ / - add - | - (x,y,z,w)[v3] -*/ -void elemwise_dimension_alignment(sc_graph_t &graph, const context_ptr &ctx) { - op_visitor_t vis = op_visitor_t::dfs_topology_sort(); - vis.visit_graph(graph, [&](op_visitor_t *vis, const sc_op_ptr &node) { - if (auto may_broadcast_node = - node->dyn_cast()) { - COMPILE_ASSERT( - !may_broadcast_node->get_non_broadcast_input_index(false).empty(), - "elemwise_dimension_alignment requires the broadcast-able " - "op to have at least 1 non-broadcast input."); - for (size_t i = 0; i < node->get_inputs().size(); ++i) { - auto cur_in_lt = node->get_inputs()[i]->details_; - auto cur_out_lt = node->get_outputs()[0]->details_; - if (cur_in_lt.get_plain_dims().size() < - cur_out_lt.get_plain_dims().size()) { - const auto &plain_bc_axis = - may_broadcast_node->get_plain_bc_axis()[i]; - sc_dims shape; - std::vector aligned_axis; - sc_data_format_t format; - infer_aligned_shape(cur_out_lt, cur_in_lt, plain_bc_axis, shape, - aligned_axis, format); - if (!shape.empty()) { - // insert tensor view - auto ret = graph.make("tensor_view", {node->info_.inputs_[i]}, {}, - {{"shape", shape}, - {"format", format}, - {"expand_dim", aligned_axis}}); - node->replace_input(i, ret->get_outputs()[0]); - } - } - } - } else if (auto broadcast_op = node->dyn_cast()) { - auto cur_in_lt = node->get_inputs()[0]->details_; - auto cur_out_lt = node->get_outputs()[0]->details_; - if (cur_in_lt.get_plain_dims().size() < - cur_out_lt.get_plain_dims().size()) { - const auto &plain_bc_axis = broadcast_op->get_plain_bc_axis(); - sc_dims shape; - std::vector aligned_axis; - sc_data_format_t format; - infer_aligned_shape(cur_out_lt, cur_in_lt, plain_bc_axis, shape, - aligned_axis, format); - if (!shape.empty()) { - // insert tensor view - auto ret = graph.make("tensor_view", {node->info_.inputs_[0]}, {}, - {{"shape", shape}, - {"format", format}, - {"expand_dim", aligned_axis}}); - node->replace_input(0, ret->get_outputs()[0]); - } - } - } - }); - graph.reset_op_ids(); -} -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/transform/eliminate_zero_shaped_tensors.cpp b/legacy/core/src/compiler/ir/graph/transform/eliminate_zero_shaped_tensors.cpp deleted file mode 100644 index fefd897e7..000000000 --- a/legacy/core/src/compiler/ir/graph/transform/eliminate_zero_shaped_tensors.cpp +++ /dev/null @@ -1,151 +0,0 @@ -/******************************************************************************* - * Copyright 2023-2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "../graph.hpp" -#include "../visitor.hpp" -#include "transform.hpp" -#include -#include -#include -#include -#include -#include -#include - -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -SC_MODULE(graph.eliminate_zero_shaped_tensors); - -static bool is_tensor_zero_shape(const graph_tensor_ptr >) { - const sc_dims dims = gt->details_.get_plain_dims(); - if (std::all_of(dims.begin(), dims.end(), [](sc_dim d) { return d != 0; })) { - return false; - } - return true; -} - -/* -It's possible that the shape of some tensors contains zeros, for example, (4, -32, 0, 128). These tensors are actually empty in size. We need to eliminate them -after infering shape and before other passes. -*/ - -SC_INTERNAL_API void eliminate_zero_shaped_tensors(sc_graph_t &graph, - const context_ptr &ctx) { - op_visitor_t vis = op_visitor_t::bfs(); - vis.visit_graph(graph, [&](op_visitor_t *vis, const sc_op_ptr &op) { - std::vector is_zero_shape(op->info_.inputs_.size(), false); - for (size_t idx = 0; idx < op->info_.inputs_.size(); ++idx) { - if (is_tensor_zero_shape(op->info_.inputs_[idx])) { - is_zero_shape[idx] = true; - } - } - if (std::all_of(is_zero_shape.begin(), is_zero_shape.end(), - [](bool is_zero) { return !is_zero; })) { - return; - } - if (op->isa() || op->isa()) { - graph_tensor_ptr input_left = nullptr; - for (auto in_tsr_itr = op->info_.inputs_.begin(); - in_tsr_itr != op->info_.inputs_.end();) { - auto &in_tsr = *in_tsr_itr; - if (!is_tensor_zero_shape(in_tsr)) { - ++in_tsr_itr; - continue; - } - if (op->info_.inputs_.size() == 1) { - // we have 1 tensor left and it is a zero-shape tensor - input_left = in_tsr; - break; - } - SC_MODULE_INFO << "Disconnect input from concat or output op."; - in_tsr->detach_use(op); - // Since we will delete this input, the inputs after it should - // modify their uses_ by decrease index by 1 - for (auto latter_itr = in_tsr_itr + 1; - latter_itr != op->info_.inputs_.end(); ++latter_itr) { - auto &latter_uses = (*latter_itr)->uses_; - for (auto &idx_op : latter_uses) { - idx_op.first -= 1; - } - } - in_tsr_itr = op->info_.inputs_.erase(in_tsr_itr); - } - if (input_left) { - vis->update_state_for_visited(op); - for (auto &out_tsr : op->get_outputs()) { - auto uses = out_tsr->uses_; - for (auto &idx_op : uses) { - idx_op.second->replace_input(idx_op.first, input_left); - } - } - op->remove(); - } - } else if (op->isa() || op->isa()) { - COMPILE_ASSERT(0, "padding and shape op do not support zero-shape tensor " - "input."); - } else { - vis->update_state_for_visited(op); - if (!op->isa() && - !op->isa() && !op->isa()) { - COMPILE_ASSERT(is_zero_shape[0], - "Current op's first input must be a zero tensor."); - } - int64_t input_idx_left = -1; - for (size_t i = 0; i < is_zero_shape.size(); ++i) { - if (is_zero_shape[i]) { - input_idx_left = i; - break; - } - } - COMPILE_ASSERT(input_idx_left >= 0, - "The current op must have a zero-shape input."); - int64_t visiting_idx = 0; - for (auto in_tsr_itr = op->info_.inputs_.begin(); - in_tsr_itr != op->info_.inputs_.end();) { - if (visiting_idx == input_idx_left) { - in_tsr_itr++; - } else { - (*in_tsr_itr)->detach_use(op); - in_tsr_itr = op->info_.inputs_.erase(in_tsr_itr); - } - visiting_idx++; - } - COMPILE_ASSERT(op->info_.inputs_.size() == 1, - "The zero-input op must have only 1 input left."); - auto &input_left = op->info_.inputs_[0]; - // replace_input may change the uses, we need to copy it - for (const auto &output : op->info_.outputs_) { - auto uses = output->uses_; - for (auto &cld : uses) { - cld.second->replace_input(cld.first, input_left); - } - } - op->remove(); - } - }); - graph.reset_op_ids(); -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/transform/flatten_conv.cpp b/legacy/core/src/compiler/ir/graph/transform/flatten_conv.cpp deleted file mode 100644 index 713715b77..000000000 --- a/legacy/core/src/compiler/ir/graph/transform/flatten_conv.cpp +++ /dev/null @@ -1,225 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "../fusible_op.hpp" -#include "../pass/pass.hpp" -#include "../visitor.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -sc_dims get_conv1d_flatten_shape(const sc_data_format_t &format, - const sc_dims &origin_shape, - int merge_bs = 1) { - COMPILE_ASSERT(origin_shape.size() == 4, - "Conv1d flatten only support 2d case"); - sc_dims out_shape; - out_shape.push_back(origin_shape[0]); - if (format == sc_data_format_t::NCHW()) { - out_shape.push_back(origin_shape[1]); - out_shape.push_back(origin_shape[2] * origin_shape[3]); - } else if (format == sc_data_format_t::NHWC()) { - out_shape.push_back(origin_shape[3]); - out_shape.push_back(origin_shape[1] * origin_shape[2]); - } - COMPILE_ASSERT(out_shape[0] % merge_bs == 0, - "N % merge_bs should be equal to zero but get " - << out_shape[0] << " % " << merge_bs << " = " - << out_shape[0] % merge_bs); - out_shape[2] *= merge_bs; - out_shape[0] /= merge_bs; - - return out_shape; -} - -// Whether to flatten N axis, which will break bs fusion but may bring perf -// benefit to single layer -int get_minibatch(const int &bs, const int &min_os) { - auto num_threads = runtime_config_t::get().get_num_threads(); - int minibatch = std::max(sc_dim(1), sc_dim(28) / sc_dim(std::sqrt(min_os))); - if ((bs / minibatch % num_threads != 0 && bs / minibatch < 4 * num_threads)) { - // TODO(zhicong): use a more general way for minibatch image affinity - if (bs % num_threads == 0) { - return 1; - } else { - return bs; - } - } - return bs % minibatch == 0 ? minibatch : 1; -} - -int minimum_spatial_shape(sc_graph_t &graph, bool &is_support) { - auto vis = op_visitor_t::bfs(); - int min_spatial = std::numeric_limits::max(); - vis.visit_graph(graph, [&](op_visitor_t *vis, const sc_op_ptr &node) { - if (auto op = node->dyn_cast()) { - int spatial_size = 1; - auto plain_shape = op->get_inputs()[0]->details_.get_plain_dims(); - if (plain_shape.size() != 4) { - is_support = false; - } - - for (auto i = 2UL; i < plain_shape.size(); i++) { - spatial_size *= plain_shape[i]; - } - if (spatial_size < min_spatial) { - min_spatial = spatial_size; - } - } - }); - return min_spatial; -} - -void conv1d_flatten(sc_graph_t &graph, const context_ptr &ctx) { - auto vis = op_visitor_t::bfs(); - bool is_support = true; - auto minimum_os = minimum_spatial_shape(graph, is_support); - if (!is_support) - return; - vis.visit_graph(graph, [&](op_visitor_t *vis, const sc_op_ptr &node) { - if (auto op = node->dyn_cast()) { - auto data_plain_shape = op->get_inputs()[0]->details_.get_plain_dims(); - // do not support dynamic shape - if (op->get_inputs()[0]->is_dynamic()) - return; - auto data_origin_shape = - op->get_inputs()[0]->details_.get_blocking_dims(); - auto weight_origin_shape = - op->get_inputs()[1]->details_.get_blocking_dims(); - auto ndims = data_plain_shape.size(); - auto &stride = op->attrs_.get("strides"); - auto &pads_begin = op->attrs_.has_key("pads_begin") - ? op->attrs_.get("pads_begin") - : op->attrs_.get("paddings"); - if (op->attrs_.get_or_else("use_rl", ops::rl_kind::NO_LOWERING) != - ops::rl_kind::NO_LOWERING) { - return; - } - sc_dim groups = op->attrs_.get_or_else("groups", 1); - if (groups > 1) { - return; - } - auto weight_plain_dims = op->get_inputs()[1]->details_.get_plain_dims(); - assert(weight_plain_dims.size() == ndims && "invalid weight_plain_dims"); - auto kh = weight_plain_dims[ndims - 2]; - auto kw = weight_plain_dims[ndims - 1]; - auto data_format = op->get_inputs()[0]->details_.get_format(); - auto weight_format = op->get_inputs()[1]->details_.get_format(); - int merge_bs = get_minibatch(data_plain_shape[0], minimum_os); - auto iw = op->get_inputs()[0]->details_.get_plain_dims()[3]; - auto ih = op->get_inputs()[0]->details_.get_plain_dims()[2]; - auto ow = op->get_outputs()[0]->details_.get_plain_dims()[3]; - auto oh = op->get_outputs()[0]->details_.get_plain_dims()[2]; - if (op->use_conv1d(ctx)) { - { // pre tensor_view(data) - auto shape = get_conv1d_flatten_shape( - sc_data_format_t::NCHW(), - sc_data_format_t::get_padded_plain_shapes(data_origin_shape, - data_format), - merge_bs); - auto reorder_op = graph.make( - "reorder", {op->get_inputs()[0]}, {}, - {{"out_format", sc_data_format_t::NHWC()}, {"internal", true}}); - auto view = graph.make("tensor_view", reorder_op->get_outputs(), {}, - {{"shape", shape}, - {"format", sc_data_format_t::NSC()}, - {"expand_dim", std::vector{}}}); - op->replace_input(0, view->get_outputs()[0]); - vis->update_state_for_visited(reorder_op); - vis->update_state_for_visited(view); - } - { // pre tensor_view(weight) - auto shape = - get_conv1d_flatten_shape(weight_format, weight_origin_shape); - auto reorder_op = graph.make( - "reorder", {op->get_inputs()[1]}, {}, - {{"out_format", sc_data_format_t::KCRS()}, {"internal", true}}); - auto view = graph.make("tensor_view", reorder_op->get_outputs(), {}, - {{"shape", shape}, - {"format", sc_data_format_t::KCS()}, - {"expand_dim", std::vector{}}}); - op->replace_input(1, view->get_outputs()[0]); - vis->update_state_for_visited(reorder_op); - vis->update_state_for_visited(view); - } - { // post tensor view(output) - auto origin_out = op->get_outputs()[0]->copy(); - op->get_outputs()[0]->replace_with(origin_out); - origin_out->producer_owner_ = nullptr; - - auto dtype = origin_out->details_.dtype_; - auto new_conv_out = std::make_shared( - &(*node), sc_data_format_t::NCS(), - get_conv1d_flatten_shape( - sc_data_format_t::NCHW(), - op->get_outputs()[0]->details_.get_blocking_dims(), merge_bs), - dtype); - op->info_.outputs_[0] = new_conv_out; - - auto reorder_op = - graph.make("reorder", op->get_outputs(), {}, - {{"out_format", sc_data_format_t::NSC()}}); - auto view = - graph.make("tensor_view", reorder_op->get_outputs(), {}, - {{"shape", origin_out->details_.get_plain_dims()}, - {"format", sc_data_format_t::NHWC()}, - {"expand_dim", std::vector{}}, - {"push_back", true}}); - origin_out->replace_with(view->get_outputs()[0]); - view->copy_dispatch_key_set_from_op(node); - vis->update_state_for_visited(reorder_op); - vis->update_state_for_visited(view); - } - if (op->attrs_.has_key("pads_begin")) { - op->attrs_.get("pads_begin") = {0}; - op->attrs_.get("pads_end") = {0}; - } else { - op->attrs_.get("paddings") = {0}; - } - op->attrs_["origin_ih"] = ih; - op->attrs_["origin_iw"] = iw; - op->attrs_["origin_ow"] = ow; - op->attrs_["origin_oh"] = oh; - } - } - }); - graph.reset_op_ids(); -} - -void flatten_conv(sc_graph_t &graph, const context_ptr &ctx) { - if (graph.attrs_.get_or_else("no_conv1d", false)) { - return; - } - conv1d_flatten(graph, ctx); -} -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/transform/fpmath_mode.cpp b/legacy/core/src/compiler/ir/graph/transform/fpmath_mode.cpp deleted file mode 100644 index 03057d4e7..000000000 --- a/legacy/core/src/compiler/ir/graph/transform/fpmath_mode.cpp +++ /dev/null @@ -1,156 +0,0 @@ -/******************************************************************************* - * Copyright 2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "transform.hpp" -#include -#include -#include -#include -#include -#include - -SC_MODULE(fpmath_mode) - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -bool down_converted_op_exit( - const context_ptr &ctx, const sc_op_ptr &node, - const sc_data_type_t &target_type, - std::vector &onedim_const) { // whether the op support - // converting to the target type - if (node->isa()) - return false; - if (node->isa()) - return false; - // avoid cast epsilon to bf16 - if (node->isa()) { - auto const_op = node->dyn_cast(); - if (const_op->get_constant_plain_dims() == sc_dims{1}) { - onedim_const.push_back(node->logical_op_id_); - return false; - } - }; - // check inputs data types. - for (auto &in : node->get_inputs()) { - if (std::find(onedim_const.begin(), onedim_const.end(), - in->producer_owner_->logical_op_id_) != onedim_const.end()) - return false; - if (!utils::is_one_of(in->details_.dtype_, datatypes::f32, target_type)) - return false; - } - // checkout outputs data types. - for (auto &out : node->get_outputs()) { - if (!utils::is_one_of(out->details_.dtype_, datatypes::f32, target_type)) - return false; - } - return true; -} - -bool check_is_quantized(sc_graph_t &graph) { - for (auto &op : graph.ops_) { - if (op->op_name_.find("quantize") != std::string::npos) - return true; - } - return false; -} - -void fpmath_mode(sc_graph_t &graph, const context_ptr &ctx) { - // checks for open fpmath_mode - int fpmath_mode_attr = - graph.attrs_.get_or_else(sc_graph_t::attr_key_t::fpmath_mode, 0); - int fpmath_mode_env = utils::getenv_int("SC_FPMATH_MODE", 0); - if (!fpmath_mode_attr && !fpmath_mode_env) - return; - if (!ctx->machine_.cpu_flags_.fAVX512AMXBF16 && - !ctx->machine_.cpu_flags_.fAVX512BF16) - return; - if (check_is_quantized(graph)) - return; - auto vis = op_visitor_t::bfs_topology_sort(graph.ops_.size()); - std::unordered_map dtype_backup; - std::vector onedim_const; - vis.visit_graph(graph, [&](op_visitor_t *vis, const sc_op_ptr &node) { - if (node->isa()) { - bool downconvert_output = - dtype_backup.count( - node->get_inputs()[0]->producer_owner_->logical_op_id_) > 0; - if (downconvert_output) { - // add the cast op: BF16 -> F32 - sc_op_ptr cast_node = graph.make("cast", node->get_inputs(), {}, - {{"dtype", datatypes::f32}}); - node->replace_input(0, cast_node->get_outputs()[0]); - } - } else { - if (down_converted_op_exit(ctx, node, datatypes::bf16, onedim_const)) { - for (auto in_id = 0; in_id < int(node->get_inputs().size()); ++in_id) { - if (node->get_inputs()[in_id]->details_.dtype_ == datatypes::f32) { - // add the cast op: F32 -> BF16 - auto cur_owner_op = node->get_inputs()[in_id]->producer_owner_; - auto is_constant_op = (cur_owner_op->isa() || - cur_owner_op->attrs_.get_or_else( - "constant", const_kind::not_const)); - sc_op_ptr cast_node = - graph.make("cast", {node->get_inputs()[in_id]}, {}, - {{"dtype", datatypes::bf16}}); - cast_node->get_outputs()[0]->details_.dtype_ = datatypes::bf16; - if (is_constant_op) { - cast_node->attrs_.set("constant", const_kind::local_const); - } - node->replace_input(in_id, cast_node->get_outputs()[0]); - } - } - // tunable ops can infer output type automatically, do not - // need to set out_dtype - if (!node->get_outputs().empty() && !node->isa()) { - if (node->get_outputs()[0]->details_.dtype_ == datatypes::f32) { - dtype_backup[node->logical_op_id_] = - node->get_outputs()[0]->details_.dtype_; - for (auto &out : node->get_outputs()) { - out->details_.dtype_ = datatypes::bf16; - } - } - } - } else if (!node->isa() && !node->isa()) { - for (auto in_id = 0; in_id < int(node->get_inputs().size()); ++in_id) { - if (node->get_inputs()[in_id]->details_.dtype_ == datatypes::bf16) { - // add the cast op: BF16 -> F32 - auto cur_owner_op = node->get_inputs()[in_id]->producer_owner_; - auto is_constant_op = (cur_owner_op->isa() || - cur_owner_op->attrs_.get_or_else( - "constant", const_kind::not_const)); - sc_op_ptr cast_node = - graph.make("cast", {node->get_inputs()[in_id]}, {}, - {{"dtype", datatypes::f32}}); - cast_node->get_outputs()[0]->details_.dtype_ = datatypes::f32; - if (is_constant_op) { - cast_node->attrs_.set("constant", const_kind::local_const); - } - node->replace_input(in_id, cast_node->get_outputs()[0]); - } - } - } - } - }); - graph.reset_op_ids(); -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/transform/global_reschedule.cpp b/legacy/core/src/compiler/ir/graph/transform/global_reschedule.cpp deleted file mode 100644 index 829fcaef8..000000000 --- a/legacy/core/src/compiler/ir/graph/transform/global_reschedule.cpp +++ /dev/null @@ -1,358 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "../fusible_op.hpp" -#include "../graph_op.hpp" -#include "../pass/pass.hpp" -#include "../tunable_op.hpp" -#include "../visitor.hpp" -#include "transform.hpp" -#include -#include -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -template -static void collect_linked_ops_nearby_input_until_opT( - sc_graph_t &graph, const context_ptr &ctx, - std::vector> &target_ops_list) { - auto vis = op_visitor_t::bfs(); - constexpr int max_step = 5; - vis.visit_graph(graph, [&](op_visitor_t *vis, const sc_op_ptr &node) { - if (!node->isa()) - return; - for (auto &user : node->get_outputs()[0]->uses_) { - sc_op_ptr next_node = user.second; - int step = 1; - std::vector target_ops; - bool found = false; - while ( - next_node->is_single_output_single_use() && - (!next_node->get_outputs()[0]->uses_[0].second->isa()) && - (next_node->attrs_.get_or_else("constant", const_kind::not_const) == - const_kind::not_const)) { - target_ops.emplace_back(next_node); - if (next_node->isa()) { - found = true; - break; - } - if ((step++) >= max_step) - return; - next_node = next_node->get_outputs()[0]->uses_[0].second; - } - if (found && target_ops.size() > 1) { - target_ops_list.emplace_back(target_ops); - } - } - }); -} - -/** - * The reschedule bypass rule is: - * 1. upwards visit. - * 2. scan bypass ops list: find the first non-elementwise op and try to swap - * reorder op position with it firstly in avoid of unnecessay fusion break. - * */ -static void reschedule_reorder_nearby_input(std::vector target_ops) { - if (target_ops.empty()) - return; - // first op in bypass list - auto begin_op = target_ops.front(); - auto end_op = target_ops.back(); - COMPILE_ASSERT(end_op->isa(), - "tensorview op is expected, but got " - << end_op->op_name_ << "_" << end_op->logical_op_id_) - // support automatically position switch for unary and special - // broadcast ops - auto last_reo = end_op->dyn_cast(); - sc_op *pre_op = end_op.get(); - sc_data_type_t cached_dtype = end_op->get_inputs()[0]->details_.dtype_; - // search the swtichable ops - int i = target_ops.size() - 1; - int j = i; - std::unordered_map idx_mp; - while (pre_op != begin_op.get()) { - pre_op = target_ops[--i].get(); - if (pre_op->isa()) { - if (pre_op->isa()) { - if (utils::get_sizeof_type(pre_op->get_inputs()[0]->details_.dtype_) > - utils::get_sizeof_type(pre_op->get_outputs()[0]->details_.dtype_)) { - break; - } - cached_dtype = pre_op->get_inputs()[0]->details_.dtype_; - } - idx_mp[i] = 0; - j = i; - } else if (pre_op->isa()) { - int bc_idx = - pre_op->dyn_cast()->get_broadcast_input(); - if (bc_idx < 0) - break; - if (auto const_op = pre_op->get_inputs()[bc_idx] - ->producer_owner_->dyn_cast()) { - auto dims = const_op->get_constant_blocking_dims(); - if (dims.size() == 1 && dims[0] == 1) { - idx_mp[i] = 1 - bc_idx; - j = i; - continue; - } - } - break; - } else { - break; - } - } - pre_op = target_ops[j].get(); - - // switch pre_op and last_reo position - if (pre_op == begin_op.get()) { - auto new_fmt = last_reo->get_outputs()[0]->details_.get_format(); - // disconnect reorder - last_reo->get_outputs()[0]->replace_with( - target_ops[target_ops.size() - 2]->get_outputs()[0]); - // insert reorder - last_reo->replace_input(0, pre_op->get_inputs()[idx_mp[j]]); - pre_op->replace_input(idx_mp[j], last_reo->get_outputs()[0]); - - // reset attribute - last_reo->info_.outputs_[0]->details_.dtype_ = cached_dtype; - // update output format - for (int k = j; k < static_cast(target_ops.size()) - 1; k++) { - target_ops[k]->get_outputs()[0]->details_.set_format(new_fmt); - } - } -} - -static void nearby_input_reorder_rule(sc_graph_t &graph, - const context_ptr &ctx) { - std::vector> target_ops_list; - collect_linked_ops_nearby_input_until_opT(graph, ctx, - target_ops_list); - // reschedule all bypass ops - for (auto &target_ops : target_ops_list) { - reschedule_reorder_nearby_input(target_ops); - } -} - -static void find_reorder_and_tv_from_matmul2D2ND( - sc_graph_t &graph, const context_ptr &ctx, - std::vector> &target_ops_list, - std::unordered_map &op_bc_idx) { - auto vis = op_visitor_t::dfs_topology_sort(); - vis.visit_graph(graph, [&](op_visitor_t *vis, const sc_op_ptr &node) { - // 1. Find the tensorview op which is generated by matmul. Its parent - // should be a reorder op, whose input format should be blocking and - // output format should be plain. If the reorder op is marked as - // "break_post_fuse", it is our target. - if (!node->isa()) - return; - if (!node->attrs_.get_or_else("source_matmul_2D2ND", false)) - return; - if (!node->is_single_output_single_use()) - return; - auto tv_parent = node->info_.inputs_[0]->producer_owner_; - if (!tv_parent->isa()) - return; - // Note: break_post_fuse is marked by mixed_partition pass, - // which is a later one. Can not get it here. if - // (!tv_parent->attrs_.get_or_else(op_attr_key::break_post_fuse, false)) - // return; - if (!tv_parent->is_single_output_single_use()) - return; - auto &reorder_input_format = - tv_parent->get_inputs()[0]->details_.get_format(); - auto &reorder_output_format = - tv_parent->get_outputs()[0]->details_.get_format(); - // this reorder is from ABab to AB - if (!reorder_input_format.format_code_.is_blocking() || - reorder_input_format.format_code_ != format_kinds::ABab || - !reorder_output_format.format_code_.is_plain()) { - return; - } - - std::vector target_ops{tv_parent->shared_from_this(), node}; - sc_op_ptr child_node = node->get_outputs()[0]->uses_[0].second; - while (child_node->is_single_output_single_use()) { - if (child_node->isa()) { - target_ops.emplace_back(child_node); - } else if (child_node->isa()) { - /* - for binary op, we need to check that it is with broadcast; - because for topo like - matmul0 - reorder0 - tv0 \ - add, - matmul1 - reorder1 - tv1 / - the add op should not be moved. - */ - std::vector non_bc_inputs = - child_node->dyn_cast() - ->get_non_broadcast_input_index(true); - if (non_bc_inputs.size() != 2) { - int bc_idx = child_node->dyn_cast() - ->get_broadcast_input(); - op_bc_idx[child_node->logical_op_id_] = bc_idx; - target_ops.emplace_back(child_node); - } else { - break; - } - } else { - break; - } - child_node = child_node->get_outputs()[0]->uses_[0].second; - } - if (target_ops.size() > 2) { - target_ops_list.emplace_back(target_ops); - } - }); -} - -static void -postpone_reorder_and_tv(sc_graph_t &graph, - const std::vector &target_ops, - const std::unordered_map &op_bc_idx) { - if (target_ops.size() < 2) - return; - auto first_op = target_ops.front(); - COMPILE_ASSERT(first_op->isa(), - "reorder op is expected, but got " - << first_op->op_name_ << "_" << first_op->logical_op_id_); - // target ops: {reorder, tensorview, unary/banary ops} - auto &reorder_input = first_op->get_inputs()[0]; - auto &reorder_input_format // ABab blocking - = reorder_input->details_.get_format(); - auto &reorder_output_format // AB plain - = first_op->get_outputs()[0]->details_.get_format(); - sc_data_format_t::blocking_t blocks = reorder_input_format.blocks_; - blocks[0] = 1; // only blocking at B-axis - sc_data_format_t new_fmt = - sc_data_format_t(reorder_input_format.format_code_, blocks); - - // Move the reorder op and tv op to last. Other ops in original order. - auto &tv_op = target_ops[1]; - auto &third_op = target_ops[2]; // the op after reorder and tensorview - auto &last_op = target_ops.back(); - for (auto &use : tv_op->get_outputs()[0]->uses_) { - // tsr tv_output is single-used by third_op - third_op->replace_input(use.first, reorder_input); - } - for (auto &use : last_op->get_outputs()[0]->uses_) { - use.second->replace_input(use.first, tv_op->get_outputs()[0]); - } - first_op->replace_input(0, last_op->get_outputs()[0]); - - for (size_t i = 2; i < target_ops.size(); ++i) { - auto &curr_op = target_ops[i]; - if (curr_op->isa()) { - auto new_output = std::make_shared( - curr_op.get(), curr_op->get_inputs()[0]->details_.get_format(), - curr_op->get_inputs()[0]->details_.get_plain_dims(), - // this may be a cast op, so use dtype of the output tensor - curr_op->get_outputs()[0]->details_.dtype_); - curr_op->get_outputs()[0]->replace_with(new_output); - curr_op->info_.outputs_[0] = new_output; - } else if (curr_op->isa()) { - // For binary ops, add tensorview to change the format and shape - int bc_idx = op_bc_idx.at(curr_op->logical_op_id_); - auto &ori_tsr = curr_op->get_inputs()[bc_idx]; - auto &parent_op = ori_tsr->producer_owner_; - if (ori_tsr->details_.get_format() != reorder_input_format && - ori_tsr->details_.get_plain_dims().size() > 1) { - // reshape to 1D - auto &ori_tsr_shape = ori_tsr->details_.get_blocking_dims(); - sc_dims plain_shape = {1, std::accumulate(ori_tsr_shape.begin(), - ori_tsr_shape.end(), 1, - std::multiplies())}; - auto new_tv = - graph.make("tensor_view", parent_op->get_outputs(), {}, - {{"shape", plain_shape}, - {"format", sc_data_format_t(format_kinds::AB)}}); - // reorder to blocking format - auto new_reorder = - graph.make("reorder", new_tv->get_outputs(), {}, - {{"internal", true}, {"out_format", new_fmt}}); - curr_op->replace_input(bc_idx, new_reorder->get_outputs()[0]); - } - auto new_output = std::make_shared( - curr_op.get(), - curr_op->get_inputs()[1 - bc_idx]->details_.get_format(), - curr_op->get_inputs()[1 - bc_idx]->details_.get_plain_dims(), - curr_op->get_inputs()[1 - bc_idx]->details_.dtype_); - curr_op->get_outputs()[0]->replace_with(new_output); - curr_op->info_.outputs_[0] = new_output; - if (!curr_op->attrs_.get_or_else("bc_axis", std::vector{}).empty()) { - // Note: broadcast happens at axis #1 (axis N of matmul) - curr_op->attrs_["bc_axis"] = std::vector{1}; - } - curr_op->dyn_cast()->set_plain_bc_axis(); - } - } - // the shape and format of output tensors of reorder and tv are not - // affected; but if the dtype is changed, rebuild them. - if (first_op->info_.inputs_[0]->details_.dtype_ != - first_op->info_.outputs_[0]->details_.dtype_) { - for (size_t i = 0; i < 2; ++i) { - auto &curr_op = target_ops[i]; - auto new_output = std::make_shared( - curr_op.get(), curr_op->get_outputs()[0]->details_.get_format(), - curr_op->get_outputs()[0]->details_.get_plain_dims(), - curr_op->get_inputs()[0]->details_.dtype_); - curr_op->get_outputs()[0]->replace_with(new_output); - curr_op->info_.outputs_[0] = new_output; - } - } -} - -static void postpone_reorder_and_tv_from_matmul2D2ND(sc_graph_t &graph, - const context_ptr &ctx) { - std::vector> target_ops_list; - std::unordered_map op_bc_idx; - find_reorder_and_tv_from_matmul2D2ND(graph, ctx, target_ops_list, op_bc_idx); - for (auto &target_ops : target_ops_list) { - postpone_reorder_and_tv(graph, target_ops, op_bc_idx); - } -} - -using gr_rule = std::function; - -void global_reschedule(sc_graph_t &graph, const context_ptr &ctx) { - if (!graph.attrs_.get_or_else("temp.fuse", 1)) { - return; - } - if (graph.is_dynamic()) { - return; - } - std::vector gr_rule_list; - gr_rule_list.emplace_back(nearby_input_reorder_rule); - gr_rule_list.emplace_back(postpone_reorder_and_tv_from_matmul2D2ND); - for (auto &gr_rule : gr_rule_list) { - gr_rule(graph, ctx); - } - graph.reset_op_ids(); -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/transform/graph_concat_memory_planning.cpp b/legacy/core/src/compiler/ir/graph/transform/graph_concat_memory_planning.cpp deleted file mode 100644 index 2e020f497..000000000 --- a/legacy/core/src/compiler/ir/graph/transform/graph_concat_memory_planning.cpp +++ /dev/null @@ -1,363 +0,0 @@ -/******************************************************************************* - * Copyright 2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "../visitor.hpp" -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -SC_MODULE(graph.graph_concat_memory_planning); - -/* -For the following graph: - op0 op1 op2 - \ | / - \ | / - concat0 op3 - \ / - \ / - concat1 -Merge the consecutive two concats into one, i.e., merge concat0 into concat1: - op0 op1 op2 - \ | / - \ | / - \ | / op3 - \ | / / - \ | / / - concat1 -Constraints: -0. the output of concat0 is only use by concat1 -1. concat dim must be equal. -*/ -SC_INTERNAL_API void -merge_concats(sc_graph_t &graph, - const context_ptr &ctx = get_default_context()) { - op_visitor_t vis = op_visitor_t::bfs(); - vis.visit_graph(graph, [&](op_visitor_t *vis, const sc_op_ptr &op) { - if (!op->isa()) { - return; - } - auto concat = op->stc_cast(); - std::vector ori_inputs = concat->info_.inputs_; - std::vector new_inputs; - std::unordered_map new_idx_to_ori_idx; - std::unordered_set ops_to_remove; - for (size_t i = 0; i < ori_inputs.size(); ++i) { - auto &ori_input = ori_inputs[i]; - sc_op *parent_op = ori_input->producer_owner_; - // 1. parent op is concat and its only output is only used by - // current op. - // 2. the concat dim must be equal for current op and parent op - if (parent_op->isa() && - parent_op->stc_cast()->get_axis() == - concat->get_axis() && - parent_op->get_outputs()[0]->uses_.size() == 1) { - SC_MODULE_INFO << "Meets two consecutive concat ops: " - << parent_op->logical_op_id_ << " --> " - << concat->logical_op_id_ - << ", same concat dim: " << concat->get_axis(); - // the inputs of parent op are now inputs of current op - new_inputs.insert(new_inputs.end(), parent_op->info_.inputs_.begin(), - parent_op->info_.inputs_.end()); - ops_to_remove.insert(parent_op); - } else { - new_idx_to_ori_idx[new_inputs.size()] = i; - new_inputs.push_back(ori_input); - } - } - if (ori_inputs != new_inputs) { - op->info_.inputs_ = new_inputs; - concat->is_input_valid_ = std::vector(new_inputs.size(), true); - for (size_t j = 0; j < op->info_.inputs_.size(); ++j) { - // can not call replace_input here because the input idx is - // changed, and detach_use and attach_use uses different index - op->info_.inputs_[j]->detach_use(op, new_idx_to_ori_idx[j]); - new_inputs[j]->attach_use(op, j); - } - for (auto &op_to_remove : ops_to_remove) { - SC_MODULE_INFO << "Remove op: " << op_to_remove->op_name_ - << op_to_remove->logical_op_id_; - op_to_remove->remove(); - } - vis->update_state_for_visited(op); - } - }); - graph.reset_op_ids(); -} - -// Calc multi-dimensional offset of each input to concat. -// Each input has specific offset to the output buffer of concat. -static std::vector calc_offsets(std::vector &inputs_dims, - unsigned concat_dim) { - size_t num_dims = inputs_dims[0].size(); - std::vector offsets(inputs_dims.size(), sc_dims(num_dims, 0)); - sc_dim offset = 0; - // the offset of the first tensor is 0 - for (size_t i = 1; i < inputs_dims.size(); ++i) { - offset += inputs_dims[i - 1][concat_dim]; - offsets[i][concat_dim] = offset; - } - return offsets; -} - -static bool has_output_user(const graph_tensor_ptr >) { - for (auto &use : gt->uses_) { - if (use.second->isa()) { - return true; - } - } - return false; -} - -/* This function is the preparation for TensorIR pass concat_memory_planning_t. - * We calculate the offset for each input of concat op and set strides of output - * of parent op. For detailed explanation, please refer to - * concat_memory_planning_t. */ -static bool set_offsets_and_strides_for_op( - const std::unordered_set &ops, sc_op *op, const sc_dims &strides, - std::unordered_map> - &concat_in_out) { - auto concat = op->stc_cast(); - unsigned concat_dim = concat->get_axis(); - SC_MODULE_INFO << "Set offset and strides for concat_op: " << op->op_name_ - << op->logical_op_id_; - std::vector &inputs = concat->info_.inputs_; - std::unordered_set inputs_set(inputs.begin(), inputs.end()); - if (inputs_set.size() != inputs.size()) { - SC_MODULE_INFO << "There are duplicate inputs to current concat, cannot " - "do memory planning, donot set offset and strides"; - return false; - } - std::vector inputs_dims(inputs.size()); - for (size_t i = 0; i < inputs.size(); ++i) { - inputs_dims[i] = inputs[i]->details_.get_blocking_dims(); - } - std::vector offsets = calc_offsets(inputs_dims, concat_dim); - - // The input of concat is the output of its parent op, so we need to - // set strides of the output of the parent op. - for (size_t i = 0; i < inputs.size(); ++i) { - auto parent_op = inputs[i]->producer_owner_->shared_from_this(); - if (parent_op->isa() || parent_op->isa() || - has_output_user(inputs[i])) { - continue; - } - if (ops.count(parent_op)) { - if (parent_op->isa() && - parent_op->stc_cast()->parti_list_.size() != 1) { - SC_MODULE_INFO << "Do not support mixed fuse op with " - "multiple partitions"; - continue; - } - - auto it = find(parent_op->info_.outputs_.begin(), - parent_op->info_.outputs_.end(), inputs[i]); - COMPILE_ASSERT(it != parent_op->info_.outputs_.end(), - "Cannot find input tensor"); - size_t output_idx = it - parent_op->info_.outputs_.begin(); - - concat_in_out[inputs[i]] = {concat->info_.outputs_[0], offsets[i]}; - inputs[i]->details_.set_strides(strides); - concat->is_input_valid_[i] = false; - if (parent_op->isa()) { - auto parti = - parent_op->stc_cast()->parti_list_[0].get(); - std::vector strides_expr; - for (auto o : strides) { - strides_expr.emplace_back(uint64_t(o)); - } - parti->func_->params_[output_idx].checked_as()->strides_ = - strides_expr; - } - SC_MODULE_INFO << "Set strides of tensor: #" - << it - parent_op->info_.outputs_.begin() - << " output of op " << parent_op->op_name_ - << parent_op->logical_op_id_ - << " to : " << utils::print_vector(strides); - } - } - - concat->info_.outputs_[0]->details_.set_strides(strides); - - for (auto &id_op_pair : concat->info_.outputs_[0]->uses_) { - auto child_op = id_op_pair.second.lock(); - if (ops.count(child_op)) { - if (child_op->isa()) { - auto child_mixed = child_op->stc_cast(); - // TODO(niuxiaoguang): to support this case - COMPILE_ASSERT(child_mixed->parti_list_.size() == 1, - "The child op of concat should not have multi " - "partitions: " - << child_mixed->parti_list_.size()); - auto parti = child_mixed->parti_list_[0].get(); - std::vector strides_expr; - for (auto o : strides) { - strides_expr.emplace_back(uint64_t(o)); - } - int input_idx = id_op_pair.first + child_op->info_.outputs_.size(); - parti->func_->params_[input_idx].checked_as()->strides_ = - strides_expr; - SC_MODULE_INFO << "Set strides of tensor: #" << id_op_pair.first - << " input of op " << child_op->op_name_ - << child_op->logical_op_id_ - << " to : " << utils::print_vector(strides); - } - } - } - return true; -} - -static void find_final_tensor_and_offset( - std::unordered_map> - &concat_in_out) { - for (auto &pair : concat_in_out) { - graph_tensor_ptr curr = pair.first; - size_t n_dims = curr->details_.get_blocking_dims().size(); - sc_dims final_offset(n_dims, 0); - while (concat_in_out.find(curr) != concat_in_out.end()) { - auto parent = concat_in_out[curr].first; - auto offset = concat_in_out[curr].second; - curr = parent; - for (size_t i = 0; i < n_dims; ++i) { - final_offset[i] = final_offset[i] + offset[i]; - } - } - pair.second = {curr, final_offset}; - curr->producer_owner_->attrs_[concat_optim_attr_keys::is_final_concat] = - true; - } -} - -static void set_final_offsets( - sc_op *op, - std::unordered_map> - &concat_in_out) { - auto concat = op->stc_cast(); - std::vector &inputs = concat->info_.inputs_; - for (size_t i = 0; i < inputs.size(); ++i) { - if (concat_in_out.find(inputs[i]) != concat_in_out.end()) { - SC_MODULE_INFO << "Set final offset for concat_op: " << op->op_name_ - << op->logical_op_id_ << " #" << i << " input"; - inputs[i]->attrs_[concat_optim_attr_keys::graph_memory_offset_to] = - concat_in_out[inputs[i]].first; - std::vector offset_expr; - for (auto o : concat_in_out[inputs[i]].second) { - offset_expr.emplace_back(uint64_t(o)); - } - inputs[i]->attrs_[concat_optim_attr_keys::graph_memory_offset] = - offset_expr; - } - } -} - -bool set_offsets_and_strides_recursively(std::vector &ops) { - std::unordered_set ops_set(ops.begin(), ops.end()); - std::vector> concats_seqs; - for (sc_op_ptr &op : ops) { - if (!op->isa()) { - continue; - } - auto concat = op->stc_cast(); - SC_MODULE_INFO << "Meet concat_op: " << op->op_name_ << op->logical_op_id_; - bool new_seq = true; - // We put the directly connected concats into one sequence. - // TODO(niuxiaoguang): support the topo that multiple parents are - // concat. - for (const auto &inp_lt : concat->info_.inputs_) { - sc_op *parent_op = inp_lt->producer_owner_; - if (parent_op->isa()) { - // tensor_view_op_t will be skiped when lowering, so the concat - // op cannot get memory_offset attr from its new input - SC_MODULE_INFO << "Do not optimize curr input because it is " - "from a tensor_view_op_t"; - continue; - } - if (parent_op->isa() && - parent_op->stc_cast()->get_axis() == - concat->get_axis()) { - SC_MODULE_INFO << "Current concat has a concat parent with same dim"; - if (ops_set.count(parent_op->shared_from_this()) == 0) { - SC_MODULE_INFO << "The parent op is not in current " - "partition, so start a new seq."; - continue; - } - new_seq = false; - for (auto &seq : concats_seqs) { - if (seq.back() == parent_op) { - seq.push_back(op.get()); - break; - } - } - } - } - if (new_seq) { - concats_seqs.push_back({op.get()}); - } - } - - SC_MODULE_INFO << "There are " << concats_seqs.size() - << " sequences of concat ops."; - if (!concats_seqs.empty()) { - std::unordered_map> - concat_in_out; - for (auto &concat_seq : concats_seqs) { - SC_MODULE_INFO << "Process concats seq with " << concat_seq.size() - << " concat ops."; - sc_dims strides = - concat_seq.back()->info_.outputs_[0]->details_.get_strides(); - for (int i = concat_seq.size() - 1; i >= 0; --i) { - set_offsets_and_strides_for_op(ops_set, concat_seq[i], strides, - concat_in_out); - } - } - - find_final_tensor_and_offset(concat_in_out); - - for (auto &concat_seq : concats_seqs) { - for (int i = concat_seq.size() - 1; i >= 0; --i) { - set_final_offsets(concat_seq[i], concat_in_out); - } - } - } - return !concats_seqs.empty(); -} - -SC_INTERNAL_API void -graph_concat_memory_planning(sc_graph_t &graph, - const context_ptr &ctx = get_default_context()) { - SC_MODULE_INFO << "Run graph concat memory planning on graph with " - << graph.ops_.size() << " ops."; - if (graph.ops_.size() < 2) { - return; - } - set_offsets_and_strides_recursively(graph.ops_); -} - -SC_INTERNAL_API bool concat_memory_planning_on_graph(sc_graph_t &graph) { - return set_offsets_and_strides_recursively(graph.ops_); -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/transform/graph_inline.cpp b/legacy/core/src/compiler/ir/graph/transform/graph_inline.cpp deleted file mode 100644 index ec11fc4e5..000000000 --- a/legacy/core/src/compiler/ir/graph/transform/graph_inline.cpp +++ /dev/null @@ -1,159 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "../fusible_op.hpp" -#include "../graph_op.hpp" -#include "../pass/pass.hpp" -#include "../visitor.hpp" -#include "transform.hpp" -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -static void do_inline_graph(const context_ptr &ctx, sc_graph_t &full_graph, - sc_op_ptr &cur_node, sc_graph_t &sub_graph) { - std::unordered_map> *tunable_op_map = - full_graph.attrs_ - .get_or_null>>( - "temp.op_map"); - sc_op_ptr corresponding_node; - bool need_tuning = tunable_op_map; - if (need_tuning) { - if (tunable_op_map->find(cur_node) != tunable_op_map->end()) { - corresponding_node = (*tunable_op_map)[cur_node][0]; - (*tunable_op_map)[corresponding_node].clear(); - } - } - int cur_op_id = cur_node->logical_op_id_; - for (auto &op : sub_graph.ops_) { - if (op->isa()) { - auto cur_op_ori_ins = cur_node->get_inputs(); - COMPILE_ASSERT(cur_op_ori_ins.size() == op->get_outputs().size(), - "cur_node " << cur_node->op_name_ - << " 's input size should be equal with its " - "sub_graph input op 's output size"); - if (ctx->flags_.opt_level_ == sc_opt_level::lv1 || - cur_node->attrs_.get_or_else(op_attr_key::break_pre_fuse, false) || - cur_node->attrs_.get_or_else(op_attr_key::no_fuse, false)) { - for (auto &cur : op->get_outputs()) { - for (auto &u : cur->uses_) { - auto user_op = u.second; - bool need_break = true; - for (auto &in : user_op->get_inputs()) { - if (!(in->producer_owner_->isa() || - in->producer_owner_->isa())) { - need_break = false; - break; - } - } - u.second->attrs_.set(op_attr_key::break_pre_fuse, need_break); - } - } - } - for (size_t i = 0; i < op->get_outputs().size(); ++i) { - op->get_outputs()[i]->replace_with(cur_op_ori_ins.at(i)); - } - op->remove(); - } else if (op->isa()) { - auto cur_op_ori_outs = cur_node->get_outputs(); - COMPILE_ASSERT(cur_op_ori_outs.size() == op->get_inputs().size(), - "cur_node " << cur_node->op_name_ - << " 's output size should be equal with its " - "sub_graph output op's input size"); - if (ctx->flags_.opt_level_ == sc_opt_level::lv1 || - cur_node->attrs_.get_or_else(op_attr_key::break_post_fuse, false) || - cur_node->attrs_.get_or_else(op_attr_key::no_fuse, false)) { - for (auto &cur : op->get_inputs()) { - cur->producer_owner_->attrs_.set(op_attr_key::break_post_fuse, true); - } - } - for (size_t i = 0; i < cur_op_ori_outs.size(); ++i) { - while (!cur_op_ori_outs[i]->uses_.empty()) { - auto &use = cur_op_ori_outs[i]->uses_.front(); - use.second->replace_input(use.first, op->get_inputs()[i]); - } - } - op->remove(); - } else { - if (op->isa() && need_tuning) { - (*tunable_op_map)[corresponding_node].push_back(op); - } - op->set_owner_graph(&full_graph); - full_graph.ops_.emplace_back(op); - op->set_owner_graph(&full_graph); - } - } -} - -const std::set &get_op_blocked_lists() { - static std::set blocked_list{ - "quantize", "dequantize", "dynamic_quantize", "dynamic_dequantize"}; - return blocked_list; -} - -void graph_inline(sc_graph_t &graph, const context_ptr &ctx) { - auto &blocked_list = get_op_blocked_lists(); - constexpr const int max_recursion = 10; - int i = 0; - for (; i < max_recursion; i++) { - auto vis = op_visitor_t::bfs(); - vis.visit_graph(graph, [&](op_visitor_t *vis, sc_op_ptr node) { - if (auto graph_node = node->dyn_cast()) { - if (blocked_list.find(node->op_name_) == blocked_list.end()) { - auto sub_graph = graph_node->get_graph(); - vis->update_state_for_visited(node); - do_inline_graph(ctx, graph, node, *sub_graph); - node->remove(); - } - } - }); - graph.reset_op_ids(); - if (std::all_of( - graph.ops_.begin(), graph.ops_.end(), [&](const sc_op_ptr &op) { - return !op->isa() || - blocked_list.find(op->op_name_) != blocked_list.end(); - })) { - break; - } - } - COMPILE_ASSERT(i < max_recursion, "Reached max inline recursion depth"); - graph.attrs_[sc_graph_t::attr_key_t::gflop] = graph.get_gflop(); -} - -namespace quantize { -void quantize_inline(sc_graph_t &graph, const context_ptr &ctx) { - auto vis = op_visitor_t::bfs(); - vis.visit_graph(graph, [&](op_visitor_t *vis, sc_op_ptr node) { - if (auto graph_node = node->dyn_cast()) { - auto sub_graph = graph_node->get_graph(); - vis->update_state_for_visited(node); - do_inline_graph(ctx, graph, node, *sub_graph); - node->remove(); - } - }); - graph.reset_op_ids(); -} -} // namespace quantize - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/transform/graph_simplify.cpp b/legacy/core/src/compiler/ir/graph/transform/graph_simplify.cpp deleted file mode 100644 index 67816dba5..000000000 --- a/legacy/core/src/compiler/ir/graph/transform/graph_simplify.cpp +++ /dev/null @@ -1,616 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include - -#include "../fusible_op.hpp" -#include "../graph_op.hpp" -#include "../pass/pass.hpp" -#include "../visitor.hpp" -#include "transform.hpp" -#include -#include -#include -#include -#include -#include -#include - -SC_MODULE(graph.simplify) - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -struct hash_sc_op_t { - std::size_t operator()(const sc_op_ptr &v) const { - size_t hash_ = 0; - hash_combine(hash_, v->op_name_); - hash_combine(hash_, v->info_.outputs_[0]->details_); - hash_combine(hash_, v->hash_contents()); - return hash_; - } -}; - -struct compare_sc_op_t { - bool operator()(const sc_op_ptr &v0, const sc_op_ptr &v1) const { - return v0->op_name_ == v1->op_name_ && - v0->info_.outputs_[0]->details_ == v1->info_.outputs_[0]->details_ && - v0->compare_contents(v1.get()); - } -}; - -static void insert_tensor_view_op(sc_graph_t &graph, const graph_tensor_ptr &in, - size_t in_index, const sc_op_ptr &cur_op) { - auto ret = graph.make("tensor_view", {in}, {}, - {{"shape", in->details_.get_plain_dims()}}); - cur_op->replace_input(in_index, ret->get_outputs()[0]); -} - -void drop_same_op_on_output(sc_graph_t &graph, const graph_tensor_ptr &output) { - std::unordered_map, hash_sc_op_t, compare_sc_op_t> - same_op_map; - for (size_t i = 0; i < output->uses_.size(); i++) { - auto node = output->uses_[i]; - // do not eliminate redundant ops marked as "temp.not_redundant" - if (node.second->attrs_.get_or_else(op_attr_key::not_redundant, false)) { - continue; - } - if (node.second->get_inputs().empty() || - node.second->get_outputs().empty()) { - continue; - } - if (node.second->get_inputs().size() > 1 || - node.second->get_outputs().size() > 1) { - SC_MODULE_INFO - << "Currently we don't support multi-input/multi-output op " - "elimination."; - continue; - } - // when uses is an output op, we should not remove the global buffer - auto temp_node = node.second->get_outputs()[0]; - if (std::any_of(temp_node->uses_.begin(), temp_node->uses_.end(), - [](const std::pair &j) { - return j.second->isa(); - })) { - continue; - } - same_op_map[node.second].push_back(i); - } - std::vector> next_nodes(output->uses_.begin(), - output->uses_.end()); - for (auto &it : same_op_map) { - if (it.second.size() > 1) { - auto reserve_node = next_nodes[it.second[0]].second; - std::vector del_node_list; - for (size_t i = 1; i < it.second.size(); i++) { - if (it.second[i] >= static_cast(next_nodes.size()) || - next_nodes[it.second[i]].second->get_outputs()[0]->uses_.empty()) { - break; - } - auto del_node = next_nodes[it.second[i]].second; - std::vector> del_uses = - del_node->get_outputs()[0]->uses_; - for (size_t u = 0; u < del_uses.size(); u++) { - auto node_after_del = del_uses[u]; - node_after_del.second->replace_input(node_after_del.first, - reserve_node->get_outputs()[0]); - } - del_node_list.push_back(del_node); - } - for (auto &del_node : del_node_list) { - del_node->remove(); - } - } - } -} - -// eliminate the tensor_view in which the shape and format are the same for both -// input and output -void useless_tensor_view_elimination(sc_graph_t &graph, - const context_ptr &ctx) { - auto vis = op_visitor_t::bfs(); - vis.visit_graph(graph, [&](op_visitor_t *vis, const sc_op_ptr &node) { - if (node->isa() && - !node->get_inputs()[0]->producer_owner_->isa()) { - const auto &in_tensor = node->get_inputs()[0]->details_; - const auto &in_shape = in_tensor.get_plain_dims(); - const auto &in_real_shape = in_tensor.get_blocking_dims(); - const auto &in_format = in_tensor.get_format(); - const auto &out_tensor = node->get_outputs()[0]->details_; - const auto &out_shape = out_tensor.get_plain_dims(); - const auto &out_real_shape = out_tensor.get_blocking_dims(); - const auto &out_format = out_tensor.get_format(); - if (in_real_shape == out_real_shape && in_shape == out_shape && - in_format == out_format) { - vis->update_state_for_visited(node); - node->get_outputs()[0]->replace_with(node->get_inputs()[0]); - node->remove(); - } - } - }); - graph.reset_op_ids(); -} - -// eliminate horizontal same ops, e.g. qkv input reorder -void horizontal_same_op_elimination(sc_graph_t &graph, const context_ptr &ctx) { - auto vis = op_visitor_t::bfs(); - vis.visit_graph(graph, [&](op_visitor_t *vis, const sc_op_ptr &node) { - if (!node->isa()) { - for (size_t i = 0; i < node->get_outputs().size(); i++) { - auto output = node->get_outputs()[i]; - drop_same_op_on_output(graph, output); - } - } - }); - graph.reset_op_ids(); -} - -static bool is_single_use(const sc_op_ptr &node) { - return node->get_outputs()[0]->uses_.size() == 1; -} - -static void merge_dispatch_key_sets(const sc_op_ptr &op0, sc_op_ptr &op1) { - auto &dispatch_key_set0 = op0->get_dispatch_key_set()->get_inner_set(); - auto &dispatch_key_set1 = op1->get_dispatch_key_set()->get_inner_set(); - std::unordered_map cached_map; - for (auto &key1 : dispatch_key_set1) { - auto &in_fmt1 = key1.in_out_formats_[0]; - auto &out_fmt1 = key1.in_out_formats_[1]; - cached_map[in_fmt1] = out_fmt1; - } - dispatch_key_set_t::inner_set_t dispatch_key_set_new0; - for (auto key0 : dispatch_key_set0) { - auto &out_fmt0 = key0.in_out_formats_[1]; - auto it = cached_map.find(out_fmt0); - assert(it != cached_map.end()); - key0.in_out_formats_[1] = it->second; - dispatch_key_set_new0.insert(key0); - } - op0->get_dispatch_key_set()->get_inner_set() = - std::move(dispatch_key_set_new0); -} - -// eliminate excess tensor view, e.g. tensor_view->tensor_view->tensor_view -void excess_tensor_view_elimination(sc_graph_t &graph, const context_ptr &ctx) { - auto vis = op_visitor_t::bfs(); - vis.visit_graph(graph, [&](op_visitor_t *vis, const sc_op_ptr &node) { - if (node->isa() && is_single_use(node)) { - sc_op_ptr next_node = - node->get_outputs()[0]->uses_[0].second.get_shared(); - sc_op_ptr pre_node = next_node; - std::vector node_to_remove; - while (next_node->isa() && is_single_use(next_node)) { - merge_dispatch_key_sets(node, next_node); - node_to_remove.push_back(next_node); - pre_node = next_node; - next_node = next_node->get_outputs()[0]->uses_[0].second.get_shared(); - } - if (next_node->isa()) { - pre_node = next_node; - } - if (pre_node != next_node || pre_node->isa()) { - if (pre_node == next_node) { - merge_dispatch_key_sets(node, pre_node); - } - node->get_outputs()[0]->details_ = pre_node->get_outputs()[0]->details_; - std::vector> uses = - pre_node->get_outputs()[0]->uses_; - for (size_t i = 0; i < uses.size(); i++) { - int pre_idx = uses[i].first; - uses[i].second->replace_input(pre_idx, node->get_outputs()[0]); - } - for (auto &del_node : node_to_remove) { - del_node->remove(); - } - if (pre_node->isa()) { - pre_node->remove(); - } - if (node->attrs_.has_key("order")) { - node->attrs_.remove("order"); - } - } - vis->update_state_for_visited(node); - } - }); - graph.reset_op_ids(); -} - -static bool can_simplify(const sc_op_ptr &node, - const constant_op_t *in_const_op) { - auto const_dtype = in_const_op->get_constant_dtype(); - if (in_const_op->get_constant_plain_dims() != sc_dims{1}) { - return false; - } - if (const_dtype == datatypes::f32) { - float constant_val = - reinterpret_cast(in_const_op->get_constant_values()->data_)[0]; - if ((constant_val == 0.f && - (node->isa() || node->isa())) || - (constant_val == 1.f && - (node->isa() || node->isa()))) - return true; - } else if (const_dtype == datatypes::s32) { - int constant_val = - reinterpret_cast(in_const_op->get_constant_values()->data_)[0]; - if ((constant_val == 0 && - (node->isa() || node->isa())) || - (constant_val == 1 && (node->isa() || node->isa()))) - return true; - } else if (const_dtype == datatypes::bf16) { - bf16_t constant_val = reinterpret_cast( - in_const_op->get_constant_values()->data_)[0]; - if ((constant_val == bf16_t(0) && - (node->isa() || node->isa())) || - (constant_val == bf16_t(1) && - (node->isa() || node->isa()))) - return true; - } else if (const_dtype == datatypes::f16) { - fp16_t constant_val = reinterpret_cast( - in_const_op->get_constant_values()->data_)[0]; - if ((constant_val == fp16_t(0) && - (node->isa() || node->isa())) || - (constant_val == fp16_t(1) && - (node->isa() || node->isa()))) - return true; - } - return false; -} - -// eliminate redundant binary op -void redundant_binary_op_elimination(sc_graph_t &graph, - const context_ptr &ctx) { - auto vis = op_visitor_t::bfs(); - vis.visit_graph(graph, [&](op_visitor_t *vis, const sc_op_ptr &node) { - vis->update_state_for_visited(node); - // x + 0 or 0 + x or x * 1 or 1 * x - if (node->isa() || node->isa()) { - for (size_t i = 0; i < node->get_inputs().size(); i++) { - if (node->get_inputs()[i]->producer_owner_->isa()) { - auto in_const_op = - node->get_inputs()[i]->producer_owner_->dyn_cast(); - bool do_simplify = can_simplify(node, in_const_op); - if (do_simplify) { - size_t use_size = node->get_outputs()[0]->uses_.size(); - int use_idx = 0; - for (size_t j = 0; j < use_size; j++) { - sc_op_ptr next_node = - node->get_outputs()[0]->uses_.at(use_idx).second.get_shared(); - int idx = node->get_outputs()[0]->uses_.at(use_idx).first; - if (next_node->isa() && - node->get_inputs()[1 - i]->producer_owner_->isa()) { - insert_tensor_view_op(graph, node->get_inputs()[1 - i], idx, - next_node); - } else { - next_node->replace_input(idx, node->get_inputs()[1 - i]); - use_idx--; - } - use_idx++; - } - node->remove(); - if (in_const_op->get_outputs()[0]->uses_.empty()) { - in_const_op->remove(); - } - break; - } - } - } - } - // x - 0 or x / 1 - else if (node->isa() || node->isa()) { - if (node->get_inputs()[1]->producer_owner_->isa()) { - auto in_const_op = - node->get_inputs()[1]->producer_owner_->dyn_cast(); - bool do_simplify = can_simplify(node, in_const_op); - if (do_simplify) { - size_t use_size = node->get_outputs()[0]->uses_.size(); - int use_idx = 0; - for (size_t j = 0; j < use_size; j++) { - sc_op_ptr next_node = - node->get_outputs()[0]->uses_.at(use_idx).second.get_shared(); - int idx = node->get_outputs()[0]->uses_.at(use_idx).first; - if (next_node->isa() && - node->get_inputs()[0]->producer_owner_->isa()) { - insert_tensor_view_op(graph, node->get_inputs()[0], idx, - next_node); - } else { - next_node->replace_input(idx, node->get_inputs()[0]); - use_idx--; - } - use_idx++; - } - node->remove(); - if (in_const_op->get_outputs()[0]->uses_.empty()) { - in_const_op->remove(); - } - } - } - } - }); - graph.reset_op_ids(); -} - -static bool is_basic_binary_calculate_op(const sc_op_ptr &node) { - return node->isa() || node->isa() || - node->isa() || node->isa(); -} - -static bool is_high_priority_op(const sc_op_ptr &node) { - return node->isa() || node->isa(); -} - -static bool is_lower_priority_op(const sc_op_ptr &node) { - return node->isa() || node->isa(); -} - -static bool is_constant_op(const sc_op *node) { - return node->isa() || - node->attrs_.get_or_else("constant", const_kind::not_const) != - const_kind::not_const; -} - -static bool is_all_positive(const sc_op *node) { - return node->attrs_.get_or_else("all_positive", false); -} - -// a add/mul op with two inputs: in0 and in1, when in0 is a constant and in1 -// not, exchange them. -static void exchange_binary_const_ops(sc_graph_t &graph, - const context_ptr &ctx) { - auto vis = op_visitor_t::dfs_topology_sort(); - vis.visit_graph(graph, [&](op_visitor_t *vis, const sc_op_ptr &node) { - if (node->isa() || node->isa()) { - if (node->get_inputs()[0]->producer_owner_->attrs_.get_or_else( - "constant", const_kind::not_const) != const_kind::not_const && - node->get_inputs()[1]->producer_owner_->attrs_.get_or_else( - "constant", const_kind::not_const) == const_kind::not_const) { - auto new_node = graph.make( - node->op_name_, {node->get_inputs()[1], node->get_inputs()[0]}, {}, - node->attrs_); - node->replace_uses_with_and_remove(new_node); - vis->update_state_for_visited(new_node); - } - } - }); - graph.reset_op_ids(); -} - -// For case like: mul + add + relu + (mul/div), change to mul + add + (mul/div) -// + relu for more folding opportunities. -static void push_relu_back(sc_graph_t &graph, const context_ptr &ctx) { - auto vis = op_visitor_t::dfs_topology_sort_unchecked(); - vis.visit_graph(graph, [&](op_visitor_t *vis, const sc_op_ptr &node) { - if (node->isa()) { - sc_op_ptr cur_node = node, pre_node = node; - while (cur_node->is_single_output_single_use() && - utils::is_one_of( - cur_node->get_outputs()[0]->uses_[0].second->op_name_, - std::string("mul"), std::string("div"))) { - cur_node = cur_node->get_outputs()[0]->uses_[0].second; - auto inp1 = cur_node->get_inputs()[1]->producer_owner_; - if (!(is_constant_op(inp1) && is_all_positive(inp1))) { - cur_node = pre_node; - break; - } - if (pre_node == node) { - cur_node->replace_input(node->get_outputs()[0]->uses_[0].first, - node->get_inputs()[0]); - } - pre_node = cur_node; - } - if (cur_node != node) { - auto &out_tsr = cur_node->get_outputs()[0]; - auto uses = out_tsr->uses_; - for (auto &use : uses) { - use.second->replace_input(use.first, node->get_outputs()[0]); - } - node->replace_input(0, out_tsr); - } - } - }); - graph.reset_op_ids(); -} - -static bool is_same_priority(const sc_op_ptr &op0, const sc_op_ptr &op1, - const sc_data_type_t &dtype, - std::string &elt_type) { - std::string add = "add", sub = "sub", mul = "mul", div = "div"; - bool ret = false; - if (utils::is_one_of(op0->op_name_, add, sub) && - utils::is_one_of(op1->op_name_, add, sub)) { - if (op0->op_name_ == sub) { - elt_type = op1->op_name_ == add ? sub : add; - } else { - elt_type = op1->op_name_; - } - return true; - } - if (dtype == datatypes::f32) { - if (utils::is_one_of(op0->op_name_, mul, div) && - utils::is_one_of(op1->op_name_, mul, div)) { - ret = true; - } - } else if (dtype == datatypes::s32) { - if ((op0->op_name_ == mul && op1->op_name_ == mul) || - (op0->op_name_ == div && op1->op_name_ == div)) { - ret = true; - } - } - if (ret) { - if (op0->op_name_ == div) { - elt_type = op1->op_name_ == mul ? div : mul; - } else { - elt_type = op1->op_name_; - } - } - return ret; -} - -// Process the pattern like "(a + b) + c" => "a + (b + c)" where b and c are -// constant. Return the new folded node if processing is successful. -static sc_op_ptr same_priority_pattern_fold(sc_graph_t &graph, - const sc_op_ptr &node) { - if (is_basic_binary_calculate_op(node) && - node->is_single_output_single_use()) { - auto &out_tsr = node->get_outputs()[0]; - auto next_node = out_tsr->uses_[0].second; - std::string elt_type; - if (utils::is_one_of(out_tsr->details_.dtype_, datatypes::f32, - datatypes::s32) && - is_same_priority(node, next_node, out_tsr->details_.dtype_, elt_type)) { - auto *cur_inp1 = node->get_inputs()[1]->producer_owner_; - auto *next_inp1 = next_node->get_inputs()[1]->producer_owner_; - if (is_constant_op(cur_inp1) && is_constant_op(next_inp1) && - next_inp1 != node.get()) { - sc_op_ptr cal_const; - cal_const = graph.make( - elt_type, {cur_inp1->get_outputs()[0], next_inp1->get_outputs()[0]}, - {}, {}); - cal_const->attrs_.set("constant", const_kind::local_const); - cal_const->attrs_.set("all_positive", is_all_positive(cur_inp1) && - is_all_positive(next_inp1)); - auto new_node = - graph.make(node->op_name_, - {node->get_inputs()[0], cal_const->get_outputs()[0]}, {}, - node->attrs_); - new_node->copy_dispatch_key_set_from_op(node); - node->replace_uses_with_and_remove(new_node); - auto uses = next_node->get_outputs()[0]->uses_; - for (auto &use : uses) { - use.second->replace_input(use.first, new_node->get_outputs()[0]); - } - next_node->remove(); - return new_node; - } - } - } - return nullptr; -} - -// process the partten like "(a * b + c) * d => (a * b * d + c * d)" where b, c, -// d are constant. Return the new folded node if processing is successful. -static sc_op_ptr diff_priority_pattern_fold(sc_graph_t &graph, - const sc_op_ptr &node) { - if (is_high_priority_op(node) && node->is_single_output_single_use()) { - auto &out_tsr = node->get_outputs()[0]; - auto &dtype = out_tsr->details_.dtype_; - auto next_node = out_tsr->uses_[0].second; - auto *cur_inp1 = node->get_inputs()[1]->producer_owner_; - if (!is_constant_op(cur_inp1)) { - return nullptr; - } - if (is_lower_priority_op(next_node) && - node->is_single_output_single_use()) { - auto nnext_node = next_node->get_outputs()[0]->uses_[0].second; - auto *next_inp1 = next_node->get_inputs()[1]->producer_owner_; - if (!is_constant_op(next_inp1)) { - return nullptr; - } - if (is_high_priority_op(nnext_node) && - next_node->is_single_output_single_use()) { - auto nnext_inp1 = nnext_node->get_inputs()[1]->producer_owner_; - if (!is_constant_op(nnext_inp1)) { - return nullptr; - } - std::string cur_elt_type = "mul", next_elt_type = nnext_node->op_name_; - if ((dtype == datatypes::s32 && node->isa() && - nnext_node->isa()) || - (dtype == datatypes::f32 && - is_same_priority(node, nnext_node, dtype, cur_elt_type))) { - auto cur_cal_const = graph.make( - cur_elt_type, - {cur_inp1->get_outputs()[0], nnext_inp1->get_outputs()[0]}, {}, - {}); - cur_cal_const->attrs_.set("constant", const_kind::local_const); - cur_cal_const->attrs_.set("all_positive", - is_all_positive(cur_inp1) && - is_all_positive(nnext_inp1)); - auto new_node = graph.make( - node->op_name_, - {node->get_inputs()[0], cur_cal_const->get_outputs()[0]}, {}, - node->attrs_); - new_node->copy_dispatch_key_set_from_op(node); - node->replace_uses_with_and_remove(new_node); - auto next_cal_const = graph.make( - next_elt_type, - {next_inp1->get_outputs()[0], nnext_inp1->get_outputs()[0]}, {}, - {}); - next_cal_const->attrs_.set("constant", const_kind::local_const); - next_cal_const->attrs_.set("all_positive", - is_all_positive(next_inp1) && - is_all_positive(nnext_inp1)); - auto new_next_node = graph.make( - next_node->op_name_, - {next_node->get_inputs()[0], next_cal_const->get_outputs()[0]}, - {}, next_node->attrs_); - new_next_node->copy_dispatch_key_set_from_op(next_node); - next_node->replace_uses_with_and_remove(new_next_node); - auto uses = nnext_node->get_outputs()[0]->uses_; - auto last_out = new_next_node->get_outputs()[0]; - for (auto &use : uses) { - use.second->replace_input(use.first, last_out); - } - nnext_node->remove(); - return new_node; - } - } - } - } - return nullptr; -} - -static void fold_polynomial(sc_graph_t &graph, const context_ptr &ctx) { - auto vis = op_visitor_t::bfs(); - constexpr const int MAX_TRY_TIMES = 100; - vis.visit_graph(graph, [&](op_visitor_t *vis, const sc_op_ptr &node) { - sc_op_ptr last_folded = node, pre_folded = last_folded; - for (int i = 0; i < MAX_TRY_TIMES; i++) { - auto same_folded = same_priority_pattern_fold(graph, last_folded); - if (same_folded) { - last_folded = same_folded; - } - auto diff_folded = diff_priority_pattern_fold(graph, last_folded); - if (diff_folded) { - last_folded = diff_folded; - } - if (pre_folded == last_folded) { - break; - } - pre_folded = last_folded; - } - vis->update_state_for_visited(last_folded); - }); - graph.reset_op_ids(); -} - -void graph_constant_folding(sc_graph_t &graph, const context_ptr &ctx) { - exchange_binary_const_ops(graph, ctx); - push_relu_back(graph, ctx); - fold_polynomial(graph, ctx); -} - -void graph_simplify(sc_graph_t &graph, const context_ptr &ctx) { - redundant_binary_op_elimination(graph, ctx); - excess_tensor_view_elimination(graph, ctx); - useless_tensor_view_elimination(graph, ctx); - horizontal_same_op_elimination(graph, ctx); - graph_constant_folding(graph, ctx); -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/transform/inplace_transform.cpp b/legacy/core/src/compiler/ir/graph/transform/inplace_transform.cpp deleted file mode 100644 index 8773efe2b..000000000 --- a/legacy/core/src/compiler/ir/graph/transform/inplace_transform.cpp +++ /dev/null @@ -1,211 +0,0 @@ -/******************************************************************************* - * Copyright 2021-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "../fusible_op.hpp" -#include "../graph.hpp" -#include "../visitor.hpp" -#include -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -enum inplace_status : int { no_linked, linked_output, directly_linked_output }; -static bool is_copy_reorder(const sc_op_ptr &node) { - return node->isa() && - node->attrs_.get_or_else("actually_copy", false); -} - -static sc_op_ptr skip_tensor_view(const sc_op_ptr &node, int &index) { - auto cur_node = node; - while (cur_node->isa() && - cur_node->is_single_output_single_use()) { - index = cur_node->get_outputs()[0]->uses_[0].first; - cur_node = cur_node->get_outputs()[0]->uses_[0].second.lock(); - } - return cur_node; -} -static bool has_linked_input(const sc_op_ptr &node) { - if (node->isa()) { - return true; - } - auto cur_node = node.get(); - while (cur_node->isa()) { - cur_node = cur_node->get_inputs()[0]->producer_owner_; - } - return cur_node->isa(); -} - -static bool has_linked_output(const sc_op_ptr &node) { - if (node->isa()) { - return true; - } - if (node->isa()) { - for (auto &use : node->get_outputs()[0]->uses_) { - if (has_linked_output(use.second)) { - return true; - } - } - } - return false; -} - -// return if the copy_reorder is linked to output. -inplace_status has_linked_output_and_modify_copy(int out_idx, - const sc_op_ptr &node, - bool is_del = true, - bool force_insert = false, - bool force_delete = false) { - auto cur_node = skip_tensor_view(node, out_idx); - auto has_linked_out = has_linked_output(cur_node); - if (is_del && is_copy_reorder(cur_node)) { - assert(cur_node->get_outputs()[0]->uses_.size() == 1); - auto next_node = cur_node->get_outputs()[0]->uses_[0].second.lock(); - // skip tensor view - auto nnext_node = skip_tensor_view(next_node, out_idx); - auto nnext_linked_out = has_linked_output(nnext_node); - if (nnext_linked_out && !force_delete) { - return linked_output; - } - // delete copy. - next_node->replace_input(cur_node->get_outputs()[0]->uses_[0].first, - cur_node->get_inputs()[0]); - cur_node->remove(); - return nnext_linked_out ? linked_output : no_linked; - } else if (!is_del && force_insert && has_linked_out) { - // insert copy - sc_graph_t &graph = node->get_owner_graph(); - auto old_ltsr = cur_node->get_inputs()[out_idx]; - auto cp_reorder = - graph.make("reorder", {old_ltsr}, {}, - {{"internal", true}, - {"actually_copy", true}, - {"out_format", old_ltsr->details_.get_format()}}); - cur_node->replace_input(out_idx, cp_reorder->get_outputs()[0]); - return linked_output; - } - // judge cur node is a output op. - return has_linked_out ? directly_linked_output : no_linked; -} - -// This function aims to automatically detect wrong inplacement of external -// input/output buffers and insert copying reorders between them. -void invalid_inplacement_detection(sc_graph_t &graph) { - auto vis = op_visitor_t::bfs(); - vis.visit_graph(graph, [&](op_visitor_t *vis, const sc_op_ptr &node) { - // skip tensor view with single use. - int dummy_idx = 0; - auto cur_node = skip_tensor_view(node, dummy_idx); - if (is_copy_reorder(cur_node) || cur_node->isa()) { - return; - } - size_t inp_count = 0, out_count = 0; - if (has_linked_input(cur_node)) { - inp_count++; - } - for (auto &customer_use : cur_node->get_outputs()[0]->uses_) { - auto customer = customer_use.second.lock(); - if (has_linked_output(customer)) { - out_count++; - } - } - // If the buffer is linked to 2 or more inputs/outputs, need to - // insert copy. - if (inp_count + out_count >= 2) { - // if linked to input, then all output needs copy, else one output - // could directly use the buffer. - int insert_num = inp_count > 0 ? out_count : out_count - 1; - auto uses = cur_node->get_outputs()[0]->uses_; - for (auto &customer_use : uses) { - if (insert_num == 0) { - break; - } - auto customer = customer_use.second.lock(); - int out_idx = customer_use.first; - auto status = has_linked_output_and_modify_copy(out_idx, customer, - /*is_del*/ false, - /*force_insert*/ true); - if (status == linked_output) { - --insert_num; - } - } - } - vis->update_state_for_visited(cur_node); - }); - graph.reset_op_ids(); -} - -// This function aims to do reduction copying op elimination, copying reorders -// could be inserted by users or past pass. -void redundant_copy_elimination(sc_graph_t &graph) { - auto vis = op_visitor_t::bfs(); - vis.visit_graph(graph, [&](op_visitor_t *vis, const sc_op_ptr &node) { - // skip tensor view with single use. - int dummy_idx = 0; - auto cur_node = skip_tensor_view(node, dummy_idx); - if (is_copy_reorder(cur_node) || cur_node->isa()) { - return; - } - size_t inp_count = 0, out_count = 0, direct_out_count = 0; - if (has_linked_input(cur_node)) { - inp_count++; - } - auto uses = cur_node->get_outputs()[0]->uses_; - for (auto &customer_use : uses) { - auto customer = customer_use.second.lock(); - int out_idx = customer_use.first; - auto status = has_linked_output_and_modify_copy(out_idx, customer); - if (utils::is_one_of(status, linked_output, directly_linked_output)) { - out_count++; - if (status == directly_linked_output) { - direct_out_count++; - } - } - } - assert(direct_out_count <= 1); - // If the buffer is linked to single input/output, eliminate the copy, - // else if linked to 2 or more outputs, eliminate the first copy. - if ((inp_count + out_count < 2 && inp_count + out_count > 0) || - (inp_count == 0 && out_count >= 2 && direct_out_count == 0)) { - int delete_num = 1; - auto uses = cur_node->get_outputs()[0]->uses_; - for (auto &customer_use : uses) { - if (delete_num == 0) { - break; - } - auto customer = customer_use.second.lock(); - int out_idx = customer_use.first; - auto status = has_linked_output_and_modify_copy(out_idx, customer, - /*is_del*/ true, - /*force insert*/ false, - /*force delete*/ true); - if (status == linked_output) { - delete_num--; - } - } - } - vis->update_state_for_visited(cur_node); - }); - graph.reset_op_ids(); -} - -void inplace_transform(sc_graph_t &graph, const context_ptr &ctx) { - invalid_inplacement_detection(graph); - redundant_copy_elimination(graph); -} -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/transform/layout_propagation.cpp b/legacy/core/src/compiler/ir/graph/transform/layout_propagation.cpp deleted file mode 100644 index 8e603c225..000000000 --- a/legacy/core/src/compiler/ir/graph/transform/layout_propagation.cpp +++ /dev/null @@ -1,704 +0,0 @@ -/******************************************************************************* - * Copyright 2021-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include - -#include "../dynamic_dispatch_key.hpp" -#include "../fusible_op.hpp" -#include "../graph_op.hpp" -#include "../pass/pass.hpp" -#include "../tunable_op.hpp" -#include "../visitor.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -SC_MODULE(graph.layout_propagation); - -using reorder_callback_type = std::function; -using reorder_map_t = - std::unordered_map>; - -static constexpr const char *reorder_format_set_key = "temp.reorder_format_set"; -static void insert_reorder_op(sc_graph_t &graph, reorder_map_t &reorder_map, - const graph_tensor_ptr &in, size_t in_index, - const format_stride_pair &out_format_stride, - const sc_op_ptr &cur_op, bool is_input_plain, - reorder_callback_type &on_insert_reorder) { - // if we don't need to keep plain format for input op and input op tensor is - // currently not blocking, copy the blocking format - // If the output has multiply uses, choose one format for all that now to - // minimize the number of reorder, mainly for bottleneck. - if (!is_input_plain && !cur_op->is_dynamic() && - in->producer_owner_->isa() && - in->details_.get_format().is_plain() && - !in->producer_owner_->attrs_.get_or_else("keep_plain", false)) { - in->details_.set_format_and_stride(out_format_stride.first, - out_format_stride.second); - return; - } - bool is_graph_dynamic = graph.is_dynamic(); - auto &in_format = in->details_.get_format(); - auto dispatch_key = op_dispatch_key_t( - std::vector{in_format, out_format_stride.first}); - auto is_key_valid = [](const op_dispatch_key_t &key) { - return !(key.in_out_formats_[0].is_blocking() && - key.in_out_formats_[1].is_blocking()) || - key.in_out_formats_[0] == key.in_out_formats_[1]; - }(dispatch_key); - sc_op_ptr ret; - // find reorder in map, if not insert one. - auto tsr_it = reorder_map.find(in); - if (tsr_it != reorder_map.end()) { - auto &op_map = tsr_it->second; - auto op_it = op_map.find(cur_op); - if (op_it != op_map.end()) { - ret = op_it->second; - ret->get_outputs()[0]->details_.set_format_and_stride( - out_format_stride.first, out_format_stride.second); - ret->attrs_.set("out_format", out_format_stride.first); - ret->attrs_.set("out_stride", out_format_stride.second); - // update fuse attr for latest format - ret->stc_cast()->update_fuse_attr(); - // map reorder's in/out - if (is_graph_dynamic && is_key_valid) { - auto &dynamic_formats = ret->get_dispatch_key_set()->get_inner_set(); - // todo: remove internal reorder by following last tunable op - // output layout. - if (dynamic_formats.find(dispatch_key) == dynamic_formats.end()) { - ret->get_outputs()[0]->details_.add_format_candidate( - out_format_stride.first); - dynamic_formats.insert(dispatch_key); - } - ret->get_outputs()[0]->details_.add_format_candidate( - out_format_stride.first); - } - } - } - bool create_reorder = !ret; - if (create_reorder) { - ret = - graph.make("reorder", {in}, {}, - {{"internal", true}, - {"out_format", out_format_stride.first}, - {"out_stride", out_format_stride.second}, - {op_attr_key::no_fuse, // work around for conv - // graph. will be dropped - // after yijie's refactor - graph.attrs_.get_or_else("reorder_not_to_fuse", false)}}); - // map reorder's in/out - if (is_graph_dynamic && is_key_valid) { - ret->get_dispatch_key_set()->get_inner_set().insert(dispatch_key); - } - ret->get_outputs()[0]->details_.add_format_candidate( - out_format_stride.first); - reorder_map[in][cur_op] = ret; - } - // if we are in trying-mode, don't insert an reorder. Instead, - // notify that there will be a reorder to be inerted - if (on_insert_reorder) { - in->details_.set_format_and_stride(out_format_stride.first, - out_format_stride.second); - on_insert_reorder(in, out_format_stride.first); - } else { - cur_op->replace_input(in_index, ret->get_outputs()[0]); - } -} - -static void remove_inserted_reorder_op(sc_graph_t &graph, - reorder_map_t &reorder_map, - const graph_tensor_ptr &in, - const sc_op_ptr &cur_op) { - auto tsr_it = reorder_map.find(in); - if (tsr_it != reorder_map.end()) { - auto &op_map = tsr_it->second; - auto op_it = op_map.find(cur_op); - if (op_it != op_map.end()) { - op_it->second->remove(); - } - } -} - -static void update_output_formats( - std::vector &outs, - const std::vector> &out_supported_pairs, - size_t layout_choice) { - for (size_t i = 0; i < outs.size(); ++i) { - if (!out_supported_pairs.empty()) { - auto &fs_pair = out_supported_pairs[i][layout_choice]; - outs[i]->details_.set_format_and_stride(fs_pair.first, fs_pair.second); - } else if (outs[i]->details_.get_format().is_any()) { - outs[i]->details_.set_format({sc_data_format_t::get_plain_by_dims( - (int)outs[i]->details_.get_plain_dims().size())}); - } - outs[i]->details_.add_format_candidate(outs[i]->details_.get_format()); - } -} - -static void check_input_format(const std::vector &ins) { - for (auto &in : ins) { - COMPILE_ASSERT(!in->details_.get_format().is_any(), - "input format don't allow any format"); - } -} - -static void -insert_reorder_for_output_op(reorder_map_t &reorder_map, const sc_op_ptr &node, - bool use_channel_last_format, bool is_out_plain, - bool is_input_plain, bool is_graph_dynamic, - sc_graph_t &graph, - reorder_callback_type &insert_reorder_callback) { - auto given_target_formats = - node->attrs_.get_or_null>("target_formats"); - auto given_target_strides = - node->attrs_.get_or_null>("target_strides"); - if (given_target_formats) { - COMPILE_ASSERT(given_target_formats->size() == node->get_inputs().size(), - "Output op's target_formats' size should be equal to " - "number of tensors"); - } - - if (given_target_strides) { - COMPILE_ASSERT(given_target_strides->size() == node->get_inputs().size(), - "Output op's target_strides' size should be equal to " - "number of tensors"); - } - for (size_t i = 0; i < node->get_inputs().size(); ++i) { - auto &in_detail = node->get_inputs()[i]->details_; - sc_data_format_t target_format; - sc_dims target_stride; - if (given_target_formats) { - target_format = (*given_target_formats)[i]; - } else if (is_out_plain) { - if (use_channel_last_format) { - target_format = in_detail.get_format().to_channel_last(); - } else { - target_format = in_detail.get_format().to_plain(); - } - } else { - target_format = in_detail.get_format(); - } - if (given_target_strides) { - target_stride = (*given_target_strides)[i]; - } else if (is_out_plain) { - // here stride is calculated according to plain dims & format - if (use_channel_last_format) { - // permute dense stride to channel last stride - sc_dims permuted_dims = in_detail.get_plain_dims(); - size_t ndims = permuted_dims.size(); - sc_dim channel = permuted_dims[1]; - for (size_t d = 1; d < ndims - 1; ++d) { - permuted_dims[d] = permuted_dims[d + 1]; - } - permuted_dims[ndims - 1] = channel; - target_stride = logical_tensor_t::compute_dense_stride(permuted_dims); - } else { - target_stride = - logical_tensor_t::compute_dense_stride(in_detail.get_plain_dims()); - } - } else { - if (given_target_formats) { - auto dims = logical_tensor_t(target_format, in_detail.get_plain_dims(), - in_detail.dtype_) - .get_blocking_dims(); - target_stride = logical_tensor_t::compute_dense_stride(dims); - } else { - target_stride = in_detail.get_strides(); - } - } - - auto in = node->get_inputs()[i]; - COMPILE_ASSERT(!in_detail.get_format().is_any(), - "output op's input format should have a concrete " - "format, instead of any format"); - bool plain_check_failed = is_out_plain && target_format.is_blocking(); - COMPILE_ASSERT(!target_format.is_any() && !plain_check_failed, - "output op's target format should be plain or " - "permuted.") - - if (is_graph_dynamic) { - auto old_format = in_detail.get_format(); - format_stride_pair target_fs_pair(target_format, - std::move(target_stride)); - for (auto &candidate : in_detail.get_format_candidates()) { - in_detail.set_format(candidate); - insert_reorder_op(graph, reorder_map, in, i, target_fs_pair, node, - is_input_plain, insert_reorder_callback); - } - in_detail.set_format(old_format); - } else if (target_format != in_detail.get_format() || - target_stride != in_detail.get_strides()) { - format_stride_pair target_fs_pair(target_format, - std::move(target_stride)); - insert_reorder_op(graph, reorder_map, in, i, target_fs_pair, node, - is_input_plain, insert_reorder_callback); - } else if (!insert_reorder_callback) { - // if static and no need of reorder - remove_inserted_reorder_op(graph, reorder_map, in, node); - } - } -} - -static void combine_layout_and_impl_dispatch(const context_ptr &ctx, - sc_graph_t &graph) { - auto &ops = graph.ops_; - for (auto &op : ops) { - if (!op->is_dynamic()) { - continue; - } - auto impl_candidates = op->get_impl_dispatch_candidates(ctx); - if (impl_candidates.empty() || !op->is_dynamic()) { - continue; - } - auto &key_set = op->get_dispatch_key_set()->get_inner_set(); - dispatch_key_set_t::inner_set_t new_set(key_set); - for (auto key : key_set) { - for (auto &impl : impl_candidates) { - if (impl) { - key.impl_ = impl; - new_set.insert(key); - } - } - } - op->get_dispatch_key_set()->get_inner_set() = new_set; - } -} - -static bool has_channel_last_input(sc_graph_t &graph) { - for (const auto &inputs : graph.get_input_ops()) { - for (const auto &in_gt : inputs->get_outputs()) { - if (in_gt->details_.get_format().is_channel_last() && - !in_gt->details_.get_format().is_blocking()) { - return true; - } - } - } - return false; -} - -static bool has_conv_op(sc_graph_t &graph) { - auto pos = std::find_if(graph.ops_.begin(), graph.ops_.end(), - [](const sc_op_ptr &op) { - if (op->dyn_cast()) { - return true; - } else { - return false; - } - }); - return pos != graph.ops_.end(); -} - -// max times of layout tries with static shape -constexpr int STATIC_MAX_LAYOUT_TRIES = 64; - -SC_INTERNAL_API void layout_propagation(sc_graph_t &graph, - const context_ptr &ctx) { - bool use_channel_last_format = - has_channel_last_input(graph) && has_conv_op(graph) && - graph.attrs_.get_or_else( - sc_graph_t::attr_key_t::allow_channel_last_output, false); - bool is_input_plain = - graph.attrs_.get_or_else(sc_graph_t::attr_key_t::is_input_plain, true); - bool is_graph_dynamic = - graph.is_dynamic() && graph.attrs_.get_or_else("insert_reorder", true); - std::vector sorted_ops; - sorted_ops.reserve(graph.ops_.size()); - std::vector num_choices; - num_choices.resize(graph.ops_.size(), 1); - size_t total_choices = 1; - std::vector cur_choice; - cur_choice.resize(graph.ops_.size(), 0); - // the try-run will reset the input op's format. need to remember them - std::vector> format_backup; - std::vector input_ops; - - op_visitor_t vis = op_visitor_t::dfs_topology_sort(graph.ops_.size()); - // stage 1, collect all ops and the number of choices of formats - vis.visit_graph(graph, [&](op_visitor_t *vis, const sc_op_ptr &node) { - sorted_ops.emplace_back(node); - if (is_graph_dynamic) { - return; - } - if (node->isa() || node->isa()) { - // backup the input's format - std::vector backup_info; - for (auto &in : node->get_outputs()) { - backup_info.emplace_back(std::make_pair(in->details_.get_format(), - in->details_.get_strides())); - } - format_backup.emplace_back(backup_info); - input_ops.push_back(node.get()); - } - if (auto tunable_node = node->dyn_cast()) { - std::vector> in_supported_pairs, - out_supported_pairs; - bool has_config = bool(tunable_node->get_config()); - node->query_format(ctx, in_supported_pairs, out_supported_pairs); - if (!has_config) { - tunable_node->set_config(nullptr); - } - if (in_supported_pairs.empty()) { - num_choices[node->logical_op_id_] = 1; - } else { - num_choices[node->logical_op_id_] = in_supported_pairs[0].size(); - if (!graph.is_dynamic() && total_choices <= STATIC_MAX_LAYOUT_TRIES) - total_choices = total_choices * num_choices[node->logical_op_id_]; - } - SC_MODULE_INFO << node->op_name_ << '_' << node->logical_op_id_ - << " has num_choices=" - << num_choices[node->logical_op_id_]; - } else { - num_choices[node->logical_op_id_] = 1; - } - }); - - auto reset_input_fmt = [&]() { - // restore input formats - for (size_t input_cnt = 0; input_cnt < input_ops.size(); input_cnt++) { - auto input_node = input_ops[input_cnt]; - // backup the input's format - auto &fmts = format_backup[input_cnt]; - auto &outs = input_node->get_outputs(); - for (size_t i = 0; i < outs.size(); i++) { - outs[i]->details_.set_format_and_stride(fmts[i].first, fmts[i].second); - } - } - }; - reorder_callback_type insert_reorder_callback; - // the map is used to record the reorder between original tensor and op. In - // try mode, we insert an reorder but does not change the connection. If we - // work on dynamic shape or really need the reorder in static, we reserve it - // and insert it between graph tensor and op. If not, we delete it. - reorder_map_t reorder_map; - bool is_out_plain = - graph.attrs_.get_or_else(sc_graph_t::attr_key_t::is_output_plain, true); - auto do_visit = [&](const sc_op_ptr &node) { - if (node->isa()) { - insert_reorder_for_output_op( - reorder_map, node, use_channel_last_format, is_out_plain, - is_input_plain, is_graph_dynamic, graph, insert_reorder_callback); - } else if (node->isa() || node->isa()) { - update_output_formats(node->info_.outputs_, {}, 0); - } else { - std::vector> in_supported_pairs, - out_supported_pairs; - auto reset_in_out_supported_pairs = [&]() { - in_supported_pairs.clear(); - out_supported_pairs.clear(); - }; - // we need to reset the config after query_format if we need to use - // the default config for tunable_ops - bool reset_config = false; - - if (insert_reorder_callback) { - // we only need to reset the config when we are in "try" mode - auto tunable_node = node->dyn_cast(); - if (tunable_node && !tunable_node->get_config()) { - // if it is tunable and has no user-defined config - reset_config = true; - } - } - node->query_format(ctx, in_supported_pairs, out_supported_pairs); - - if (reset_config) { - node->stc_cast()->set_config(nullptr); - } - // as we will insert reorder in dynamic mode, we should record - // previous inputs. - auto inputs_backup = node->info_.inputs_; - auto &inputs = is_graph_dynamic ? inputs_backup : node->info_.inputs_; - auto &outputs = node->info_.outputs_; - check_input_format(inputs); - size_t cur_layout_choice = cur_choice[node->logical_op_id_]; - if (node->isa() || node->isa() || - node->isa() || - node->isa() || node->isa() || - node->isa() || node->isa() || - node->isa() || node->isa()) { - std::vector old_formats; - std::vector> input_format_candidates; - std::vector format_choice; - size_t total_format_choice = 1; - old_formats.reserve(inputs.size()); - input_format_candidates.reserve(inputs.size()); - for (size_t i = 0; i < inputs.size(); ++i) { - old_formats.push_back(inputs[i]->details_.get_format()); - std::set format_candidates; - if (is_graph_dynamic) { - auto &inp_candidates = inputs[i]->details_.get_format_candidates(); - format_candidates.insert(inp_candidates.begin(), - inp_candidates.end()); - } else { - format_candidates.insert(inputs[i]->details_.get_format()); - } - input_format_candidates.emplace_back(format_candidates.begin(), - format_candidates.end()); - format_choice.push_back(format_candidates.size()); - total_format_choice *= format_choice.back(); - } - for (size_t c = 0; c < total_format_choice; c++) { - size_t format_idx = c; - // for dynamic dispatch of fusible op. - std::vector dispatch_format; - dispatch_format.reserve(inputs.size() + outputs.size()); - if (is_graph_dynamic) { - for (size_t i = input_format_candidates.size(); i > 0; i--) { - inputs[i - 1]->details_.set_format( - input_format_candidates[i - 1] - [format_idx % format_choice[i - 1]]); - format_idx /= format_choice[i - 1]; - } - } else { - COMPILE_ASSERT(total_format_choice == 1, - "Static traverses one format at a time."); - } - // tunable op should return same formats in dynamic - // shape. - if (is_graph_dynamic) { - // tuanble op also need re-query as the internal layer - // rely on previous layer's output layout. - assert(cur_layout_choice == 0); - reset_in_out_supported_pairs(); - if (node->isa()) { - node->stc_cast()->set_config(nullptr); - } - node->query_format(ctx, in_supported_pairs, out_supported_pairs); - } - // need to unify input formats - // todo: should check add_op input shape, output shape - // size = max(input size), so need to enhance - if (!in_supported_pairs.empty() && !out_supported_pairs.empty()) { - for (size_t i = 0; i < inputs.size(); i++) { - auto &target_fs_pair = in_supported_pairs[i][cur_layout_choice]; - format_stride_pair in_fs_pair(inputs[i]->details_.get_format(), - inputs[i]->details_.get_strides()); - auto &format_candidates = - inputs[i]->details_.get_format_candidates(); - auto &target_format = target_fs_pair.first; - // tunable op always need reorder before op in - // dynamic. - if (is_graph_dynamic) { - if (node->isa()) { - // Reverse order here to match tunable op's - // config. - for (size_t k = in_supported_pairs[i].size(); k > 0; k--) { - // ensure that each internal tunable op - // follows last format - // clang-format off - if (!in_supported_pairs[i][k - 1] - .first.is_blocking() - || (format_candidates.find( - in_supported_pairs - [i][k - 1] - .first) - == format_candidates - .end())) { - insert_reorder_op(graph, - reorder_map, inputs[i], i, - in_supported_pairs[i] - [k - 1], - node, is_input_plain, - insert_reorder_callback); - } - // clang-format on - } - } else { - // binary fusible, if input format - // candidates contains the target format, do - // not need reorder - if (format_candidates.find(target_format) == - format_candidates.end()) { - insert_reorder_op(graph, reorder_map, inputs[i], i, - target_fs_pair, node, is_input_plain, - insert_reorder_callback); - } - } - } else if (in_fs_pair != target_fs_pair) { - if (graph.attrs_.get_or_else("insert_reorder", true)) { - // static update - insert_reorder_op(graph, reorder_map, inputs[i], i, - target_fs_pair, node, is_input_plain, - insert_reorder_callback); - } else { - // For dynamic fused op internal format - // update. - inputs[i]->details_.set_format_and_stride( - target_fs_pair.first, target_fs_pair.second); - } - } else if (!insert_reorder_callback) { - // if static and no need of reorder - remove_inserted_reorder_op(graph, reorder_map, inputs[i], node); - } - if (is_graph_dynamic) { - dispatch_format.push_back(target_fs_pair.first); - } - } - if (is_graph_dynamic) { - assert(!out_supported_pairs[0].empty()); - for (size_t j = out_supported_pairs[0].size(); j > 0; j--) { - update_output_formats(node->info_.outputs_, out_supported_pairs, - j - 1); - } - if (!node->isa()) { - // dispatch_key of tunable op is decided by - // itself. - dispatch_format.push_back(out_supported_pairs[0][0].first); - // update fusible_op's dispatch key in pass as - // it follows format of tunable op. - node->get_dispatch_key_set()->get_inner_set().insert( - dispatch_format); - } - } - } else { - COMPILE_ASSERT( - 0, "The op must support query_format: " << node->op_name_); - } - } - // reset old format - if (is_graph_dynamic) { - for (size_t i = 0; i < inputs.size(); ++i) { - inputs[i]->details_.set_format(old_formats[i]); - } - reset_in_out_supported_pairs(); - if (node->isa()) { - node->stc_cast()->set_config(nullptr); - } - node->query_format(ctx, in_supported_pairs, out_supported_pairs); - // if (node->isa()) { - // reset one unified layout for fusion. - for (size_t i = 0; i < node->get_inputs().size(); ++i) { - node->get_inputs()[i]->details_.set_format( - in_supported_pairs[i][0].first); - } - // } - } - update_output_formats(node->info_.outputs_, out_supported_pairs, - cur_layout_choice); - } else if (node->isa()) { - // split/flatten/reshape/concat/matmul/reduce/reorder/trans2d/transpose - // has itself query_format func - // relu/exp/tanh/erf/squared_root/triangle has utility - // query_format func - if (is_graph_dynamic) { - auto &format_candidates = inputs[0]->details_.get_format_candidates(); - auto old_format = inputs[0]->details_.get_format(); - for (auto &candidate : format_candidates) { - std::vector dispatch_format; - dispatch_format.reserve(inputs.size() + outputs.size()); - inputs[0]->details_.set_format(candidate); - dispatch_format.push_back(candidate); - reset_in_out_supported_pairs(); - node->query_format(ctx, in_supported_pairs, out_supported_pairs); - assert(out_supported_pairs[0].size() == 1); - update_output_formats(node->info_.outputs_, out_supported_pairs, - cur_layout_choice); - dispatch_format.push_back(out_supported_pairs[0][0].first); - node->get_dispatch_key_set()->get_inner_set().insert( - dispatch_format); - } - inputs[0]->details_.set_format(old_format); - } - reset_in_out_supported_pairs(); - node->query_format(ctx, in_supported_pairs, out_supported_pairs); - update_output_formats(node->info_.outputs_, out_supported_pairs, - cur_layout_choice); - } else { - COMPILE_ASSERT(0, "Only support fusible op/tunable op/in op/out op " - "in " - "the layout_propagation pass"); - } - } - }; - - // try all combinations of possible layouts and select the least cost - // one - if (total_choices <= STATIC_MAX_LAYOUT_TRIES && total_choices > 1) { - size_t best_cost = std::numeric_limits::max(); - std::vector best_choice = cur_choice; - for (size_t tr = 0; tr < total_choices; tr++) { - reset_input_fmt(); - size_t cost = 0; - insert_reorder_callback = [&cost](const graph_tensor_ptr &in, - const sc_data_format_t &target_format) { - size_t cur_cost = in->details_.get_blocking_byte_size(); - // give blocking format a discount as it may - // benefit performance on other ops - if (target_format.is_blocking()) { - cur_cost = cur_cost * 0.9; - } - cost += cur_cost; - }; - size_t cur_idx = tr; - for (size_t i = 0; i < cur_choice.size(); i++) { - cur_choice[i] = cur_idx % num_choices[i]; - cur_idx /= num_choices[i]; - } - for (auto &op : sorted_ops) { - do_visit(op); - } - SC_MODULE_INFO << "cost=" << cost << " " - << utils::print_vector(cur_choice); - if (!is_graph_dynamic && cost < best_cost) { - best_cost = cost; - best_choice = cur_choice; - } - } - cur_choice = std::move(best_choice); - } else { - // if there are too many choices, we can choose the default all-zero - // choice - if (total_choices > STATIC_MAX_LAYOUT_TRIES) { - SC_MODULE_WARN << "Too many choices, using default"; - } - } - - // clear the callback to let insert_reorder_op really work - insert_reorder_callback = reorder_callback_type(); - reset_input_fmt(); - // visit again to insert the reorder ops - for (auto &op : sorted_ops) { - do_visit(op); - } - - // it should be refactor to one standalone pass to finally fix constant - // value - auto vis2 = op_visitor_t::bfs(); - vis2.visit_graph(graph, [&](op_visitor_t *vis2, const sc_op_ptr &node) { - if (node->isa() && node->attrs_.has_key("temp.var")) { - auto const_op = node->dyn_cast(); - const_op->reset_const_values(); - } - }); - if (is_graph_dynamic) { - combine_layout_and_impl_dispatch(ctx, graph); - } - graph.reset_op_ids(); -} -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/transform/partial_reduce_replace.cpp b/legacy/core/src/compiler/ir/graph/transform/partial_reduce_replace.cpp deleted file mode 100644 index 3ba647913..000000000 --- a/legacy/core/src/compiler/ir/graph/transform/partial_reduce_replace.cpp +++ /dev/null @@ -1,56 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "transform.hpp" -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -void partial_reduce_replace(sc_graph_t &graph, const context_ptr &ctx) { - auto num_threads = runtime_config_t::get().get_num_threads(); - if (num_threads <= 1 || graph.is_dynamic()) { - return; - } - auto ops = graph.ops_; - for (auto &op : ops) { - if (auto rdop = op->dyn_cast()) { - auto rxax = rdop->get_rd_axis(); - bool is_first_axis = false; - auto &in_dims = rdop->get_inputs()[0]->details_.get_blocking_dims(); - sc_dim reduction_size = 1; - for (auto ax : rxax) { - if (ax == 0) { - is_first_axis = true; - } - reduction_size *= in_dims[ax]; - } - if (is_first_axis && reduction_size >= num_threads * 16) { - rdop->split_op(ctx, graph, num_threads); - } - } - } - graph.reset_op_ids(); -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/transform/permute_propagation.cpp b/legacy/core/src/compiler/ir/graph/transform/permute_propagation.cpp deleted file mode 100644 index de4e21831..000000000 --- a/legacy/core/src/compiler/ir/graph/transform/permute_propagation.cpp +++ /dev/null @@ -1,98 +0,0 @@ -/******************************************************************************* - * Copyright 2021-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#include - -#include "../fusible_op.hpp" -#include "../graph_op.hpp" -#include "../tunable_op.hpp" -#include "../visitor.hpp" - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -static bool check_have_any(const sc_graph_t &graph) { - bool have_any = false; - for (const auto &op : graph.ops_) { - for (auto &in : op->get_inputs()) { - if (in->details_.get_format().is_any()) { - have_any = true; - break; - } - } - } - return have_any; -} - -static void insert_reorder_op(sc_graph_t &graph, graph_tensor_ptr in, - size_t in_index, - const sc_data_format_t &out_format, - const sc_op_ptr &cur_op) { - auto ret = - graph.make("reorder", {std::move(in)}, {}, - {{"out_format", out_format}, - {"internal", true}, - {op_attr_key::no_fuse, // walk around for conv graph. will - // be dropped after yijie's refactor - graph.attrs_.get_or_else("reorder_not_to_fuse", false)}}); - cur_op->replace_input(in_index, ret->get_outputs()[0]); -} - -static bool need_to_reorder(const sc_data_format_t &format) { - bool needed = false; - for (size_t i = 0; i < sc_data_format_kind_t::MAX_DIMS; ++i) { - if (i > 0) { - if ((format.format_code_.get(i) - format.format_code_.get(i - 1)) < 0) { - needed = true; - break; - } - } - } - return needed; -} - -void permute_propagation(sc_graph_t &graph, const context_ptr &ctx) { - op_visitor_t vis = op_visitor_t::dfs_topology_sort(graph.ops_.size()); - vis.visit_graph(graph, [&](op_visitor_t *vis, const sc_op_ptr &node) { - for (size_t i = 0; i < node->get_inputs().size(); ++i) { - auto in = node->get_inputs()[i]; - if (in->details_.get_format().is_any()) { - in->details_.set_format(sc_data_format_t::get_plain_by_dims( - (int)in->details_.get_plain_dims().size())); - } else if (in->details_.get_format().is_plain() && - need_to_reorder(in->details_.get_format())) { - // todo: should query the Op if it accepts a permuted layout. - // TODO(yifei): consider remove this entire pass - // since we will not run into this condition - insert_reorder_op(graph, in, i, - sc_data_format_t::get_plain_by_dims( - (int)in->details_.get_plain_dims().size()), - node); - } - } - }); - graph.reset_op_ids(); - COMPILE_ASSERT( - !check_have_any(graph), - "After permute_propagation, each op's graph_tensor should have no " - "any format"); -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/transform/pre_padding.cpp b/legacy/core/src/compiler/ir/graph/transform/pre_padding.cpp deleted file mode 100644 index d0daf2c39..000000000 --- a/legacy/core/src/compiler/ir/graph/transform/pre_padding.cpp +++ /dev/null @@ -1,141 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include -#include -#include -#include -#include -#include - -#include "../fusible_op.hpp" -#include "../graph_op.hpp" -#include "../pass/pass.hpp" -#include "../visitor.hpp" -#include "transform.hpp" - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -SC_MODULE(graph.pre_padding); - -void pre_padding(sc_graph_t &graph, const context_ptr &ctx) { - if (graph.is_dynamic()) { - return; - } - COMPILE_ASSERT(!graph.is_dynamic(), "PrePadding only support static cases"); - auto visitor = op_visitor_t::dfs_topology_sort(graph.ops_.size()); - auto is_zero_paddings = [](sc_dims &paddings) { - bool zero_paddings = true; - for (auto &p : paddings) { - if (p != 0) { - zero_paddings = false; - break; - } - } - return zero_paddings; - }; - visitor.visit_graph(graph, [&](op_visitor_t *visitor, const sc_op_ptr &node) { - // if current node is a conv op with paddings, - // insert a padding op before current node - - if (node->isa()) { - auto data_plain_dims = node->info_.inputs_[0]->details_.get_plain_dims(); - auto weight_plain_dims = - node->info_.inputs_[1]->details_.get_plain_dims(); - bool is_amx_dtype = - ops::is_amx_dtype(ctx, node->get_inputs()[0]->details_.dtype_); - sc_dim groups = node->attrs_.get_or_else("groups", 1); - bool is_dw_brdgmm = groups > 1 && weight_plain_dims.size() >= 5 && - data_plain_dims.size() >= 5 && - 1 == weight_plain_dims[1] && 1 == data_plain_dims[2]; - auto padding_value = node->attrs_.get_or_else("padding_value", 0); - // TODO(xurui) - // Only support extract padding op from 2d conv for now. - if (((data_plain_dims.size() != 4) || !is_amx_dtype || - (is_dw_brdgmm && padding_value == 0))) { - return; - } - // Only apply to inference - bool is_weight_constant = - node->get_inputs()[1]->producer_owner_->isa() || - node->get_inputs()[1]->producer_owner_->attrs_.get_or_else( - "constant", const_kind::not_const) || - node->get_inputs()[1]->attrs_.get_or_else("constant", - const_kind::not_const); - if (!is_weight_constant) { - return; - } - if (node->attrs_.get_or_else("use_rl", ops::rl_kind::NO_LOWERING) == - ops::rl_kind::FULL_LOWERING) - return; - - auto pads_begin = node->attrs_.has_key("pads_begin") - ? node->attrs_.get("pads_begin") - : node->attrs_.get("paddings"); - - auto pads_end = node->attrs_.has_key("pads_end") - ? node->attrs_.get("pads_end") - : node->attrs_.get("paddings"); - - if (is_zero_paddings(pads_begin)) { - return; - } - - auto parent_node = node->get_inputs()[0]->producer_owner_; - - if (parent_node->isa()) { - return; - } - auto padding_node = graph.make("padding", {node->get_inputs()[0]}, {}, - {{"pads_begin", pads_begin}, - {"pads_end", pads_end}, - {"padding_value", padding_value}}); - if (node->get_inputs()[0]->details_.get_plain_dims()[0] == 1) { - padding_node->attrs_.set(op_attr_key::break_post_fuse, true); - } - - // clear paddings from original conv node - node->attrs_.set("pads_begin", sc_dims{0}); - node->attrs_.set("pads_end", sc_dims{0}); - node->attrs_.set("paddings", sc_dims{0}); - - visitor->update_state_for_visited(node); - auto conv_new = - graph.make(node->op_name_, - {padding_node->get_outputs()[0], node->get_inputs()[1]}, - node->get_outputs(), node->attrs_); - - // Copy configs from old node to new node - // TODO(xurui) A better way for doing this is put the config - // setting step after all grpah rewrite so that we do not - // need to copy config here - if (auto tunable_node = node->dyn_cast()) { - conv_new->dyn_cast()->set_config( - tunable_node->get_config()); - } - node->remove(); - visitor->update_state_for_visited(padding_node); - } - }); - graph.reset_op_ids(); -} -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/transform/quantization/annotate_fuse_break.cpp b/legacy/core/src/compiler/ir/graph/transform/quantization/annotate_fuse_break.cpp deleted file mode 100644 index 71f59e2ae..000000000 --- a/legacy/core/src/compiler/ir/graph/transform/quantization/annotate_fuse_break.cpp +++ /dev/null @@ -1,115 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "compiler/ir/graph/quantization/quantize_op.hpp" -#include "ops/fusible/binary_elemwise.hpp" -#include "ops/fusible/unary_elemwise.hpp" -#include "ops/graph_convolution.hpp" -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace quantize { -/* -Add break_post_fuse after quantize with the following pattern -Convolution/[Convolution + BiasAdd] - | - Quantize (break_post_fuse) - | - Dequantize - | - Add --------------OR------------ - Dequantize - | - Add - | - Relu - | - Quantize -*/ -void annotate_fusion_break(sc_graph_t &mgr, const context_ptr &ctx) { - if (!mgr.attrs_.get_or_else(sc_graph_t::attr_key_t::quantize, false)) - return; - for (auto &op : mgr.ops_) { - if (op->isa()) { - for (const auto &in : op->get_inputs()) { - auto prev1 = in->producer_owner_; - if (prev1->isa()) { - auto prev2 = prev1->get_inputs()[0]->producer_owner_; - if (prev2->isa()) { - auto prev3 = prev2->get_inputs()[0]->producer_owner_; - if (prev3->isa() || prev3->isa()) { - prev1->attrs_[op_attr_key::break_pre_fuse] = true; - } - } - } - } - } - if (op->isa()) { - auto prev1 = op->get_inputs()[0]->producer_owner_; - if (prev1->isa()) { - auto prev2 = prev1->get_inputs()[0]->producer_owner_; - if (prev2->isa()) { - auto prev3_lhs = prev2->get_inputs()[0]->producer_owner_; - auto prev3_rhs = prev2->get_inputs()[1]->producer_owner_; - if (prev3_lhs->isa() || - prev3_rhs->isa()) { - for (auto &user : op->get_outputs()[0]->uses_) { - if (auto deq = - user.second->dyn_cast()) { - deq->attrs_[op_attr_key::break_pre_fuse] = true; - } - } - } - } - } - } - // auto break qadd branch - else if (op->isa()) { - bool is_bf16 = - op->get_outputs()[0]->details_.dtype_ == sc_data_etype::BF16; - sc_op *prev = nullptr; - for (size_t i = 0; i < op->get_inputs().size(); i++) { - auto producer = op->get_inputs()[i]->producer_owner_; - if (is_bf16 && producer->isa()) - prev = producer; - else if (!is_bf16 && producer->isa()) - prev = producer; - } - if (!prev) - continue; - if (is_bf16) { - prev = prev->get_inputs()[0]->producer_owner_; - } - if (!prev->isa()) - continue; - auto forked_op = prev->get_inputs()[0]->producer_owner_; - if (!forked_op->isa() || - forked_op->get_outputs()[0]->uses_.size() != 2) - continue; - // set deq op break_pre_fuse - prev->attrs_[op_attr_key::break_pre_fuse] = true; - } - } -} -} // namespace quantize -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/transform/quantization/graph_reschedule.cpp b/legacy/core/src/compiler/ir/graph/transform/quantization/graph_reschedule.cpp deleted file mode 100644 index e6664bdd9..000000000 --- a/legacy/core/src/compiler/ir/graph/transform/quantization/graph_reschedule.cpp +++ /dev/null @@ -1,338 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace quantize { -bool reschedule_forbid_op(const sc_op_ptr &node) { - // transpose could be processed as it only changes the order of dim. - // tensor_view without changing the non-broadcast(non-one) axis could also - // be processed. - return ((node->isa() || - (node->isa() && - !dynamic_cast(node.get()) - ->is_only_expand_or_penetrate()) || - node->isa() || node->isa() || - node->isa()) && - node->attrs_.get_or_else(attr_keys::per_channel, false) && - !node->attrs_.get_or_else("allow_quantize_reschedule", false)) || - node->isa(); -} -void dequantize_elimination(sc_graph_t &mgr, const context_ptr &ctx) { - op_visitor_t vis = op_visitor_t::bfs(); - vis.visit_graph(mgr, [&](op_visitor_t *vis, const sc_op_ptr &node) { - if (node->isa() || node->isa()) { - auto dequantize_input = node->info_.inputs_[0]; - auto dequantize_output = node->get_outputs()[0]; - vis->update_state_for_visited(node); - - auto cld_nodes = dequantize_output->uses_; - std::vector node_to_remove = {node}; - // whether replace inputs with next node according to two op nodes - // back check when int8-bf16 mixed type. - // For the case like: - // dequantize - // / - // matmul cast - // \ / - // add - if (cld_nodes.size() == 1) { - auto cast_op = cld_nodes[0].second; - if (cast_op->isa() && - cast_op->attrs_.get_or_else(attr_keys::mixed_dtype, false)) { - vis->update_state_for_visited(cast_op); - node_to_remove.emplace_back(cast_op); - cld_nodes = cast_op->get_outputs()[0]->uses_; - } - } - int use_count = cld_nodes.size(); - for (auto &cld_node : cld_nodes) { - if ((cld_node.second->isa() && - cld_node.second->dyn_cast() - ->should_quantized_ && - !reschedule_forbid_op(cld_node.second))) { - cld_node.second->replace_input(cld_node.first, dequantize_input); - use_count--; - } - } - if (!use_count) { - std::for_each(node_to_remove.begin(), node_to_remove.end(), - [](const sc_op_ptr &op) { op->remove(); }); - } - } - }); - mgr.reset_op_ids(); -} - -void insert_back_dequantize(sc_graph_t &mgr, const context_ptr &ctx) { - op_visitor_t vis = op_visitor_t::dfs_topology_sort(mgr.ops_.size()); - vis.visit_graph(mgr, [&](op_visitor_t *vis, const sc_op_ptr &node) { - if (node->dyn_cast()) { - if (node->isa()) { - // create a new quantized node every time may confuse - // `to_visit` list. - auto may_quantize_node = node->dyn_cast(); - if (!may_quantize_node->should_quantized_ || - may_quantize_node->is_quantized_) { - return; - } - if (may_quantize_node->should_quantized_ && - node->get_inputs()[0]->details_.dtype_.is_etype( - sc_data_etype::BF16)) { - may_quantize_node->is_quantized_ = true; - vis->update_state_for_visited(node); - return; - } - node->get_outputs()[0]->details_.dtype_.type_code_ = sc_data_etype::S32; - may_quantize_node->is_quantized_ = true; - node->op_name_ = "quantized_" + node->op_name_; - assert((node->attrs_.has_key(attr_keys::data_scales) && - node->attrs_.has_key(attr_keys::weight_scales)) || - (node->attrs_.has_key(attr_keys::dyn_data_scales) && - node->attrs_.has_key(attr_keys::dyn_weight_scales))); - bool is_dyn_quan = node->attrs_.has_key(attr_keys::dyn_data_scales); - auto weight_scales = node->attrs_.get_or_else(attr_keys::weight_scales, - std::vector()); - int output_channel_axis; - if (node->attrs_.has_key(attr_keys::output_channel_axis)) { - output_channel_axis = - node->attrs_.get(attr_keys::output_channel_axis); - } else if (node->isa() && - node->attrs_.has_key("data_format")) { - // infer output_channel_axis based on op attributes - auto data_format = node->attrs_.get("data_format"); - auto ndims = node->get_outputs()[0]->details_.get_plain_dims().size(); - output_channel_axis = data_format == "NCX" ? 1 : ndims - 1; - } else { - output_channel_axis = - node->attrs_.get(attr_keys::weight_channel_axis); - if (weight_scales.size() > 1 && - node->attrs_.has_key(attr_keys::weight_channel_axis)) { - SC_WARN << "Weight_channel_axis is specified but " - "output_channel_axis not specified. " - "Assuming output_channel_axis == " - "weight_channel_axis"; - } - } - auto tunable_op_consumers = node->get_outputs()[0]->uses_; - for (auto &child : tunable_op_consumers) { - auto cur_child = child; - auto cur_parent = node; - while (cur_child.second->dyn_cast() && - !cur_child.second->isa() - // reserve dynamic version here for debug. - // && - // !cur_child.second->isa()) - // { - && cur_child.second->get_outputs().size() == 1) { - cur_child.second->replace_input(cur_child.first, - cur_parent->get_outputs()[0]); - cur_parent = cur_child.second; - cur_child = cur_child.second->get_outputs()[0]->uses_[0]; - cur_parent->get_outputs()[0]->details_.dtype_.type_code_ = - sc_data_etype::S32; - if (cur_parent->get_outputs()[0]->uses_.size() > 1) { - break; - } - } - auto cur_parent_uses = - cur_parent == node - ? std::vector>{child} - : cur_parent->get_outputs()[0]->uses_; - // TODO(yifei): overcome the constraints here - for (const auto &use : cur_parent_uses) { - COMPILE_ASSERT( - use.second->isa() || - !use.second->dyn_cast(), - "may_quantize op with multiple consumers " - "shouldn't have any of may_quantize " - "consumers (except concat)."); - } - sc_op_ptr dequantize_node; - if (is_dyn_quan) { - assert(node->attrs_.has_key(attr_keys::dyn_data_scales) && - (node->attrs_.has_key(attr_keys::dyn_weight_scales) || - node->attrs_.has_key(attr_keys::weight_scales))); - auto data_scales_gt = - node->attrs_.get(attr_keys::dyn_data_scales); - auto weight_scales_gt = node->attrs_.get_or_else( - attr_keys::dyn_weight_scales, graph_tensor_ptr()); - sc_op_ptr out_scales; - if (weight_scales_gt) { - out_scales = - mgr.make("mul", {data_scales_gt, weight_scales_gt}, {}, {}); - } else { - auto wei_scale_const = mgr.make( - "constant", {}, {}, - {{"values", std::make_shared(weight_scales)}, - {"dtype", datatypes::f32}, - {"plain_dims", - sc_dims{static_cast(weight_scales.size())}}, - {"format", sc_data_format_t()}}); - auto &wei_details = wei_scale_const->get_outputs()[0]->details_; - out_scales = mgr.make( - "mul", {data_scales_gt, wei_scale_const->get_outputs()[0]}, - {std::make_shared( - nullptr, wei_details.get_format(), - wei_details.get_plain_dims(), wei_details.dtype_)}, - {}); - } - bool is_per_channel = - out_scales->get_outputs()[0]->details_.get_plain_dims()[0] > 1; - dequantize_node = mgr.make( - "dynamic_dequantize", - {cur_parent->get_outputs()[0], out_scales->get_outputs()[0]}, - {}, - {{attr_keys::quan_dtype, datatypes::f32}, - {attr_keys::per_channel, is_per_channel}, - {attr_keys::channel_axis, output_channel_axis}}); - } else { - auto data_scales = - node->attrs_.get>(attr_keys::data_scales); - auto output_scales = - math_utils::vector_mul(data_scales, weight_scales); - dequantize_node = - mgr.make("dequantize", cur_parent->get_outputs(), - std::vector{}, - {{attr_keys::quan_dtype, datatypes::f32}, - {attr_keys::scales, output_scales}, - {attr_keys::channel_axis, output_channel_axis}}); - } - if (node->attrs_.get_or_else(attr_keys::mixed_dtype, false)) { - dequantize_node = mgr.make("cast", dequantize_node->get_outputs(), - std::vector{}, - {{"dtype", datatypes::bf16}}); - } - for (const auto &use : cur_parent_uses) { - use.second->replace_input(use.first, - dequantize_node->get_outputs()[0]); - } - vis->update_state_for_visited(dequantize_node); - } - } else if (!reschedule_forbid_op(node)) { - // align output datatype with input - for (auto &out : node->get_outputs()) { - if (node->isa()) { - node->get_outputs()[0]->details_.dtype_.type_code_ = - sc_data_etype::F32; - } else { - out->details_.dtype_ = node->get_inputs()[0]->details_.dtype_; - } - // insert dequantize if last op of pattern is output op, - // for pattern like `qua->deq->reshape->output` - std::vector> uses = out->uses_; - for (auto &use : uses) { - auto cld_op = use.second.lock(); - if ((!(cld_op->dyn_cast() && - cld_op->get_outputs().size() == 1) || - reschedule_forbid_op(cld_op)) && - ((node->attrs_.has_key(attr_keys::scales) && - node->attrs_.has_key(attr_keys::zero_points)) || - (node->attrs_.has_key(attr_keys::dyn_scales)))) { - bool is_dyn_quan = node->attrs_.has_key(attr_keys::dyn_scales); - sc_op_ptr dequantize_node; - if (is_dyn_quan) { - auto &scales = - node->attrs_.get(attr_keys::dyn_scales); - auto &zero_points = node->attrs_.get( - attr_keys::dyn_zero_points); - std::vector ins = {out, scales}; - if (zero_points) { - ins.emplace_back(zero_points); - } - dequantize_node = - mgr.make("dynamic_dequantize", ins, {}, node->attrs_); - } else { - dequantize_node = - mgr.make("dequantize", {out}, - std::vector{}, node->attrs_); - } - if (node->attrs_.get_or_else(attr_keys::mixed_dtype, false)) { - dequantize_node = - mgr.make("cast", dequantize_node->get_outputs(), - std::vector{}, - {{"dtype", datatypes::bf16}}); - } - cld_op->replace_input(use.first, - dequantize_node->get_outputs()[0]); - vis->update_state_for_visited(dequantize_node); - } - } - } - } - } - }); - mgr.reset_op_ids(); -} - -/** - * reschedule graph by add/remove dequantize op and replace calculate op - * with quantize calculate op. - * - * for dequantize op, as quantize information is propagated by info - * propagation, if it has no non-quantize uses, we remove the node. - * before: - * | - * dequantize_op - * | - * conv_op - * after: - * | - * | - * conv_op - * | - * - * for calculate op with quantize info, we replace it with a quantized - * one and add a dequantize node after. before: - * | - * conv_op(with quantize info) - * | - * after: - * | - * qconv_op(u8/s8 in, s32 out) - * | - * dequantize_op - * | - * - * */ -void graph_reschedule(sc_graph_t &mgr, const context_ptr &ctx) { - if (!mgr.attrs_.get_or_else(sc_graph_t::attr_key_t::quantize, false)) - return; - dequantize_elimination(mgr, ctx); - insert_back_dequantize(mgr, ctx); -} -} // namespace quantize -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/transform/quantization/info_propagation.cpp b/legacy/core/src/compiler/ir/graph/transform/quantization/info_propagation.cpp deleted file mode 100644 index eb0f9da01..000000000 --- a/legacy/core/src/compiler/ir/graph/transform/quantization/info_propagation.cpp +++ /dev/null @@ -1,380 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace quantize { - -static std::unordered_set data_wei_op_set = { - "conv_fwd_core", "matmul_core", "managed_matmul_core"}; -// quantize awared nodes search starts from dequantize node and end in -// quantize node. -static std::vector> -find_quantize_aware_nodes(const sc_op_ptr &node) { - if (node == nullptr) { - return std::vector>(); - } - std::vector> aware_nodes; - for (auto &child_lt : node->get_outputs()) { - for (const auto &child_op : child_lt->uses_) { - if (child_op.second->isa()) { - // Concat op has multiple inputs whose data types must be same. - // To assure this constraint, we do not consider it as quantize - // aware node. - continue; - } - if ((child_op.second->dyn_cast() && - child_op.second->attrs_.get_or_else(sc_graph_t::attr_key_t::quantize, - true)) || - child_op.second->isa()) { - aware_nodes.emplace_back(child_op); - } - } - } - return aware_nodes; -} - -template -void has_key_and_set(any_map_t &attrs, const std::string &key, - const sc_op_ptr &aware_node, - const std::string &new_key = std::string()) { - if (attrs.has_key(key)) { - aware_node->attrs_.set(new_key.empty() ? key : new_key, attrs.get(key)); - } -} - -static void -propagate_quantize_info(const sc_op_ptr &quantize_node, - const std::pair &aware_node) { - assert((!(aware_node.second->attrs_.has_key(attr_keys::data_scales) && - aware_node.second->attrs_.has_key(attr_keys::dyn_data_scales)) || - !(aware_node.second->attrs_.has_key(attr_keys::weight_scales) && - aware_node.second->attrs_.has_key(attr_keys::dyn_weight_scales))) && - "aware node has been set a quantized info"); - if (data_wei_op_set.find(aware_node.second->op_name_) != - data_wei_op_set.end()) { - const auto qinfos = get_quantize_info_from_attrs(quantize_node->attrs_); - std::string prefix, cur_dyn_scales, cur_dyn_zero_points; - switch (aware_node.first) { - case 0: - prefix = "data_"; - cur_dyn_scales = attr_keys::dyn_data_scales; - cur_dyn_zero_points = attr_keys::dyn_data_zero_points; - break; - case 1: - prefix = "weight_"; - cur_dyn_scales = attr_keys::dyn_weight_scales; - cur_dyn_zero_points = attr_keys::dyn_weight_zero_points; - break; - default: - assert(0 && "invalid tensor type!"); - break; - }; - aware_node.second->attrs_.set(prefix + attr_keys::scales, qinfos.scales_); - aware_node.second->attrs_.set(prefix + attr_keys::zero_points, - qinfos.zero_points_); - aware_node.second->attrs_.set(prefix + attr_keys::per_channel, - qinfos.per_channel_); - aware_node.second->attrs_.set(prefix + attr_keys::channel_axis, - qinfos.channel_axis_); - // dynamic quantize - if (quantize_node->isa()) { - auto &inputs = quantize_node->get_inputs(); - aware_node.second->attrs_.set(cur_dyn_scales, inputs[1]); - if (inputs.size() == 3) { - aware_node.second->attrs_.set(cur_dyn_zero_points, inputs[2]); - } - } else { - has_key_and_set(quantize_node->attrs_, - attr_keys::dyn_scales, - aware_node.second, cur_dyn_scales); - has_key_and_set(quantize_node->attrs_, - attr_keys::dyn_zero_points, - aware_node.second, cur_dyn_zero_points); - } - if (quantize_node->isa() || - quantize_node->isa() || - quantize_node->attrs_.get_or_else(attr_keys::may_quantize, false)) { - if (!aware_node.second->attrs_.has_key(attr_keys::may_quantize)) { - aware_node.second->attrs_.set(attr_keys::may_quantize, true); - } - if (aware_node.second->attrs_.get(attr_keys::may_quantize)) { - aware_node.second->dyn_cast() - ->should_quantized_ = true; - } - } else { - aware_node.second->attrs_.set(attr_keys::may_quantize, false); - aware_node.second->dyn_cast() - ->should_quantized_ = false; - } - } else { - // dynamic quantize - if (quantize_node->isa()) { - auto &inputs = quantize_node->get_inputs(); - aware_node.second->attrs_.set(attr_keys::dyn_scales, inputs[1]); - aware_node.second->attrs_.set(attr_keys::dyn_zero_points, - inputs.size() == 3 ? inputs[2] : nullptr); - } else { - has_key_and_set( - quantize_node->attrs_, attr_keys::dyn_scales, aware_node.second); - has_key_and_set( - quantize_node->attrs_, attr_keys::dyn_zero_points, aware_node.second); - } - // static quantize - has_key_and_set>(quantize_node->attrs_, - attr_keys::scales, aware_node.second); - has_key_and_set>( - quantize_node->attrs_, attr_keys::zero_points, aware_node.second); - // common propagation. - has_key_and_set(quantize_node->attrs_, attr_keys::per_channel, - aware_node.second); - has_key_and_set(quantize_node->attrs_, attr_keys::channel_axis, - aware_node.second); - bool is_transpose = aware_node.second->isa(); // NOLINT - if (is_transpose && - quantize_node->attrs_.has_key(attr_keys::channel_axis)) { - int channel_axis = - quantize_node->attrs_.get(attr_keys::channel_axis); - std::unordered_map axis_map; - auto order = aware_node.second->attrs_.get>("order"); - for (size_t i = 0; i < order.size(); ++i) { - axis_map[order[i]] = i; - } - aware_node.second->attrs_.set(attr_keys::channel_axis, - axis_map[channel_axis]); - } - has_key_and_set(quantize_node->attrs_, attr_keys::mixed_dtype, - aware_node.second); - has_key_and_set>( - quantize_node->attrs_, attr_keys::data_scales, aware_node.second); - has_key_and_set>( - quantize_node->attrs_, attr_keys::weight_scales, aware_node.second); - has_key_and_set>( - quantize_node->attrs_, attr_keys::data_zero_points, aware_node.second); - has_key_and_set>(quantize_node->attrs_, - attr_keys::weight_zero_points, - aware_node.second); - has_key_and_set( - quantize_node->attrs_, attr_keys::dyn_data_scales, aware_node.second); - has_key_and_set( - quantize_node->attrs_, attr_keys::dyn_weight_scales, aware_node.second); - has_key_and_set(quantize_node->attrs_, - attr_keys::dyn_data_zero_points, - aware_node.second); - has_key_and_set(quantize_node->attrs_, - attr_keys::dyn_weight_zero_points, - aware_node.second); - has_key_and_set(quantize_node->attrs_, attr_keys::weight_channel_axis, - aware_node.second); - if (quantize_node->isa() || - quantize_node->isa() || - quantize_node->attrs_.get_or_else(attr_keys::may_quantize, false)) { - aware_node.second->attrs_.set(attr_keys::may_quantize, true); - if (auto may_quantize_node = - aware_node.second->dyn_cast()) { - may_quantize_node->should_quantized_ = true; - } - } - } -} - -static void check_and_set_mixed_dtype(const sc_op_ptr &cast_node) { - assert(cast_node->isa()); - auto &attrs = cast_node->attrs_; - // if after tunable op - if (attrs.has_key(attr_keys::data_scales) || - attrs.has_key(attr_keys::dyn_data_scales)) { - assert((attrs.has_key(attr_keys::weight_scales) || - attrs.has_key(attr_keys::dyn_weight_scales)) && - !(attrs.has_key(attr_keys::scales) || - attrs.has_key(attr_keys::dyn_scales))); - } else if (attrs.has_key(attr_keys::scales) || - attrs.has_key(attr_keys::dyn_scales)) { - assert(attrs.get(attr_keys::quan_dtype) == datatypes::bf16); - attrs.set(attr_keys::mixed_dtype, true); - } -} - -// Do cast u8/s8 => s32 for dynamic zero points -void change_dyn_zp_to_s32(sc_graph_t &mgr, const context_ptr &ctx) { - op_visitor_t vis = op_visitor_t::dfs_topology_sort(mgr.ops_.size()); - vis.visit_graph(mgr, [&](op_visitor_t *vis, const sc_op_ptr &node) { - if (node->isa() || - node->isa()) { - if (node->get_inputs().size() == 3 && - node->get_inputs()[2]->details_.dtype_ != datatypes::s32) { - auto &zp = node->get_inputs()[2]; - auto casts32 = mgr.make("cast", {zp}, {}, {{"dtype", datatypes::s32}}); - node->replace_input(2, casts32->get_outputs()[0]); - } - } - }); - mgr.reset_op_ids(); -} - -// Currenly we change u8 to s8 for weight -void change_weight_u8_to_s8(sc_graph_t &mgr, const context_ptr &ctx) { - if (ctx->use_amx()) { - return; - } - op_visitor_t vis = op_visitor_t::dfs_topology_sort(mgr.ops_.size()); - vis.visit_graph(mgr, [&](op_visitor_t *vis, const sc_op_ptr &node) { - if (node->isa() || node->isa()) { - bool dyn_quan_cur = node->isa(); - if (node->get_inputs()[0]->details_.dtype_ == datatypes::u8) { - bool need_s8 = false; - for (auto cld : node->get_outputs()[0]->uses_) { - while (!cld.second->isa() && - !cld.second->isa() && - cld.second->get_outputs().size() == 1) { - cld = cld.second->get_outputs()[0]->uses_[0]; - } - if (data_wei_op_set.find(cld.second->op_name_) != - data_wei_op_set.end() && - cld.first == 1) { - need_s8 = true; - break; - } - } - if (need_s8) { - auto *node_before = node->get_inputs()[0]->producer_owner_; - if (node_before->isa() || - node_before->isa()) { - bool dyn_quan_before = node_before->isa(); - assert(node_before->attrs_.get_or_else( - attr_keys::quan_dtype, datatypes::s8) == datatypes::u8); - node_before->attrs_.get(attr_keys::quan_dtype) = - datatypes::s8; - node->get_inputs()[0]->details_.dtype_ = datatypes::s8; - assert(node->attrs_.get_or_else(attr_keys::quan_dtype, - datatypes::s8) == datatypes::f32); - node->attrs_.get(attr_keys::quan_dtype) = - datatypes::f32; - if (dyn_quan_before) { - assert(node_before->get_inputs().size() == 3); - auto &zp = node_before->get_inputs()[2]; - int value = 128; - auto const_128 = mgr.make( - "constant", {}, {}, - {{"dtype", datatypes::s32}, - {"format", sc_data_format_t()}, - {"values", - std::make_shared(&value, sizeof(int))}, - {"plain_dims", sc_dims{1}}}); - auto shift_128 = - mgr.make("sub", {zp, const_128->get_outputs()[0]}, {}, {}); - node_before->replace_input(2, shift_128->get_outputs()[0]); - node->replace_input(2, shift_128->get_outputs()[0]); - } else { - assert( - node_before->attrs_ - .get_or_else(attr_keys::zero_points, std::vector()) - .size() == 1); - assert( - node->attrs_ - .get_or_else(attr_keys::zero_points, std::vector()) - .size() == 1); - node_before->attrs_.get>( - attr_keys::zero_points)[0] -= 128; - node->attrs_.get>(attr_keys::zero_points)[0] -= - 128; - } - } else { - auto *node_before = node->get_inputs()[0]->producer_owner_; - const auto &qinfos = get_quantize_info_from_attrs(node->attrs_); - auto new_zero_point = qinfos.zero_points_[0] - 128; - auto casts32 = mgr.make("cast", node_before->get_outputs(), {}, - {{"dtype", datatypes::s32}}); - auto const128 = mgr.make( - "constant", {}, {}, - {{"values", - std::make_shared(std::vector{128})}, - {attr_keys::quan_dtype, datatypes::s32}, - {"plain_dims", sc_dims{1}}}); - auto sub128 = mgr.make( - "sub", {casts32->get_outputs()[0], const128->get_outputs()[0]}, - {}, {}); - auto casts8 = mgr.make("cast", sub128->get_outputs(), {}, - {{"dtype", datatypes::s8}}); - sc_op_ptr deq; - if (dyn_quan_cur) { - assert(node->get_inputs().size() == 3); - auto &scales = node->get_inputs()[1]; - auto &zp = node->get_inputs()[2]; - auto shift_128 = - mgr.make("sub", {zp, const128->get_outputs()[0]}, {}, {}); - deq = mgr.make("dynamic_dequantize", - {casts8->get_outputs()[0], scales, - shift_128->get_outputs()[0]}, - {}, {{attr_keys::quan_dtype, datatypes::f32}}); - } else { - deq = mgr.make( - "dequantize", casts8->get_outputs(), {}, - {{attr_keys::quan_dtype, datatypes::f32}, - {attr_keys::scales, qinfos.scales_}, - {attr_keys::zero_points, std::vector{new_zero_point}}}); - } - node->replace_uses_with_and_remove(deq); - vis->update_state_for_visited(deq); - } - } - } - } - }); - mgr.reset_op_ids(); -} - -// do two things: infer input tensor is data/weight;transfer quantize info -// to calculation op -SC_INTERNAL_API void quantize_info_propagation(sc_graph_t &mgr, - const context_ptr &ctx) { - if (!mgr.attrs_.get_or_else(sc_graph_t::attr_key_t::quantize, false)) - return; - change_dyn_zp_to_s32(mgr, ctx); - change_weight_u8_to_s8(mgr, ctx); - op_visitor_t vis = op_visitor_t::dfs_topology_sort(mgr.ops_.size()); - vis.visit_graph(mgr, [&](op_visitor_t *vis, const sc_op_ptr &node) { - if (node->isa() || node->isa() || - (node->isa() && !node->isa()) || - node->isa()) { - if (node->isa()) { - check_and_set_mixed_dtype(node); - } - auto aware_ops = find_quantize_aware_nodes(node); - for (const auto &aware_op : aware_ops) { - propagate_quantize_info(node, aware_op); - } - } - }); -} -} // namespace quantize -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/transform/quantization/op_rewrite.cpp b/legacy/core/src/compiler/ir/graph/transform/quantization/op_rewrite.cpp deleted file mode 100644 index 56eb19da2..000000000 --- a/legacy/core/src/compiler/ir/graph/transform/quantization/op_rewrite.cpp +++ /dev/null @@ -1,93 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#include -#include -#include -#include -#include -#include -#include -#include -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace quantize { -void calculate_op_compensation(sc_graph_t &mgr, const context_ptr &ctx) { - if (!mgr.attrs_.get_or_else(sc_graph_t::attr_key_t::quantize, false)) - return; - op_visitor_t vis = op_visitor_t::dfs(); - vis.visit_graph(mgr, [&](op_visitor_t *vis, const sc_op_ptr &node) { - if (auto qnode = node->dyn_cast()) { - if (qnode->is_quantized_ && qnode->need_compensation_) { - std::vector> uses; - for (auto &out : node->get_outputs()) { - uses.insert(uses.end(), out->uses_.begin(), out->uses_.end()); - } - sc_op_ptr compensation_qnode = qnode->do_compensations(mgr, ctx); - auto new_out = compensation_qnode->get_outputs()[0]; - for (auto &use : uses) { - use.second->replace_input(use.first, new_out); - } - vis->update_state_for_visited(compensation_qnode); - } - } -/* I want to keep this comment now in case we do bias add in s32 - * dtype.*/ -#if 0 - else if (auto bst_node = node->dyn_cast()) { - if (bst_node->is_quantized_ && bst_node->need_compensation_) { - std::vector data_scales - = bst_node->attrs_.get>( - "data_scales"); - std::vector weight_scales - = bst_node->attrs_.get>( - "weight_scales"); - std::vector mul_scales - = math_utils::vector_mul(data_scales, - weight_scales); - std::vector union_scales( - mul_scales.begin(), mul_scales.end()); - sc_op_ptr mul_scales_const = mgr.make("constant", {}, {}, - {{"values", union_scales}, {"dtype", - datatypes::f32}}); - sc_op_ptr bst_div = mgr.make("div", - {bst_node->get_inputs()[1], - mul_scales_const->get_outputs()[0]}, - {}, {}); - sc_op_ptr bst_cast = mgr.make("cast", - bst_div->get_outputs(), - {}, {{"dtype", datatypes::s32}}); - sc_op_ptr bst_new = mgr.make("broadcast", - {bst_node->get_inputs()[0], - bst_cast->get_outputs()[0]}, - {}, node->attrs_); - - bst_node->need_compensation_ = false; - bst_node->replace_uses_with_and_remove(bst_new); - vis->update_state_for_visited(bst_new); - } - } - } -#endif - }); - mgr.reset_op_ids(); -} -} // namespace quantize -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/transform/rl_conv_weight_transform.cpp b/legacy/core/src/compiler/ir/graph/transform/rl_conv_weight_transform.cpp deleted file mode 100644 index dd1f5ea9d..000000000 --- a/legacy/core/src/compiler/ir/graph/transform/rl_conv_weight_transform.cpp +++ /dev/null @@ -1,255 +0,0 @@ -/******************************************************************************* - * Copyright 2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "../visitor.hpp" -#include "transform.hpp" -#include -#include -#include -#include -#include -#include - -SC_MODULE(rl_conv_weight_transform) - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -static inline int get_full_lowering_threshold(int tile_col) { - int threshold = static_cast(tile_col * 0.75); - return threshold; -} - -static bool use_rl(const context_ptr &ctx, const sc_data_type_t &data_dtype, - const sc_dims &data_dims, const sc_dims &weight_dims, - const sc_dims &pads_begin, const sc_dims &pads_end, - const int &groups) { - auto ndims = data_dims.size(); - assert(ndims == 4UL + (groups > 1) && weight_dims.size() == ndims); - if (!ops::is_amx_dtype(ctx, data_dtype)) { - return false; - } - bool is_vnni_low_fp = ops::is_vnni_low_fp(ctx, data_dtype); - int vnni_blk = is_vnni_low_fp ? 2 : 4; - int tile_col = is_vnni_low_fp ? 32 : 64; - int threshold = get_full_lowering_threshold(tile_col); - bool is_1x1 = std::all_of(weight_dims.begin() + 2 + (groups > 1), - weight_dims.end(), [](int x) { return x == 1; }); - bool is_small_padding = std::all_of(pads_begin.begin(), pads_begin.end(), - [weight_dims, ndims](int x) { - return x <= weight_dims[ndims - 1]; - }) && - std::all_of(pads_end.begin(), pads_end.end(), - [weight_dims, ndims](int x) { - return x <= weight_dims[ndims - 1]; - }); - auto ic = weight_dims[1 + (groups > 1)]; - auto kh = weight_dims[ndims - 2]; - auto ih = data_dims[ndims - 2]; - - // Note: the current rl algorithm expects kw <= iw for init aux buffer - // handling - return ( - !is_1x1 && ndims == 4UL + (groups > 1) && is_small_padding && - (ic <= (tile_col / 2)) && - ((ic % vnni_blk != 0 && kh * ic <= threshold) || (ic % vnni_blk == 0)) && - kh <= ih); -} - -static void query_accu_info_for_rl(const context_ptr &ctx, - const sc_data_type_t &dtype, const int kh, - const int kw, const int ic, - int &num_brgemm_k, int &brgemm_k, - int &extra_padding, int &kind, int groups) { - assert(ops::is_amx_dtype(ctx, dtype)); - bool is_vnni_low_fp = ops::is_vnni_low_fp(ctx, dtype); - int vnni_blk = is_vnni_low_fp ? 2 : 4; - int tile_col = is_vnni_low_fp ? 32 : 64; - int threshold = get_full_lowering_threshold(tile_col); - - // Notes: Here force group conv with small ic to conv_rl because group convs - // will prefer NHWGC while the kw_lowering only support NGHWC now - if (kw * ic <= threshold || ic % vnni_blk != 0 || groups > 1) { - kind = ops::rl_kind::FULL_LOWERING; - auto total_raw_accu = kw * kh * ic; - num_brgemm_k = utils::divide_and_ceil(total_raw_accu, tile_col); - auto total_padded_accu = - utils::rnd_up(total_raw_accu, num_brgemm_k * vnni_blk); - brgemm_k = total_padded_accu / num_brgemm_k; - extra_padding = total_padded_accu - total_raw_accu; - } else { - kind = ops::rl_kind::KW_LOWERING; - - brgemm_k = ic * kw; - num_brgemm_k = kh; - extra_padding = 0; - } -} - -void rl_conv_weight_transform(sc_graph_t &graph, const context_ptr &ctx) { - if ((graph.attrs_.has_key("use_rl") && - graph.attrs_.get("use_rl") == ops::rl_kind::NO_LOWERING) || - graph.is_dynamic()) { - return; - } - auto vis = op_visitor_t::bfs(); - vis.visit_graph(graph, [&](op_visitor_t *vis, const sc_op_ptr &node) { - if (auto op = node->dyn_cast()) { - auto data_plain_dims = op->info_.inputs_[0]->details_.get_plain_dims(); - auto weight_plain_dims = op->info_.inputs_[1]->details_.get_plain_dims(); - auto data_dtype = op->info_.inputs_[0]->details_.dtype_; - auto ndims = data_plain_dims.size(); - sc_dim groups = op->attrs_.get_or_else("groups", 1); - if (ndims != 4UL + (groups > 1)) { - return; - } - // depthwise convolution - bool is_dw_brdgmm = - groups > 1 && 1 == weight_plain_dims[1] && 1 == data_plain_dims[2]; - if (is_dw_brdgmm) { - return; - } - auto dilations = ops::get_dilations(op->attrs_); - auto has_dilation = std::any_of(dilations.begin(), dilations.end(), - [](int x) { return x != 1; }); - // Note, dilation will introduce non-contiguous data when packing - // kw, might be supported in future - if (has_dilation) { - return; - }; - - COMPILE_ASSERT(weight_plain_dims.size() == ndims, - "Weight dims size is expected equal to data dims, but got " - << weight_plain_dims.size() << " vs. " << ndims - << "."); - COMPILE_ASSERT(weight_plain_dims[1 + (groups > 1)] == - data_plain_dims[1 + (groups > 1)], - "Weight_plain_dims[1] is expected equal to " - "data_plain_dims[1], but got " - << weight_plain_dims[1 + (groups > 1)] << " vs. " - << data_plain_dims[1 + (groups > 1)] << "."); - sc_dims pads_begin, pads_end; - if (op->attrs_.has_key("pads_begin")) { - pads_begin = op->attrs_.get("pads_begin"); - pads_end = op->attrs_.get("pads_end"); - } else { - pads_begin = op->attrs_.get("paddings"); - pads_end = pads_begin; - } - - sc_dims strides = op->attrs_.get("strides"); - if (!use_rl(ctx, data_dtype, data_plain_dims, weight_plain_dims, - pads_begin, pads_end, groups)) { - return; - } - - auto oc = weight_plain_dims[(groups > 1)]; - auto kh = weight_plain_dims[ndims - 2]; - auto kw = weight_plain_dims[ndims - 1]; - auto ic = data_plain_dims[1 + (groups > 1)]; - auto &stride = op->attrs_.get("strides"); - auto sw = !stride.empty() ? stride[1] : stride[0]; - int num_brgemm_k = 1; - int brgemm_k = 1; - int extra_padding = 0; - auto kind = ops::rl_kind::NO_LOWERING; - - query_accu_info_for_rl(ctx, data_dtype, kh, kw, ic, num_brgemm_k, - brgemm_k, extra_padding, kind, groups); - if (kind == ops::rl_kind::NO_LOWERING) { - return; - } -#if 0 - // fall-back for small amx utilization - auto low_tmul_utilization - = [](int N, int K, sc_data_type_t &dtype) { - auto dtype_size = (dtype == datatypes::bf16) ? - 2 : 4; if (N <= 8 || K <= (64 / dtype_size)) { - return true; - } else { - return false; - } - }; - if (ops::is_amx_dtype(ctx, data_dtype) - && low_tmul_utilization( - oc, brgemm_k, data_dtype)) { - return; - } -#endif - - op->attrs_["use_rl"] = kind; - op->attrs_["num_brgemm_k"] = num_brgemm_k; - op->attrs_["brgemm_k"] = brgemm_k; - op->attrs_["extra_padding"] = extra_padding; - op->attrs_["origin_wei_plain_dims"] = - groups > 1 ? sc_dims{groups, oc, ic, kh, kw} - : sc_dims{oc, ic, kh, kw}; - - auto weight = op->get_inputs()[1]; - auto weight_op = weight->producer_owner_; - auto is_constant_weight = - (weight_op->isa() || - weight_op->attrs_.get_or_else("constant", const_kind::not_const)); - if (kind == ops::rl_kind::FULL_LOWERING) { - // B-OIHW->reorder->HWIO->view->KN->(padding on "K") - auto wei_shape_2d = sc_dims{ic * kh * kw, groups * oc}; - auto trans_wei = graph.make( - "reorder", {weight}, {}, - {{"out_format", sc_data_format_t(groups > 1 ? format_kinds::EDCAB - : format_kinds::DCBA)}, - {"internal", true}}); - if (is_constant_weight) { - trans_wei->attrs_.set("constant", const_kind::local_const); - } - - auto view_wei = - graph.make("tensor_view", trans_wei->get_outputs(), {}, - {{"shape", wei_shape_2d}, - {"format", sc_data_format_t(format_kinds::AB)}, - {"expand_dim", std::vector{}}}); - if (is_constant_weight) { - view_wei->attrs_.set("constant", const_kind::local_const); - } - vis->update_state_for_visited(trans_wei); - vis->update_state_for_visited(view_wei); - - if (extra_padding > 0) { - auto padded_wei = graph.make("padding", view_wei->get_outputs(), {}, - {{"pads_begin", sc_dims{0}}, - {"pads_end", sc_dims{extra_padding}}}); - - if (is_constant_weight) { - padded_wei->attrs_.set("constant", const_kind::local_const); - } - op->replace_input(1, padded_wei->get_outputs()[0], true); - vis->update_state_for_visited(padded_wei); - } else { - op->replace_input(1, view_wei->get_outputs()[0], true); - } - } else { - // noops as no need to re-layout weight/inputs - } - } - }); - graph.reset_op_ids(); -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/transform/shape_relationship_binding.cpp b/legacy/core/src/compiler/ir/graph/transform/shape_relationship_binding.cpp deleted file mode 100644 index 124296594..000000000 --- a/legacy/core/src/compiler/ir/graph/transform/shape_relationship_binding.cpp +++ /dev/null @@ -1,223 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include - -#include "../dynamic_dispatch_key.hpp" -#include "../fusible_op.hpp" -#include "../graph_op.hpp" -#include "../pass/pass.hpp" -#include "../tunable_op.hpp" -#include "../visitor.hpp" -#include -#include -#include -#include -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -SC_MODULE(graph.shape_relationship_binding); -struct dyn_plhd_union_t { - dyn_plhd_union_t(sc_dim cur_plhd) { - count_ = dynamic_lower_info_t::init_placeholder - cur_plhd; - parent_.resize(count_); - for (size_t i = 0; i < parent_.size(); i++) { - parent_[i] = i; - } - } - sc_dim to_idx(sc_dim in) { - return -1 * (in - dynamic_lower_info_t::init_placeholder); - } - sc_dim to_plhd(sc_dim in) { - return -1 * in + dynamic_lower_info_t::init_placeholder; - } - sc_dim find(sc_dim in) { return to_plhd(find_impl(to_idx(in))); } - void merge(sc_dim in1, sc_dim in2) { - assert(is_dynamic_dim(in1) || is_dynamic_dim(in2)); - if (in1 > 0) { - static_map_.insert(std::make_pair(find(in2), in1)); - return; - } - if (in2 > 0) { - static_map_.insert(std::make_pair(find(in1), in2)); - return; - } - merge_impl(to_idx(in1), to_idx(in2)); - } - sc_dim get_count() const { return count_; } - const std::unordered_map &get_static_map() { - for (auto it = static_map_.begin(); it != static_map_.end();) { - auto p = find(it->first); - if (p != it->first) { - static_map_.insert(std::make_pair(p, it->second)); - it = static_map_.erase(it); - } else { - it++; - } - }; - return static_map_; - } - -private: - std::vector parent_; - sc_dim count_; - // parent to static value map - std::unordered_map static_map_; - sc_dim find_impl(sc_dim in) { - if (parent_[in] == in) { - return in; - } - parent_[in] = find_impl(parent_[in]); - return parent_[in]; - } - void merge_impl(sc_dim in1, sc_dim in2) { - sc_dim p1 = find_impl(in1), p2 = find_impl(in2); - if (p1 == p2) { - return; - } - parent_[p2] = p1; - count_--; - } -}; - -SC_INTERNAL_API void shape_relationship_binding(sc_graph_t &graph, - const context_ptr &ctx) { - if (graph.empty() || !graph.is_dynamic() || !graph.dyn_info_) { - return; - } - // stage 1, first satisfy all dynamic dimensions are placeholder not any. - for (auto &node : graph.ops_) { - auto ins = node->get_inputs(); - auto outs = node->get_outputs(); - for (auto &in : ins) { - auto plain_dims = in->details_.get_plain_dims(); - for (auto &dim : plain_dims) { - if (dim == dimensions::dynamic_any) { - dim = graph.get_next_dynamic_placeholder(); - } - } - in->details_.set_plain_dims(plain_dims); - } - for (auto &out : outs) { - auto plain_dims = out->details_.get_plain_dims(); - for (auto &dim : plain_dims) { - if (dim == dimensions::dynamic_any) { - dim = graph.get_next_dynamic_placeholder(); - } - } - out->details_.set_plain_dims(plain_dims); - } - } - // stage 2, use disjoint-set to merge all dynamic placeholders. - // initialize disjoint-set. - int cur_old_plhd = graph.dyn_info_->cur_dynamic_placeholder_; - dyn_plhd_union_t dyn_un(cur_old_plhd); - op_visitor_t vis = op_visitor_t::dfs_topology_sort(graph.ops_.size()); - vis.visit_graph(graph, [&](op_visitor_t *vis, const sc_op_ptr &node) { - std::vector> shape_relations = - node->get_dynamic_shape_relations(); - for (auto &it : shape_relations) { - dyn_un.merge(it.first, it.second); - } - }); - // stage 3: construct a map of old placeholder to new. - // map of parent to new. - std::unordered_map parent_map; - // map of old placeholder to new. - std::unordered_map plhd_map; - sc_dim cur_new_plhd = dynamic_lower_info_t::init_placeholder; - auto plhd_count = dyn_un.get_count(); - auto static_map = dyn_un.get_static_map(); - - parent_map.insert(static_map.begin(), static_map.end()); - plhd_count -= static_map.size(); - for (sc_dim old_plhd = dynamic_lower_info_t::init_placeholder; - old_plhd > cur_old_plhd; old_plhd--) { - auto it = plhd_map.find(old_plhd); - if (it == plhd_map.end()) { - auto p = dyn_un.find(old_plhd); - auto pit = parent_map.find(p); - if (pit == parent_map.end()) { - parent_map[p] = cur_new_plhd; - plhd_map[old_plhd] = cur_new_plhd; - cur_new_plhd--; - plhd_count--; - } else { - plhd_map[old_plhd] = pit->second; - } - } - } - assert(plhd_count == 0); - SC_UNUSED(plhd_count); - // stage 4: replace old placeholders in graph with map. - std::unordered_map visited; - op_visitor_t vis2 = op_visitor_t::dfs_topology_sort(graph.ops_.size()); - vis2.visit_graph(graph, [&](op_visitor_t *vis2, const sc_op_ptr &node) { - auto ins = node->get_inputs(); - auto outs = node->get_outputs(); - SC_MODULE_INFO << "Op " << node->op_name_ << "\n Inputs:"; - for (auto &in : ins) { - auto plain_dims = in->details_.get_plain_dims(); - if (!visited[in]) { - std::for_each(plain_dims.begin(), plain_dims.end(), - [&plhd_map](sc_dim &in) { - if (is_dynamic_dim(in)) { - auto it = plhd_map.find(in); - assert(it != plhd_map.end()); - in = it->second; - } - }); - in->details_.set_plain_dims(plain_dims); - visited[in] = true; - } - SC_MODULE_INFO << utils::print_vector(plain_dims); - } - SC_MODULE_INFO << "Outputs: "; - for (auto &out : outs) { - auto plain_dims = out->details_.get_plain_dims(); - if (!visited[out]) { - std::for_each(plain_dims.begin(), plain_dims.end(), - [&plhd_map](sc_dim &in) { - if (is_dynamic_dim(in)) { - auto it = plhd_map.find(in); - assert(it != plhd_map.end()); - in = it->second; - } - }); - out->details_.set_plain_dims(plain_dims); - visited[out] = true; - } - SC_MODULE_INFO << utils::print_vector(plain_dims); - } - }); - // stage 5: remove old extra placeholder info. - graph.dyn_info_->cur_dynamic_placeholder_ = cur_new_plhd; - for (sc_dim old_plhd = cur_new_plhd; old_plhd > cur_old_plhd; old_plhd--) { - graph.dyn_info_->dim2expr_map_.erase(old_plhd); - } - // stage 6: set calculation relationship between output and input - op_visitor_t vis3 = op_visitor_t::dfs_topology_sort(graph.ops_.size()); - vis3.visit_graph(graph, [&](op_visitor_t *vis3, const sc_op_ptr &node) { - node->calculate_dynamic_shape_expression(); - }); - graph.reset_op_ids(); -} -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/transform/tensor_view_transform.cpp b/legacy/core/src/compiler/ir/graph/transform/tensor_view_transform.cpp deleted file mode 100644 index 3bf624db7..000000000 --- a/legacy/core/src/compiler/ir/graph/transform/tensor_view_transform.cpp +++ /dev/null @@ -1,264 +0,0 @@ -/******************************************************************************* - * Copyright 2021-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include -#include - -#include "../fusible_op.hpp" -#include "../visitor.hpp" -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -// Helper function to create a block-compressed format for tensor_view transform -// check. The new format tries to convert the axis `with num_of_tiles == 1` to -// previous different axis's block. -// E.g block [128, 1, 1, 6, 1, 384] (ABCDcd) => (ABCDdd) -sc_data_format_t try_compress_format(const sc_dims &plain_dims, - const sc_data_format_t &format) { - auto &fcode = format.format_code_; - auto block_dims = sc_data_format_t::get_blocking_shapes(plain_dims, format); - std::unordered_map> block_axis = - format.get_blocked_axis(); - int blk_idx = 0, new_idx = 0; - std::unordered_map axis_count; - std::unordered_map block_idx; - sc_data_format_t new_format; - for (int i = 0; i < fcode.ndims(); i++) { - auto orig_dim = fcode.get(i); - auto it = block_axis.find(orig_dim); - // the axis has block and the block shape of idx i is 1. - // the first number of block of axis should be reserved to indicate the - // axis exists. - if (block_dims[i] == 1) { - // blocks or num_of_blocks when block > 1. - if (axis_count[orig_dim] > 0 || - (it != block_axis.end() && - std::find_if(it->second.begin(), it->second.end(), - [&](const int &x) { return x > 1; }) != - it->second.end())) { - block_idx[orig_dim]++; - continue; - } - } - new_format.format_code_.set(new_idx++, orig_dim); - if (axis_count[orig_dim] > 0) { - assert(it != block_axis.end()); - auto blk = it->second[block_idx[orig_dim]++]; - new_format.blocks_[blk_idx++] = blk; - } - axis_count[orig_dim]++; - } - new_format.format_code_.set(sc_data_format_kind_t::MAX_DIMS, - format.format_code_.get_control_block()); - return new_format; -} - -// Helper function to judge if a reorder should be transformed to tensor -// view. If the reorder actually does not result in memory movement, we -// mark it as should_transform. -bool should_transform_reorder(const sc_op_ptr &node) { - if (can_op_be_dispatched(node)) { - auto key_set = node->get_dispatch_key_set()->get_inner_set(); - return std::all_of( - key_set.begin(), key_set.end(), [](const op_dispatch_key_t &key) { - return key.in_out_formats_[0] == key.in_out_formats_[1]; - }); - } - assert(node->isa()); - auto input_plain_shapes = node->get_inputs()[0]->details_.get_plain_dims(); - auto output_plain_shapes = node->get_outputs()[0]->details_.get_plain_dims(); - auto input_format = node->get_inputs()[0]->details_.get_format(); - auto output_format = node->get_outputs()[0]->details_.get_format(); - auto input_blocking_shapes = - node->get_inputs()[0]->details_.get_blocking_dims(); - auto output_blocking_shapes = - node->get_outputs()[0]->details_.get_blocking_dims(); - // transformation not possible due to stride - if (!node->get_inputs()[0]->details_.is_dense() || - !node->get_outputs()[0]->details_.is_dense()) { - return false; - } - // inp format is equal to out format - if (input_format == output_format) { - return true; - } - // reorder for padding - if (sc_data_format_t::get_padded_plain_shapes(input_blocking_shapes, - input_format) != - sc_data_format_t::get_padded_plain_shapes(output_blocking_shapes, - output_format)) { - return false; - } - input_format = try_compress_format(input_plain_shapes, input_format); - output_format = try_compress_format(output_plain_shapes, output_format); - input_blocking_shapes = - sc_data_format_t::get_blocking_shapes(input_plain_shapes, input_format); - output_blocking_shapes = - sc_data_format_t::get_blocking_shapes(output_plain_shapes, output_format); - - int inp_idx = 0, out_idx = 0; - auto &inp_code = input_format.format_code_; - auto &out_code = output_format.format_code_; - // orig axis -> vector of block idx - auto inp_blocked_axis = input_format.get_blocked_axis(); - auto out_blocked_axis = output_format.get_blocked_axis(); - // orig axis -> current idx of block - std::unordered_map inp_block_idx, out_block_idx; - for (int i = 0; i < input_format.format_code_.norig_dims(); i++) { - inp_block_idx[i] = -1; - } - for (int i = 0; i < output_format.format_code_.norig_dims(); i++) { - out_block_idx[i] = -1; - } - while (inp_idx < sc_data_format_kind_t::MAX_DIMS && - out_idx < sc_data_format_kind_t::MAX_DIMS && - (inp_code.get(inp_idx) != sc_data_format_kind_t::UNDEF_DIM || - out_code.get(out_idx) != sc_data_format_kind_t::UNDEF_DIM)) { - // get next inp_block_idx - inp_block_idx[inp_code.get(inp_idx)]++; - out_block_idx[out_code.get(out_idx)]++; - - // skip axis == 1 - while (inp_idx < sc_data_format_kind_t::MAX_DIMS && - inp_code.get(inp_idx) != sc_data_format_kind_t::UNDEF_DIM && - input_blocking_shapes[inp_idx] == 1) { - inp_idx++; - inp_block_idx[inp_code.get(inp_idx)]++; - } - while (out_idx < sc_data_format_kind_t::MAX_DIMS && - out_code.get(out_idx) != sc_data_format_kind_t::UNDEF_DIM && - output_blocking_shapes[out_idx] == 1) { - out_idx++; - out_block_idx[out_code.get(out_idx)]++; - } - if (inp_code.get(inp_idx) != out_code.get(out_idx)) { - return false; - } - // skip same axis - while (inp_code.get(inp_idx + 1) != sc_data_format_kind_t::UNDEF_DIM && - inp_code.get(inp_idx + 1) == inp_code.get(inp_idx)) { - inp_block_idx[inp_code.get(inp_idx + 1)]++; - inp_idx++; - } - while (out_code.get(out_idx + 1) != sc_data_format_kind_t::UNDEF_DIM && - out_code.get(out_idx + 1) == out_code.get(out_idx)) { - out_block_idx[out_code.get(out_idx + 1)]++; - out_idx++; - } - int orig_inp_axis = inp_code.get(inp_idx); - int orig_out_axis = out_code.get(out_idx); - // different axis - if (orig_inp_axis != orig_out_axis) { - return false; - } - // different block - auto inp_block_itr = inp_blocked_axis.find(orig_inp_axis); - auto out_block_itr = out_blocked_axis.find(orig_out_axis); - if (inp_block_itr != inp_blocked_axis.end() || - out_block_itr != out_blocked_axis.end()) { - // inp has no block, but out has blocks remained. - if (inp_block_itr == inp_blocked_axis.end() && - out_block_idx[orig_out_axis] < out_block_itr->second.size()) { - return false; - } - // out has no block, but inp has blocks remained. - if (out_block_itr == out_blocked_axis.end() && - inp_block_idx[orig_inp_axis] < inp_block_itr->second.size()) { - return false; - } - // inp and out both have blocks. - if (inp_block_itr != inp_blocked_axis.end() && - out_block_itr != out_blocked_axis.end()) { - // inp and out have blocks remained. - if (inp_block_idx[orig_inp_axis] < inp_block_itr->second.size() && - out_block_idx[orig_out_axis] < out_block_itr->second.size()) { - // inp and out block remained should be equal. - if (inp_block_itr->second[inp_block_idx[orig_inp_axis]] != - out_block_itr->second[out_block_idx[orig_out_axis]]) { - return false; - } - } else if (inp_block_idx[orig_inp_axis] < - inp_block_itr->second.size()) { - return false; - } else if (out_block_idx[orig_out_axis] < - out_block_itr->second.size()) { - return false; - } - // else inp and out have no remained blocks. - } - } - // increase inp_idx - inp_idx++; - out_idx++; - } - return true; -} - -static bool should_transform_transpose(const sc_op_ptr &node) { - assert(node->isa()); - if (node->info_.inputs_[0]->details_.get_format() != - node->info_.outputs_[0]->details_.get_format()) { - // means that we have permuted the data format - return true; - } - return false; -} - -// Replace reorder op and transpose op that does not cause data movements with -// tensor_view op -void convert_to_tensor_view(sc_graph_t &graph, const context_ptr &ctx) { - auto vis = op_visitor_t::bfs(); - int reorder2tv = graph.attrs_.get_or_else("temp.reorder2tv", 1); - vis.visit_graph(graph, [&](op_visitor_t *vis, const sc_op_ptr &node) { - if (node->isa() && reorder2tv && - should_transform_reorder(node) && - !node->attrs_.get_or_else("actually_copy", false)) { - auto tensor_view_out = node->get_outputs()[0]->copy(); - tensor_view_out->producer_owner_ = nullptr; - auto view = graph.make( - "tensor_view", node->get_inputs(), {tensor_view_out}, - {{"shape", tensor_view_out->details_.get_blocking_dims()}}); - view->copy_dispatch_key_set_from_op(node); - node->replace_uses_with_and_remove(view); - vis->update_state_for_visited(view); - } else if (node->isa() && - should_transform_transpose(node)) { - auto tensor_view_out = node->get_outputs()[0]->copy(); - tensor_view_out->producer_owner_ = nullptr; - auto view = - graph.make("tensor_view", node->get_inputs(), {tensor_view_out}, - {{"shape", tensor_view_out->details_.get_blocking_dims()}, - {"order", node->attrs_.get>("order")}}); - view->copy_dispatch_key_set_from_op(node); - node->replace_uses_with_and_remove(view); - vis->update_state_for_visited(view); - } - }); - graph.reset_op_ids(); -} - -void tensor_view_transform(sc_graph_t &graph, const context_ptr &ctx) { - convert_to_tensor_view(graph, ctx); -} -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/transform/transform.hpp b/legacy/core/src/compiler/ir/graph/transform/transform.hpp deleted file mode 100644 index 848f45682..000000000 --- a/legacy/core/src/compiler/ir/graph/transform/transform.hpp +++ /dev/null @@ -1,148 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_TRANSFORM_TRANSFORM_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_TRANSFORM_TRANSFORM_HPP - -#include "../graph.hpp" -#include -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -SC_INTERNAL_API void -elemwise_bcast_swap(sc_graph_t &graph, - const context_ptr &ctx = get_default_context()); - -SC_INTERNAL_API void -broadcast_transform(sc_graph_t &graph, - const context_ptr &ctx = get_default_context()); - -SC_INTERNAL_API void -elemwise_dimension_alignment(sc_graph_t &graph, - const context_ptr &ctx = get_default_context()); - -SC_INTERNAL_API void -dynamic_graph_transform(sc_graph_t &graph, - const context_ptr &ctx = get_default_context()); - -SC_INTERNAL_API void -layout_propagation(sc_graph_t &graph, - const context_ptr &ctx = get_default_context()); - -SC_INTERNAL_API void -pre_padding(sc_graph_t &graph, const context_ptr &ctx = get_default_context()); - -SC_INTERNAL_API void -mixed_partition(sc_graph_t &graph, - const context_ptr &ctx = get_default_context()); - -SC_INTERNAL_API void -annotate_config(sc_graph_t &graph, - const context_ptr &ctx = get_default_context()); - -SC_INTERNAL_API void -graph_inline(sc_graph_t &graph, const context_ptr &ctx = get_default_context()); - -SC_INTERNAL_API void partial_reduce_replace(sc_graph_t &graph, - const context_ptr &ctx); -/** - * Optimize the constant_optimizable_t ops. - * */ -SC_INTERNAL_API void constant_optimization(sc_graph_t &graph, - const context_ptr &ctx); - -void permute_propagation(sc_graph_t &graph, - const context_ptr &ctx = get_default_context()); - -SC_INTERNAL_API void -tensor_view_transform(sc_graph_t &graph, - const context_ptr &ctx = get_default_context()); - -SC_INTERNAL_API void -flatten_conv(sc_graph_t &graph, const context_ptr &ctx = get_default_context()); - -SC_INTERNAL_API -void graph_simplify(sc_graph_t &graph, - const context_ptr &ctx = get_default_context()); - -SC_INTERNAL_API void -global_reschedule(sc_graph_t &graph, - const context_ptr &ctx = get_default_context()); - -SC_INTERNAL_API void -inplace_transform(sc_graph_t &graph, - const context_ptr &ctx = get_default_context()); - -SC_INTERNAL_API void -div_bcast_transform(sc_graph_t &graph, - const context_ptr &ctx = get_default_context()); - -void brgemm_fusion_transform(sc_graph_t &graph, - const context_ptr &ctx = get_default_context()); - -SC_INTERNAL_API void -shape_relationship_binding(sc_graph_t &graph, - const context_ptr &ctx = get_default_context()); - -SC_INTERNAL_API void -merge_concats(sc_graph_t &graph, - const context_ptr &ctx = get_default_context()); - -SC_INTERNAL_API void -graph_concat_memory_planning(sc_graph_t &graph, - const context_ptr &ctx = get_default_context()); - -SC_INTERNAL_API void -rl_conv_weight_transform(sc_graph_t &graph, - const context_ptr &ctx = get_default_context()); - -namespace quantize { -SC_INTERNAL_API void -annotate_fusion_break(sc_graph_t &mgr, - const context_ptr &ctx = get_default_context()); - -SC_INTERNAL_API void -quantize_info_propagation(sc_graph_t &mgr, - const context_ptr &ctx = get_default_context()); - -SC_INTERNAL_API void -graph_reschedule(sc_graph_t &mgr, - const context_ptr &ctx = get_default_context()); - -SC_INTERNAL_API void -calculate_op_compensation(sc_graph_t &mgr, - const context_ptr &ctx = get_default_context()); - -SC_INTERNAL_API void -quantize_inline(sc_graph_t &mgr, - const context_ptr &ctx = get_default_context()); -} // namespace quantize - -SC_INTERNAL_API void -fpmath_mode(sc_graph_t &mgr, const context_ptr &ctx = get_default_context()); - -SC_INTERNAL_API void -eliminate_zero_shaped_tensors(sc_graph_t &graph, - const context_ptr &ctx = get_default_context()); - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/graph/tunable_op.cpp b/legacy/core/src/compiler/ir/graph/tunable_op.cpp deleted file mode 100644 index 7fe460457..000000000 --- a/legacy/core/src/compiler/ir/graph/tunable_op.cpp +++ /dev/null @@ -1,361 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -// distiguish overloaded functions. -static std::atomic internal_idx = {0}; -tunable_op_t::tunable_op_t(const std::string &op_name, - const std::vector &ins, - const std::vector &outs, - const any_map_t &attrs) - : sc_op(op_name, ins, outs, attrs) {} - -sc_op_ptr tunable_op_t::copy(const std::vector &ins, - const std::vector &outs, - sc_graph_t &mgr) { - auto ret = mgr.make(op_name_, ins, outs, attrs_); - ret->copy_dispatch_key_set_from_op(shared_from_this()); - ret->info_.internal_info_ = info_.internal_info_; // shadow copy. - auto tune_ret = ret->stc_cast(); - tune_ret->op_name_ = op_name_; - tune_ret->config_data_ = config_data_; - tune_ret->dyn_config_candidates_ = dyn_config_candidates_; - tune_ret->is_quantized_ = is_quantized_; - tune_ret->need_compensation_ = need_compensation_; - tune_ret->should_quantized_ = should_quantized_; - return ret; -} - -bool tunable_op_t::is_valid(const context_ptr &ctx) { - if (!config_data_ || - !create_generator()->is_valid_config(ctx, config_data_.data_.get())) { - return false; - } - return true; -} - -ir_module_ptr tunable_op_t::get_func(context_ptr ctx) { - // query binding axis - query_binding_axis(get_owner_graph()); - auto ret = std::make_shared(ctx); - auto gen_ptr = create_generator(); - bool need_inner_query = need_dynamic_internal_query(); - set_config_if_empty(ctx, gen_ptr.get()); - std::vector ins; - std::vector outs; - auto func = graph::create_func_decl_for_op(this, ins, outs); - auto gen_body = [&](const body_generator_ptr &gen_ptr) { - builder::ir_builder_t bld; - bld.push_scope(); - std::vector loops; - bool status = gen_ptr->generate(ctx, config_data_.data_.get(), nullptr, ins, - outs, loops); - assert(status); - bld.push_returns(true); - auto body = bld.pop_scope(); - gen_ptr->schedule_loops(ctx, config_data_.data_.get(), body, loops); - return body; - }; - stmt body; - if (need_inner_query) { - std::vector loops; - func->params_.emplace_back( - builder::make_tensor("internal_func", {1}, datatypes::index)); - func->params_.back()->attr().set(attr_keys::no_index2var, true); - func->decl_->params_ = func->params_; - func_t single_core_func = gen_ptr->get_single_core_func( - ctx, config_data_.data_.get(), nullptr, ins, outs, loops); - auto common_args = outs; - common_args.insert(common_args.end(), ins.begin(), ins.end()); - auto common_params = common_args; - common_params.emplace_back( - builder::make_var(datatypes::pointer, "single_core_func")); - common_args.emplace_back(builder::make_func_addr(single_core_func)); - auto internal_func = builder::make_func( - std::string("internal_func") + std::to_string(internal_idx++), - common_params, builder::make_returns_unattached(true), - datatypes::boolean); - internal_func->attr().set(attr_keys::keep_func, true); - auto inter_ptr = builder::make_reinterpret( - builder::make_indexing(func->params_.back(), {0}), datatypes::pointer); - inter_ptr->attr().set("prototype", internal_func); - body = builder::make_stmts_unattached( - {builder::make_evaluate_unattached( - make_expr(inter_ptr, common_args)), - builder::make_returns_unattached(true)}); - ret->add_func({func, internal_func, single_core_func}); - } else { - body = gen_body(gen_ptr); - ret->add_func({func}); - } - - func->body_ = std::move(body); - ret->set_entry_func_idx(0); - return ret; -} - -func_t tunable_op_t::get_func(mixed_parti_t *parti, - const std::vector &ins, - const std::vector &outs) { - // query binding axis - query_binding_axis(get_owner_graph()); - bool need_inner_query = need_dynamic_internal_query(); - - auto gen_body = [&](const body_generator_ptr &gen_ptr) { - set_config_if_empty(parti->ctx_, gen_ptr.get()); - fusion_anchor_mgr_t fmgr(parti); - builder::ir_builder_t bld; - bld.push_scope(); - std::vector loops; - bool status = gen_ptr->generate(parti->ctx_, config_data_.data_.get(), - &fmgr, ins, outs, loops); - // attach parent anchor if necessary - if (parti->ready_for_op(this)) { - auto committed_anchor = parti->lookup_anchor_map(this); - for (auto &template_anchor : fmgr.get_fusion_anchor()) { - template_anchor->attach_parent_anchor(committed_anchor, nullptr, true); - } - } - assert(status); - auto body = bld.pop_scope(); - return body; - }; - - auto gen_ptr = create_generator(); - stmt body; - // single core func is assicated with format so it needs to be query first. - if (need_inner_query) { - if (!parti->dyn_inter_) { - parti->dyn_inter_ = - std::make_shared(parti->ctx_); - } - fusion_anchor_mgr_t fmgr(parti); - std::vector loops; - func_t single_core_func = gen_ptr->get_single_core_func( - parti->ctx_, config_data_.data_.get(), &fmgr, ins, outs, loops); - parti->dyn_inter_->single_core_func_ = single_core_func; - parti->dyn_inter_->single_core_func_extra_args_ = - gen_ptr->get_extra_args_from_func(single_core_func); - - auto common_args = outs; - common_args.insert(common_args.end(), ins.begin(), ins.end()); - std::for_each(common_args.begin(), common_args.end(), [](const expr &arg) { - arg->attr().set(attr_keys::always_trans, true); - }); - parti->dyn_inter_->inter_func_extra_args_ = std::vector{ - builder::make_var(datatypes::pointer, "single_core_func")}; - parti->dyn_inter_->inter_call_extra_args_ = - std::vector{builder::make_func_addr(single_core_func)}; - // only for function signature. - auto internal_func = builder::make_func( - std::string("internal_func") + std::to_string(internal_idx++), - common_args, builder::make_returns_unattached(true), - datatypes::boolean); - internal_func->attr().set(attr_keys::keep_func, true); - parti->dyn_inter_->inter_func_ = internal_func; - parti->dyn_inter_->mod_->add_func({single_core_func, internal_func}); - if (!parti->dyn_inter_->inter_funcs_param_.defined()) { - parti->dyn_inter_->inter_funcs_param_ = - builder::make_tensor("internal_funcs", {1}, datatypes::index); - parti->dyn_inter_->num_func_ = 1; - parti->dyn_inter_->inter_funcs_param_->attr().set(attr_keys::no_index2var, - true); - } else { - assert(parti->dyn_inter_->inter_funcs_param_.isa()); - auto inter_tsr = - parti->dyn_inter_->inter_funcs_param_.static_as(); - inter_tsr->dims_[0] = ++parti->dyn_inter_->num_func_; - } - auto inter_ptr = builder::make_reinterpret( - builder::make_indexing(parti->dyn_inter_->inter_funcs_param_, - {parti->dyn_inter_->num_func_ - 1}), - datatypes::pointer); - inter_ptr->attr().set("prototype", internal_func); - auto inter_call = make_expr(inter_ptr, common_args); - parti->dyn_inter_->inter_call_ = inter_call; - body = builder::make_stmts_unattached( - {builder::make_evaluate_unattached(inter_call)}); - // auto ret_params = outs; - // ret_params.emplace_back(parti->dyn_inter_->inter_funcs_param_); - } else { - body = gen_body(gen_ptr); - } - auto func = builder::make_func(std::string(""), std::vector{}, - std::move(body), datatypes::boolean); - return func; -} - -void tunable_op_t::create_mixed_partition(mixed_parti_t *parti) { - parti->buf_alloc_.allocate_buffer(this); - std::vector ins, outs; - std::tie(ins, outs) = parti->buf_alloc_.get_buffer(this); - parti->func_ = get_func(parti, ins, outs); -} - -void tunable_op_t::append_mixed_partition(mixed_parti_t *parti) { - COMPILE_ASSERT(parti->ready_for_op(this), "No suitable anchor found for " - << op_name_ << "_" - << logical_op_id_); - parti->buf_alloc_.allocate_buffer(this); - parti->buf_alloc_.update_input_buffer_info(this); - - commit_into_anchor(parti->lookup_anchor_map(this).get()); - // update output buffer info after inner anchor created - parti->buf_alloc_.update_output_buffer_info(this); -} - -void tunable_op_t::search_anchor(mixed_parti_t *parti) { - search_op_anchor_in_parti(this, parti); -} - -void tunable_op_t::commit_into_anchor(fusion_anchor_t *committed_anchor) { - auto parti = committed_anchor->get_binded_mxp(); - std::vector ins, outs; - std::tie(ins, outs) = parti->buf_alloc_.get_buffer(this); - // prepare slice - std::vector ins_slice(get_inputs().size()), - outs_slice(get_outputs().size()); - - std::transform(get_inputs().begin(), get_inputs().end(), ins_slice.begin(), - [&committed_anchor](const graph_tensor_ptr >) { - auto slice_list = committed_anchor->fsmap_.get(gt); - COMPILE_ASSERT(slice_list.size() == 1, - "multi-slice is not expected to tunable op"); - return slice_list[0]; - }); - std::transform(get_outputs().begin(), get_outputs().end(), outs_slice.begin(), - [&committed_anchor](const graph_tensor_ptr >) { - auto slice_list = committed_anchor->fsmap_.get(gt); - COMPILE_ASSERT(slice_list.size() == 1, - "multi-slice is not expected to tunable op"); - return slice_list[0]; - }); - - // prepare tptr for function call - std::vector tptr_ins(ins.size()), tptr_outs(outs.size()); - std::transform(ins.begin(), ins.end(), ins_slice.begin(), tptr_ins.begin(), - [&](const expr &tsr, const slice_range &range) { - return transform_tsr2tptr_with_range(tsr, range); - }); - std::transform(outs.begin(), outs.end(), outs_slice.begin(), - tptr_outs.begin(), - [&](const expr &tsr, const slice_range &range) { - return transform_tsr2tptr_with_range(tsr, range); - }); - - // prepare strided tsr for function definition - std::vector strd_ins(ins.size()), strd_outs(outs.size()); - std::transform(ins.begin(), ins.end(), ins_slice.begin(), strd_ins.begin(), - [&](const expr &tsr, const slice_range &range) { - return transform_tsr2stsr_with_range(tsr, range); - }); - std::transform(outs.begin(), outs.end(), outs_slice.begin(), - strd_outs.begin(), - [&](const expr &tsr, const slice_range &range) { - return transform_tsr2stsr_with_range(tsr, range); - }); - - node_ptr_map def_to_call_map; - for (size_t i = 0; i < ins.size(); i++) { - def_to_call_map[strd_ins[i].impl] = tptr_ins[i].impl; - parti->buf_alloc_.b2g_map_[strd_ins[i]] = get_inputs()[i]; - } - for (size_t i = 0; i < outs.size(); i++) { - def_to_call_map[strd_outs[i].impl] = tptr_outs[i].impl; - parti->buf_alloc_.b2g_map_[strd_outs[i]] = get_outputs()[i]; - } - // commit content id to anchor firstly - committed_anchor->append_content(static_cast(this)); - // fusible fusion anchor would also be created following op template - auto func = get_func(parti, strd_ins, strd_outs); - for (auto &stsr : strd_ins) { - parti->buf_alloc_.b2g_map_.erase(stsr); - } - for (auto &stsr : strd_outs) { - parti->buf_alloc_.b2g_map_.erase(stsr); - } - - // replace strided tensor with tensorptr - mxp_replacer_t(def_to_call_map).replace_func(func); - committed_anchor->commit_stmt(func->body_); -} - -void tunable_op_t::set_config(const config_ptr &config) { - config_data_ = config; -} - -void tunable_op_t::set_config_if_empty(context_ptr ctx, - body_generator_base_t *p) { - if (!config_data_) { - set_config(p->get_default_config(std::move(ctx))); - } -} - -config_ptr tunable_op_t::get_default_config(context_ptr ctx) { - auto gen = this->create_generator(); - return gen->get_default_config(ctx); -} - -config_ptr_vec -tunable_op_t::get_dynamic_config_candidates(const context_ptr &ctx) { - if (dyn_config_candidates_.empty()) { - if (auto gen = create_generator()) { - dyn_config_candidates_ = gen->get_dynamic_config_candidates(ctx); - } - } - return dyn_config_candidates_; -} - -impl_kind_map tunable_op_t::convert_config_candidates_to_impl_map( - const config_ptr_vec &configs) { - if (configs.empty()) { - return impl_kind_map(); - } - auto gen = create_generator(); - impl_kind_map ret; - ret.reserve(configs.size()); - for (int i = 0; i < static_cast(configs.size()); i++) { - auto &cfg = configs[i]; - ret.insert(std::make_pair(gen->convert_config_to_keys(cfg), i)); - } - return ret; -} - -std::vector -tunable_op_t::get_impl_dispatch_candidates(const context_ptr &ctx) { - return get_dynamic_impl_dispatch_candidates(this, ctx); -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/tunable_op.hpp b/legacy/core/src/compiler/ir/graph/tunable_op.hpp deleted file mode 100644 index abee6927b..000000000 --- a/legacy/core/src/compiler/ir/graph/tunable_op.hpp +++ /dev/null @@ -1,114 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_TUNABLE_OP_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_TUNABLE_OP_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -struct op_dispatch_key_t; -struct impl_op_dispatch_key_t; - -class SC_INTERNAL_API tunable_op_t - : public sc_op, - public op_traits::copyable_t, - public op_traits::may_quantize_t, - public op_traits::post_fusion_acceptable_t, - public op_traits::configurable_t, - public op_traits::mixed_partition_acceptable { -public: - tunable_op_t(const std::string &op_name, - const std::vector &ins, - const std::vector &outs, - const any_map_t &attrs); - - sc_op_ptr copy(const std::vector &ins, - const std::vector &outs, - sc_graph_t &mgr) override; - - bool is_valid(const context_ptr &) override; - - ir_module_ptr get_func(context_ptr ctx, - const std::shared_ptr &fuse_mgr, - const std::string &func_name) override { - throw std::runtime_error("unimplemented"); - } - ir_module_ptr get_func(context_ptr ctx) override; - - func_t get_func(mixed_parti_t *parti, const std::vector &ins, - const std::vector &outs); - - config_ptr get_config() override { return config_data_; } - - void set_config(const config_ptr &config) override; - void set_config_if_empty(context_ptr ctx, body_generator_base_t *p); - virtual void set_config_by_key(const op_dispatch_key_t &key, - const context_ptr &ctx) { - throw std::runtime_error("unimplemented"); - } - virtual void set_internal_config_by_key(const impl_op_dispatch_key_t &key, - const context_ptr &ctx) { - throw std::runtime_error("unimplemented"); - } - - config_ptr get_default_config(context_ptr ctx) override; - - void search_anchor(mixed_parti_t *parti) override; - - void commit_into_anchor(fusion_anchor_t *committed_anchor) override; - - config_ptr_vec get_dynamic_config_candidates(const context_ptr &ctx) override; - impl_kind_map - convert_config_candidates_to_impl_map(const config_ptr_vec &configs) override; - std::vector - get_impl_dispatch_candidates(const context_ptr &ctx) override; - - virtual body_generator_ptr create_generator() = 0; - - void create_mixed_partition(mixed_parti_t *parti) override; - - void append_mixed_partition(mixed_parti_t *parti) override; - - virtual infer_status_code infer_slice_ranges(const context_ptr &ctx, - fslice_map &fsmap) = 0; - - void infer_binding_axis(binding_axis_map &bdax_map) override {} - void pre_infer_binding_axis(binding_axis_map &bdax_map) override {} - -protected: - config_ptr config_data_; - std::vector dyn_config_candidates_; -}; - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/graph/utils.hpp b/legacy/core/src/compiler/ir/graph/utils.hpp deleted file mode 100644 index ea410e831..000000000 --- a/legacy/core/src/compiler/ir/graph/utils.hpp +++ /dev/null @@ -1,112 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_UTILS_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_UTILS_HPP - -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -// do divide and ceil on expr. -expr divide_and_ceil(const expr &, const expr &); -namespace graph { -sc_graph_t -make_single_op_graph(const std::string &opname, - const std::vector &inputs, - const std::vector &outputs = {}, - const any_map_t &attr = {}); - -expr tensor_detail_to_ir_tensor(sc_graph_t &graph, const std::string &name, - const logical_tensor_t &); -std::vector -tensor_detail_to_ir_tensor(sc_graph_t &graph, const std::string &name_prefix, - const std::vector &); -std::vector -tensor_detail_to_ir_tensor(sc_graph_t &graph, const std::string &name_prefix, - const std::vector &); -std::string decay_quantized_op_name(const std::string &op_name); - -// get logical_tensor_t from logical_tensor_t -void get_logical_tensors(ltensors *ins, - const std::vector &flts); - -// marks the "read_buffer" or "write_buffer" attributes on each expr in `args`. -// If `is_read` mark them read. Otherwise, mark them write. -void mark_read_or_write_buffers(std::vector &args, bool is_read); - -/** - * Creates an empty function declaration for an op. The function will have the - * name `op::op_name_`. Its arguments are composed of the Op's outputs and then - * the Op's inputs - * @param op the op - * @param ins the vector will be set with the input args of the generated IR - * function - * @param outs the vector will be set with the output args of the generated IR - * functio - * @return the generated IR function for the Op. Its body is an empty stmts node - * */ -func_t create_func_decl_for_op(sc_op *op, std::vector &ins, - std::vector &outs); - -/** - * Creates an empty query format function declaration for an op. The function - * will have the name `query_format_op::op_name_`. Its arguments are composed of - * the Op's outputs and then the Op's inputs and their formats. - * @param op the op - * @param ins the vector of input args - * @param ori_ins the vector of origin input args - * @param outs the vector of output args - * @param in_fmts the vector of input format args - * @param ori_in_fmts the vector of origin input format args - * @param out_fmts the vector of output format - * @param out_sizes the size tensor of output - * @param kernel the kernel tensor - * @return the generated IR function for the Op. Its body is an empty stmts node - * */ -func_t create_query_func_decl_for_op( - sc_op *op, std::vector &ins, std::vector &ori_ins, - std::vector &outs, std::vector &in_fmts, - std::vector &ori_in_fmts, std::vector &out_fmts, - std::vector &out_sizes, expr &kernel); - -ltensors -extract_detail_from_tensors(const std::vector> &); - -/** - * Checks whether lhs_shape could be identical to rhs_shape - * considering dynamic shape use cases - * */ -bool check_shape_equal(const sc_dims &lhs_shape, const sc_dims &rhs_shape); - -/** - * Checks whether lhs and rhs matches for both plain_dims and dtype - * considering dynamic shape use cases - * */ -void check_logical_tensor_shape_dtype_identical(const logical_tensor_t &lhs, - const logical_tensor_t &rhs); -} // namespace graph -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/graph/visitor.cpp b/legacy/core/src/compiler/ir/graph/visitor.cpp deleted file mode 100644 index 6be266420..000000000 --- a/legacy/core/src/compiler/ir/graph/visitor.cpp +++ /dev/null @@ -1,596 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#include "visitor.hpp" -#include "fusible_op.hpp" -#include "fusible_op_utils.hpp" -#include "fusion_anchor.hpp" -#include "tunable_op.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -void op_visitor_t::visit_graph(const sc_graph_t &mgr, const visitor_func &f) { - for (auto &v : mgr.ops_) { - if (v->isa() || v->isa()) { - to_visit_.emplace_back(v); - } - } - int original_number_of_ops = mgr.ops_.size(); - visit(f); - if (check_all_ops_visited_) { - assert_all_ops_visited(mgr, original_number_of_ops); - } -} - -void op_visitor_t::visit(const visitor_func &f) { - while (!to_visit_.empty()) { - auto ptr = select_next_node(this); - // if selector fails (e.g. found a node that has already been visited), - // try again - if (!ptr || ptr->is_removed_) { - continue; - } - f(this, ptr); - update_state_for_visited(std::move(ptr)); - } -} - -void op_visitor_t::assert_all_ops_visited(const sc_graph_t &mgr, - size_t concerned_size) { -#ifndef NDEBUG - std::vector not_visited_ops; - for (size_t i = 0; i < concerned_size; i++) { - // check whether all non-removed ops within concerned_size are visited - const auto &op = mgr.ops_[i]; - if (!op->is_removed_ && !has_visited(op->logical_op_id_)) { - not_visited_ops.emplace_back(op); - } - } - if (!not_visited_ops.empty()) { - // some ops are not visited, assertion failed - std::stringstream error_message; - error_message << "Illegal state for op_visitor_t. The following " - << not_visited_ops.size() - << " ops were not visited, possibly due to changing the " - "graph during the visit without calling " - "update_state_for_visited(): "; - for (const auto &op : not_visited_ops) { - auto name = op->op_name_ + std::to_string(op->logical_op_id_); - error_message << name << ' '; - } - COMPILE_ASSERT(false, error_message.str()); - } -#else - for (size_t i = 0; i < concerned_size; i++) { - const auto &op = mgr.ops_[i]; - COMPILE_ASSERT( - op->is_removed_ || has_visited(op->logical_op_id_), - "Illegal state for op_visitor_t. Some ops were not visited, " - "possibly due to changing the graph during the visit without " - "calling update_state_for_visited()."); - } -#endif -} - -void op_dep_matrix_t::update(const sc_op_ptr &cur) { - int cur_id = cur->logical_op_id_; - // set all inputs=1 - for (auto &input : cur->get_inputs()) { - int dep_id = input->producer_owner_->logical_op_id_; - matrix_[dep_id][cur_id] = 1; - matrix_[cur_id][dep_id] = -1; - // set all recursive inputs=1 - for (size_t i = 0; i < matrix_.size(); i++) { - if (matrix_[i][dep_id] == 1) { - matrix_[i][cur_id] = 1; - matrix_[cur_id][i] = -1; - } - } - } -} - -// Move Op in topology sequence(op_seq) from src position to dst position. -// Meanwhile, it will check op_dep_matrix_t, if dependency not met, it will do -// nothing. -static bool move_op_from_to(std::vector &op_seq, - const op_dep_matrix_t &dep_matrix, int src, int dst, - bool ignore_dep = false) { - // automatically skip if src==dst - if (src == dst) - return false; - // need to check op_seq from src+1 to dst dependence in dep_matrix - int src_op_id = op_seq[src]->logical_op_id_; - if (src < dst) { - for (int i = src + 1; i < dst; i++) { - int cur_op_id = op_seq[i]->logical_op_id_; - if (!ignore_dep && dep_matrix.lookup(src_op_id, cur_op_id) == 1) { - // can not move due to dependence - return false; - } - } - } - // need to check op_seq from dst to src-1 dependence in dep_matrix - else { - for (int i = dst; i < src; i++) { - int cur_op_id = op_seq[i]->logical_op_id_; - if (!ignore_dep && dep_matrix.lookup(cur_op_id, src_op_id) == 1) { - // can not move due to dependence - return false; - } - } - } - // move op in seq from src to dst - op_seq.insert(op_seq.begin() + dst, op_seq[src]); - op_seq.erase(op_seq.begin() + src + (src <= dst ? 0 : 1)); - return true; -} - -void op_sorting_visitor_t::visit_by_rules( - sc_graph_t &graph, const std::function &f, - std::vector rules) { - std::sort(rules.begin(), rules.end()); - std::vector f_rule_list; - for (auto &r : rules) { - switch (r) { - // This rule will make same kind op closer, which can expose more - // opportunity to loop merge pass - case sort_rule::same_kind: - f_rule_list.emplace_back(create_same_kind_rule()); - break; - default: - break; - } - } - auto f_rule = [f_rule_list](std::vector &op_seq, - const op_dep_matrix_t &dep_matrix) { - for (auto &f_rule : f_rule_list) { - f_rule(op_seq, dep_matrix); - } - }; - visit_by_rules(graph, f, f_rule); -} - -op_dep_matrix_t::op_dep_matrix_t(const sc_graph_t &graph) - : op_dep_matrix_t(graph.ops_.size()) { - int op_size = graph.ops_.size(); - auto &dep_matrix = *this; - op_visitor_t::dfs_topology_sort(op_size).visit_graph( - graph, [&dep_matrix](op_visitor_t *vis, const sc_op_ptr &cur) { - dep_matrix.update(cur); - }); -} - -void op_sorting_visitor_t::visit_by_rules( - sc_graph_t &graph, const std::function &f, - const rule_func &f_rule) { - int op_size = graph.ops_.size(); - // Dependence Matrix for Graph Ops, value of postion means whether j-th - // op depends on i-th op. - op_dep_matrix_t dep_matrix(op_size); - std::vector op_seq; - // Step 1: visit whole graph and update Adj-Matrix. - op_visitor_t::dfs_topology_sort(op_size).visit_graph( - graph, [&op_seq, &dep_matrix](op_visitor_t *vis, const sc_op_ptr &cur) { - op_seq.emplace_back(cur); - dep_matrix.update(cur); - }); - // Step 2: change order by rules - f_rule(op_seq, dep_matrix); - // Step 3: excute ops by new order - for (auto &op : op_seq) { - f(std::move(op)); - } -} - -std::vector -op_sorting_visitor_t::sort_by_rules(sc_graph_t &graph, - std::vector rules) { - std::vector sorted_op_list; - auto f = [&sorted_op_list](const sc_op_ptr &cur) { - sorted_op_list.emplace_back(cur); - }; - visit_by_rules(graph, f, std::move(rules)); - return sorted_op_list; -} - -std::vector -op_sorting_visitor_t::sort_by_rules(sc_graph_t &graph, - const rule_func &f_rule) { - std::vector sorted_op_list; - auto f = [&sorted_op_list](const sc_op_ptr &cur) { - sorted_op_list.emplace_back(cur); - }; - visit_by_rules(graph, f, f_rule); - return sorted_op_list; -} - -void op_visitor_t::update_state_for_visited(sc_op_ptr node) { - update_visit_list(this, std::move(node)); -} - -op_visitor_t::op_visitor_t(selector_func select_next_node_func, - updater_func update_visit_list_func, - bool check_all_ops_visited) - : select_next_node(std::move(select_next_node_func)), - update_visit_list(std::move(update_visit_list_func)), - check_all_ops_visited_(check_all_ops_visited) { - visited_.reserve(256); -} - -void op_visitor_t::set_visited(int id) { - assert(id >= 0); - if ((unsigned)id >= visited_.size()) { - visited_.resize(id + 1); - } - visited_[id] = true; -} - -bool op_visitor_t::has_visited(int id) { - assert(id >= 0); - if ((unsigned)id >= visited_.size()) { - return false; - } - return visited_[id]; -} - -void op_visitor_t::push_back_updater(op_visitor_t *v, const sc_op_ptr &cur) { - v->set_visited(cur->logical_op_id_); - for (auto < : cur->get_outputs()) { - for (auto &user : lt->uses_) { - v->to_visit_.emplace_back(user.second); - } - } -}; - -namespace op_kind { -static constexpr int elementwise = 0; -static constexpr int broadcast = 1; -static constexpr int reduce = 2; -// TODO(xxx): extend it. -static constexpr int others = 4; -}; // namespace op_kind - -op_sorting_visitor_t::rule_func op_sorting_visitor_t::create_same_kind_rule() { - return [](std::vector &op_seq, const op_dep_matrix_t &dep_matrix) { - // op kind taxonomy - auto get_op_kind = [](const sc_op_ptr &cur) { - if (cur->isa()) { - return op_kind::elementwise; - } else if (auto belemop = cur->dyn_cast()) { - auto anchor_id = cur->dyn_cast()->anchor_id_; - if (belemop->get_broadcast_input() >= 0) { - return op_kind::broadcast; - } else { - return op_kind::elementwise; - } - } else if (cur->isa()) { - return op_kind::reduce; - } else { - // Movement kind of op etc. - return op_kind::others; - } - }; - // use a map to record same kind ops and their index in topology - // sequence - std::unordered_map> op_name_idx_map; - for (int i = op_seq.size() - 1; i >= 0; i--) { - auto kind = get_op_kind(op_seq[i]); - if (kind != op_kind::others) { - op_name_idx_map[kind].emplace_back(i); - } - } - // Iterate map, and find opportunity for reorder. - for (auto &m : op_name_idx_map) { - int pre_idx = m.second.at(0); - for (auto &cur_idx : m.second) { - // if not neighboring, try to make them closer, note that pre is - // larger than cur here - if ((pre_idx - cur_idx) > 1) { - move_op_from_to(op_seq, dep_matrix, cur_idx, pre_idx); - } - pre_idx = cur_idx; - } - } - }; -} - -op_visitor_t::updater_func -op_visitor_t::create_DAG_updater(size_t total_hint, - const user_sort_func &sorter) { - struct count_t { - int count = -1; - }; - // the count of pending depending logical tensors for each node - std::vector pending_count; - return [pending_count, total_hint, sorter](op_visitor_t *v, - const sc_op_ptr &cur) mutable { - v->set_visited(cur->logical_op_id_); - for (auto < : cur->get_outputs()) { - auto visit_index = sorter(lt); - for (auto &idx : visit_index) { - auto user = lt->uses_[idx]; - auto id = user.second->logical_op_id_; - assert(id >= 0); - if ((unsigned)id >= pending_count.size()) { - // need to extend pending_count - if ((unsigned)id < total_hint) { - pending_count.resize(total_hint); - } else { - pending_count.resize((id + 1) * 1.5f); - } - } - if (pending_count[id].count == -1) { - // we have not met it before, initialize the dependency - // count - pending_count[id].count = user.second->get_inputs().size(); - } - // the pending count is decreased by 1 because current node is - // done - --pending_count[id].count; - assert(pending_count[id].count >= 0); - // all dependencies resolved, we can visit it now - if (pending_count[id].count == 0) { - v->to_visit_.emplace_back(user.second); - } - } - } - }; -} - -op_visitor_t::updater_func -op_visitor_t::create_DAG_updater_post(size_t total_hint) { - struct count_t { - int count = -1; - }; - // the count of pending depending logical tensors for each node - std::vector pending_count; - return [pending_count, total_hint](op_visitor_t *v, - const sc_op_ptr &cur) mutable { - v->set_visited(cur->logical_op_id_); - for (auto < : cur->get_inputs()) { - auto id = lt->producer_owner_->logical_op_id_; - if (v->has_visited(id)) { - continue; - } - assert(id >= 0); - if ((unsigned)id >= pending_count.size()) { - // need to extend pending_count - if ((unsigned)id < total_hint) { - pending_count.resize(total_hint); - } else { - pending_count.resize((id + 1) * 1.5f); - } - } - if (pending_count[id].count == -1) { - // we have not met it before, initialize the dependency - // count - size_t num_count = 0; - for (const graph_tensor_ptr &out : lt->producer_owner_->get_outputs()) { - num_count += out->uses_.size(); - } - pending_count[id].count = num_count; - } - // the pending count is decreased by 1 because current node is - // done - --pending_count[id].count; - assert(pending_count[id].count >= 0); - // all dependencies resolved, we can visit it now - if (pending_count[id].count == 0) { - v->to_visit_.emplace_back(lt->producer_owner_->shared_from_this()); - } - } - }; -} - -static std::vector usr_speculative_sorter(const graph_tensor_ptr >) { - std::vector visit_index; - visit_index.reserve(gt->uses_.size()); - std::unordered_set visited_set; - // Step 1: sorted by tunable op cnt - std::vector> priority_index_list; - auto sort_by_prority = [&priority_index_list, &visit_index]() { - std::sort( - priority_index_list.begin(), priority_index_list.end(), - [](const std::pair &p1, const std::pair &p2) { - return p1.first > p2.first; - }); - for (auto &p : priority_index_list) { - visit_index.emplace_back(p.second); - } - priority_index_list.clear(); - }; - // counter tunable op users - for (size_t i = 0; i < gt->uses_.size(); i++) { - auto u = gt->uses_[i]; - auto tun_cnt = count_tuneop_linearly(u.second, 15); - if (tun_cnt > 0) { - priority_index_list.emplace_back(std::make_pair(tun_cnt, i)); - visited_set.insert(i); - } - } - // sort tunable_cnt_index_list by descend - sort_by_prority(); - // Step 2: sorted by user tensor size - for (size_t i = 0; i < gt->uses_.size(); i++) { - if (visited_set.find(i) != visited_set.end()) - continue; - auto u = gt->uses_[i]; - if (u.second->get_outputs().empty()) - continue; - auto dt = u.second->get_outputs()[0]->details_; - size_t user_tsr_size = utils::get_sizeof_etype(dt.dtype_.type_code_) * - get_dims_product(dt.get_blocking_dims()); - priority_index_list.emplace_back(std::make_pair(user_tsr_size, i)); - visited_set.insert(i); - } - // sort tensor_size_index_list by descend - sort_by_prority(); - // Step 3: push remaining - for (size_t i = 0; i < gt->uses_.size(); i++) { - if (visited_set.find(i) != visited_set.end()) - continue; - visit_index.emplace_back(i); - } - return visit_index; -} - -op_visitor_t::updater_func -op_visitor_t::create_DAG_updater_speculative(size_t total_hint) { - return create_DAG_updater(total_hint, usr_speculative_sorter); -} - -sc_op_ptr op_visitor_t::pop_back_selector(op_visitor_t *v) { - auto ret = v->to_visit_.back(); - v->to_visit_.pop_back(); - if (v->has_visited(ret->logical_op_id_)) { - return nullptr; - } - return ret; -} - -op_visitor_t op_visitor_t::dfs() { - return op_visitor_t(pop_back_selector, push_back_updater, true); -} - -sc_op_ptr op_visitor_t::dequeue_selector(op_visitor_t *v) { - auto ret = v->to_visit_.front(); - v->to_visit_.pop_front(); - if (v->has_visited(ret->logical_op_id_)) { - return nullptr; - } - return ret; -} - -op_visitor_t op_visitor_t::bfs() { - return op_visitor_t(dequeue_selector, push_back_updater, true); -} - -op_visitor_t op_visitor_t::dfs_topology_sort(size_t total_nodes_hint) { - return op_visitor_t(pop_back_selector, create_DAG_updater(total_nodes_hint), - true); -} - -op_visitor_t -op_visitor_t::dfs_topology_speculative_sort(size_t total_nodes_hint) { - return op_visitor_t(pop_back_selector, - create_DAG_updater_speculative(total_nodes_hint), true); -} - -op_visitor_t op_visitor_t::bfs_topology_sort(size_t total_nodes_hint) { - return op_visitor_t(dequeue_selector, create_DAG_updater(total_nodes_hint), - true); -} - -op_visitor_t op_visitor_t::bfs_unchecked() { - return op_visitor_t(dequeue_selector, push_back_updater, false); -} - -op_visitor_t -op_visitor_t::dfs_topology_sort_unchecked(size_t total_nodes_hint) { - return op_visitor_t(pop_back_selector, create_DAG_updater(total_nodes_hint), - false); -} - -void op_visitor_t::post_visit_graph(const sc_graph_t &mgr, - const visitor_func &f) { - for (auto &v : mgr.ops_) { - if (dynamic_cast(v.get()) || - dynamic_cast(v.get())) { - to_visit_.emplace_back(v); - } - } - int original_number_of_ops = mgr.ops_.size(); - visit(f); - if (check_all_ops_visited_) { - assert_all_ops_visited(mgr, original_number_of_ops); - } -} - -sc_op_ptr search_tuneop_linearly(const sc_op_ptr &start_node, int max_step) { - auto next_node = start_node; - if (next_node->isa()) - return next_node; - int step = 1; - while (next_node->is_single_output_single_use()) { - next_node = next_node->get_outputs()[0]->uses_[0].second; - if (next_node->isa()) - return next_node; - if (step >= max_step) - return nullptr; - ++step; - } - return nullptr; -} - -int count_tuneop_linearly(const sc_op_ptr &start_node, int step) { - int cnt = 0; - auto next_node = start_node; - if (next_node->isa()) - cnt++; - while (next_node->is_single_output_single_use() && step > 0) { - next_node = next_node->get_outputs()[0]->uses_[0].second; - if (next_node->isa()) - cnt++; - step--; - } - return cnt; -} - -std::vector search_tuneop_bypass(const context_ptr &ctx, - const sc_op_ptr &tuneop, - const sc_op_ptr &start_node, - const op_dep_matrix_t &dep, - int max_step) { - if (!tuneop) - return {}; - auto next_node = start_node; - int step = 1; - std::vector bypass_ops; - bool found = false; - while (next_node->is_single_output_single_use()) { - // This is input fusion, rather than pre-op fusion - if (next_node == tuneop) - break; - // found bypass - if (dep.lookup(tuneop, next_node) == 1) { - found = true; - break; - } - bypass_ops.emplace_back(next_node); - next_node = next_node->get_outputs()[0]->uses_[0].second; - if ((step++) >= max_step) - break; - } - if (found) { - return bypass_ops; - } - return {}; -} -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/graph/visitor.hpp b/legacy/core/src/compiler/ir/graph/visitor.hpp deleted file mode 100644 index 93901b421..000000000 --- a/legacy/core/src/compiler/ir/graph/visitor.hpp +++ /dev/null @@ -1,301 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_VISITOR_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_GRAPH_VISITOR_HPP - -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -/** - * The util for traversing the OP graph. It contains key functions: selector and - * updater. Selector will pick the nodes to visit from the "queue": `to_visit_`. - * Updater is called after visiting a node, which "pushes" the nodes to visit in - * the `to_visit_` "queue". Different selectors and updaters results in - * different visiting orders. This class has some pre-defined selectors and - * updaters. - * Note: `op_visitor_t::visit_graph()` can be called only once. Please create a - * new object for each visit. - * */ -class SC_INTERNAL_API op_visitor_t { -public: - // the queue/stack for the nodes to visit - std::list to_visit_; - // the array to memorize the nodes that we have visited, indexed by the op - // id - std::vector visited_; - - using visitor_func = std::function; - using updater_func = std::function; - using selector_func = std::function; - // the selector to return the next node to visit in `to_visit_` list. It - // should also remove the node from the list. It can return null if it finds - // a node that has been visited. The visitor will try to call it again - selector_func select_next_node; - // will be called after a node has been visited. Usually it should update - // the `visited_`, and push/enqueue the sub-nodes to the `to_visit_` - updater_func update_visit_list; - // if true, when the user call `visit_graph()` or `post_visit_graph()`, we - // will check whether all ops in the graph are visited (not including the - // ops which are newly created and inserted into the graph during the - // visit). If any ops are not visited, an exception will be thrown - const bool check_all_ops_visited_; - - void visit(const visitor_func &f); - - // set a node as visited - void set_visited(int id); - // returns if an id is in the visited node set - bool has_visited(int id); - - void visit_graph(const sc_graph_t &mgr, const visitor_func &f); - - op_visitor_t(selector_func select_next_node_func, - updater_func update_visit_list_func, bool check_all_ops_visited); - - // updates the visitor states after a node is visited. It can be also used - // when a new node replaces an old one. Users should call this function with - // the new node - void update_state_for_visited(sc_op_ptr node); - - // the updater which pushes all uses of all output logical tensors - // to the back of the to_visit list - static void push_back_updater(op_visitor_t *, const sc_op_ptr &sc_op_ptr); - - // For DAG updater, its user visit order can be specified as demand, the - // return vector is the index of multi-users - using user_sort_func = - std::function(const graph_tensor_ptr >)>; - // the updater which pushes all nodes whose dependencies have already been - // visited. Used in topology sort. The default user order is sorted by - // ascend - static updater_func create_DAG_updater( - size_t total_nodes_hint, - const user_sort_func &func = - [](const graph_tensor_ptr >) -> std::vector { - std::vector usr_vis_ord(gt->uses_.size()); - std::iota(usr_vis_ord.begin(), usr_vis_ord.end(), 0); - return usr_vis_ord; - }); - static updater_func create_DAG_updater_post(size_t total_nodes_hint); - // Different from default create_DAG_updater, it has specific user sort func - static updater_func create_DAG_updater_speculative(size_t total_nodes_hint); - // create_DAG_updatater_post; - // post order traversing - void post_visit_graph(const sc_graph_t &mgr, const visitor_func &f); - - // the selector which pops a node in `to_visit_` from back - static sc_op_ptr pop_back_selector(op_visitor_t *v); - - // the selector which pops a node in `to_visit_` from front - static sc_op_ptr dequeue_selector(op_visitor_t *v); - - // constructs a DFS visitor, using push_back_updater and - // pop_back_selector - static op_visitor_t dfs(); - // constructs a BFS visitor, using push_back_updater and - // dequeue_selector - static op_visitor_t bfs(); - // constructs a topology sort visitor in DFS order, using - // create_DAG_updater and pop_back_selector - static op_visitor_t dfs_topology_sort(size_t total_nodes_hint = 30); - static op_visitor_t - dfs_topology_speculative_sort(size_t total_nodes_hint = 30); - // constructs a topology sort visitor in BFS order, using - // create_DAG_updater and dequeue_selector - static op_visitor_t bfs_topology_sort(size_t total_nodes_hint = 30); - // constructs a BFS visitor, and do not check whether all ops are visited - static op_visitor_t bfs_unchecked(); - // constructs a topology sort visitor in BFS order, and do not check whether - // all ops are visited - static op_visitor_t dfs_topology_sort_unchecked(size_t total_nodes_hint = 30); - -private: - // asserts that all ops in the graph whose index is within - // [0, concerned_size) have been visited - void assert_all_ops_visited(const sc_graph_t &mgr, size_t concerned_size); -}; - -/** Op Depenency Matrix - * This class is used to record depenency relationship between all graph - * ops. For n Ops in graph, it will create nxn matrix, in which: - * 1. = 1 represents the j-th OP depends on i-th OP - * 2. = -1 represents the i-th OP depends on j-th OP - * 3. = 0 represents i-th OP and j-th OP have no depenency - * @note both i and j are logical_op_id_ of original - * whole graph. - * */ -class op_dep_matrix_t { -private: - std::vector> matrix_; - int op_size_; - -public: - op_dep_matrix_t(const sc_graph_t &); - // initlize op_size x op_size matrix by default value of zero - op_dep_matrix_t(int op_size) - : matrix_(op_size, std::vector(op_size, 0)), op_size_(op_size) {} - // update depenency matrix according topology sort - void update(const sc_op_ptr &cur); - /** lookup function for matrix, it will return depenency relationship - * return 1 represents j-th op depends on i-th op. - * return -1 represents i-th op depends on j-th op - * return 0 represents i-th op and j-th op have no depenency - * */ - int lookup(int i, int j) const { - COMPILE_ASSERT(i >= 0 && i < op_size_ && j >= 0 && j < op_size_, - "illegal lookup index for depenency matrix."); - return matrix_[i][j]; - } - - int lookup(sc_op *op_i, sc_op *op_j) const { - int i = op_i->logical_op_id_, j = op_j->logical_op_id_; - return lookup(i, j); - } - - int lookup(const sc_op_ptr &op_i, const sc_op_ptr &op_j) const { - return lookup(op_i.get(), op_j.get()); - } - - // look up all op id which depend on given op id - std::vector lookup_ops_depend_on(int depend_i) const { - COMPILE_ASSERT(depend_i >= 0 && depend_i < op_size_, - "illegal lookup index for depenency matrix."); - std::vector ret; - for (int j = 0; j < op_size_; j++) { - if (matrix_[depend_i][j] == 1) { - ret.emplace_back(j); - } - } - return ret; - } -}; - -using dep_mat_ptr = std::shared_ptr; - -/** - * op_sorting_visitor_t is sort based, and visiting order is defined by the user - * sort function or rules. op_visitor_t is queue/iteration based. - * */ -class op_sorting_visitor_t { -public: - /** - * pre-defined optimize rules are listed as enumurate type, users can - * directly use this enumuate kind rules. Actually, it will call pre-defined - * create_xxx_rule function. @note multiple optimzied rules are supported, - * so these rules would be sorted by priority if more than one rules are - * used. - * */ - enum class sort_rule : int { - same_kind = 0, - // TODO(xxx): Add other pre-defined rules here. - }; - - /** - * user-defined optimize rules are the lambda function, which will accept - * two arguements. This function is expected to reorder op_seq according - * given rules and dep_matrix. - * @param op_seq: op sequence generated by op_visitor_t. - * @param dep_matrix: op depenency matrix. - * */ - using rule_func = std::function &op_seq, - const op_dep_matrix_t &dep_matrix)>; - - /** Visit topology sequence by given rules. - * This function will split this into three stages. First, it will visit all - * ops in graph.ops and generate initial sequence. Meanwhile, it will record - * its dependency in a Adjacent Matrix. Secondly, it may change the op - * visiting order by the given visit rules. Finally, all ops will be excuted - * by new order. - * @note This visit function have two versions: one is suitable for - * pre-defined rules, and another is more suitable for user-defined rule. - * @param graph: sc_graph object - * @param f: visiting function for each op in graph - * @param rules: (Version 1) the list of pre-defined rules, it will be - * reordered by priority inside. - * @param f_fule (Version 2) user-defined lambda rule function. - * */ - static void visit_by_rules(sc_graph_t &graph, - const std::function &f, - std::vector rules); - static void visit_by_rules(sc_graph_t &graph, - const std::function &f, - const rule_func &f_rule); - - static std::vector sort_by_rules(sc_graph_t &graph, - std::vector rules); - static std::vector sort_by_rules(sc_graph_t &graph, - const rule_func &f_rule); - - /** create pre-defined same_kind rules - * Due to same_kind op like elementwise_op, broadcast_op_t or reduce_op_t - * may have generate similar loop in IR, which will expose more opportunity - * for Loop Merge Pass, we can make them as neighboring as possible. - * */ - static rule_func create_same_kind_rule(); -}; - -// count amount of tuneable op -int count_tuneop_linearly(const sc_op_ptr &start_node, int step); - -// search first tunable op linearly -sc_op_ptr search_tuneop_linearly(const sc_op_ptr &start_node, int max_step = 5); - -/** - * What is bypass: it starts from the certain op which has more than one user - * ops, and one of them is tunable op. it means fuse op pass will reparition - * fused graph starting that tunable op. - * E.g. - * in_a in_b - * \ / - * matmul2d in_c - * | / - * bias - * | - * in_d quan - * \ | \ - * matmul2d dequan - * | / - * add - * | - * output - * - * In graph above, `deq` or `cast+sub+mul`(after graph inline) is the bypass - * what we want to search. For each op found in bypsas, it can be fused either - * previous or post op, global reschedule is aimed to mark suitable fuse attr - * (break pre/post fuse or even no fused) for each op them according several - * different rules. - * */ -std::vector search_tuneop_bypass(const context_ptr &ctx, - const sc_op_ptr &tuneop, - const sc_op_ptr &start_node, - const op_dep_matrix_t &dep, - int max_step = 10); -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/intrinsics.cpp b/legacy/core/src/compiler/ir/intrinsics.cpp deleted file mode 100644 index 9a805ff54..000000000 --- a/legacy/core/src/compiler/ir/intrinsics.cpp +++ /dev/null @@ -1,644 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "intrinsics.hpp" -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -void intrinsic_handler_t::to_string(const intrin_call_c &v, - ir_printer_t *printer) const { - printer->os_ << name_ << '('; - if (!v->args_.empty()) { - for (unsigned i = 0; i < v->args_.size() - 1; i++) { - printer->do_dispatch(v->args_.at(i)) << ", "; - } - printer->do_dispatch(v->args_.back()); - } - printer->os_ << ')'; -} - -intrinsic_handler_t::intrinsic_handler_t(const std::string &name) - : name_(name) {} - -x86_intrinsic_handler_t::x86_intrinsic_handler_t(const std::string &name) - : intrinsic_handler_t(name) {} - -struct binary_intrinsic_handler_t : public intrinsic_handler_t { - binary_intrinsic_handler_t(const std::string &name) - : intrinsic_handler_t(name) {} - void on_initialize(intrin_call_node &node) override; -}; - -void binary_intrinsic_handler_t::on_initialize(intrin_call_node &node) { - assert(node.args_.size() == 2); - auto &l = node.args_[0]; - auto &r = node.args_[1]; - node.dtype_ = l->dtype_ == r->dtype_ ? l->dtype_ : datatypes::undef; -} - -struct trinary_intrinsic_handler_t : public intrinsic_handler_t { - trinary_intrinsic_handler_t(const std::string &name) - : intrinsic_handler_t(name) {} - void on_initialize(intrin_call_node &node) override; -}; - -void trinary_intrinsic_handler_t::on_initialize(intrin_call_node &node) { - assert(node.args_.size() == 3); - auto &a = node.args_[0]; - auto &b = node.args_[1]; - auto &c = node.args_[2]; - if (node.type_ == intrin_type::permute || - node.type_ == intrin_type::shuffle) { - node.dtype_ = a->dtype_ == b->dtype_ ? a->dtype_ : datatypes::undef; - } else if (node.type_ == intrin_type::permutex2var) { - node.dtype_ = a->dtype_ == c->dtype_ ? a->dtype_ : datatypes::undef; - } else { - node.dtype_ = a->dtype_ == b->dtype_ && a->dtype_ == c->dtype_ - ? a->dtype_ - : datatypes::undef; - } -} - -struct min_handler_t : public binary_intrinsic_handler_t { - min_handler_t() : binary_intrinsic_handler_t("min") {} -}; - -struct max_handler_t : public binary_intrinsic_handler_t { - max_handler_t() : binary_intrinsic_handler_t("max") {} -}; - -struct abs_handler_t : public intrinsic_handler_t { - void on_initialize(intrin_call_node &node) override { - assert(node.args_.size() == 1); - node.dtype_ = node.args_[0]->dtype_; - } - abs_handler_t() : intrinsic_handler_t("abs") {} -}; - -struct round_handler_t : public intrinsic_handler_t { - void on_initialize(intrin_call_node &node) override { - assert(node.args_.size() == 1); - node.dtype_ = node.args_[0]->dtype_; - } - round_handler_t() : intrinsic_handler_t("round") {} -}; - -struct ceil_handler_t : public intrinsic_handler_t { - void on_initialize(intrin_call_node &node) override { - assert(node.args_.size() == 1); - node.dtype_ = node.args_[0]->dtype_; - } - ceil_handler_t() : intrinsic_handler_t("ceil") {} -}; - -struct floor_handler_t : public intrinsic_handler_t { - void on_initialize(intrin_call_node &node) override { - assert(node.args_.size() == 1); - node.dtype_ = node.args_[0]->dtype_; - } - floor_handler_t() : intrinsic_handler_t("floor") {} -}; - -struct exp_handler_t : public intrinsic_handler_t { - void on_initialize(intrin_call_node &node) override { - assert(node.args_.size() == 1); - node.dtype_ = node.args_[0]->dtype_; - } - exp_handler_t() : intrinsic_handler_t("exp") {} -}; - -struct log_handler_t : public intrinsic_handler_t { - void on_initialize(intrin_call_node &node) override { - assert(node.args_.size() == 1); - node.dtype_ = node.args_[0]->dtype_; - } - log_handler_t() : intrinsic_handler_t("log") {} -}; - -struct erf_handler_t : public intrinsic_handler_t { - void on_initialize(intrin_call_node &node) override { - assert(node.args_.size() == 1); - node.dtype_ = node.args_[0]->dtype_; - } - erf_handler_t() : intrinsic_handler_t("erf") {} -}; - -struct sqrt_handler_t : public intrinsic_handler_t { - void on_initialize(intrin_call_node &node) override { - assert(node.args_.size() == 1); - node.dtype_ = node.args_[0]->dtype_; - } - sqrt_handler_t() : intrinsic_handler_t("sqrt") {} -}; - -struct rsqrt_handler_t : public intrinsic_handler_t { - void on_initialize(intrin_call_node &node) override { - assert(node.args_.size() == 1); - node.dtype_ = node.args_[0]->dtype_; - } - rsqrt_handler_t() : intrinsic_handler_t("rsqrt") {} -}; - -struct reduce_add_handler_t : public intrinsic_handler_t { - void on_initialize(intrin_call_node &node) override { - assert(node.args_.size() == 1); - node.dtype_ = node.args_[0]->dtype_; - node.dtype_.lanes_ = 1; - } - reduce_add_handler_t() : intrinsic_handler_t("reduce_add") {} -}; - -struct reduce_mul_handler_t : public intrinsic_handler_t { - void on_initialize(intrin_call_node &node) override { - assert(node.args_.size() == 1); - node.dtype_ = node.args_[0]->dtype_; - node.dtype_.lanes_ = 1; - } - reduce_mul_handler_t() : intrinsic_handler_t("reduce_mul") {} -}; - -struct reduce_max_handler_t : public intrinsic_handler_t { - void on_initialize(intrin_call_node &node) override { - assert(node.args_.size() == 1); - node.dtype_ = node.args_[0]->dtype_; - node.dtype_.lanes_ = 1; - } - reduce_max_handler_t() : intrinsic_handler_t("reduce_max") {} -}; - -struct reduce_min_handler_t : public intrinsic_handler_t { - void on_initialize(intrin_call_node &node) override { - assert(node.args_.size() == 1); - node.dtype_ = node.args_[0]->dtype_; - node.dtype_.lanes_ = 1; - } - reduce_min_handler_t() : intrinsic_handler_t("reduce_min") {} -}; - -struct broadcast_handler_t : public intrinsic_handler_t { - void on_initialize(intrin_call_node &node) override { - assert(node.args_.size() == 1); - auto lanes = node.intrin_attrs_->get("lanes"); - COMPILE_ASSERT(lanes <= 512, "Expecting lanes<=512"); - node.dtype_ = node.args_[0]->dtype_; - node.dtype_.lanes_ = lanes; - } - broadcast_handler_t() : intrinsic_handler_t("broadcast") {} -}; - -struct fmadd_handler_t : public trinary_intrinsic_handler_t { - void on_initialize(intrin_call_node &node) override { - node.dtype_ = node.args_[0]->dtype_; - } - fmadd_handler_t() : trinary_intrinsic_handler_t("fmadd") {} -}; - -struct fnmadd_handler_t : public trinary_intrinsic_handler_t { - void on_initialize(intrin_call_node &node) override { - node.dtype_ = node.args_[0]->dtype_; - } - fnmadd_handler_t() : trinary_intrinsic_handler_t("fnmadd") {} -}; - -struct unpack_low_handler_t : public binary_intrinsic_handler_t { - unpack_low_handler_t() : binary_intrinsic_handler_t("unpack_low") {} -}; - -struct unpack_high_handler_t : public binary_intrinsic_handler_t { - unpack_high_handler_t() : binary_intrinsic_handler_t("unpack_high") {} -}; - -struct shuffle_handler_t : public binary_intrinsic_handler_t { - shuffle_handler_t() : binary_intrinsic_handler_t("shuffle") {} -}; - -struct permute_handler_t : public binary_intrinsic_handler_t { - permute_handler_t() : binary_intrinsic_handler_t("permute") {} -}; - -struct reinterpret_handler_t : public intrinsic_handler_t { - void on_initialize(intrin_call_node &node) override { - assert(node.args_.size() == 1); - node.dtype_ = - node.intrin_attrs_->get(intrin_attr::out_dtype); - } - reinterpret_handler_t() : intrinsic_handler_t("reinterpret") {} -}; - -struct permutex2var_handler_t : public trinary_intrinsic_handler_t { - permutex2var_handler_t() : trinary_intrinsic_handler_t("permutex2var") {} -}; - -struct permutexvar_handler_t : public binary_intrinsic_handler_t { - void on_initialize(intrin_call_node &node) override { - COMPILE_ASSERT(node.args_.size() == 2, "Expecting 2 args."); - node.dtype_ = node.args_[1]->dtype_; - } - permutexvar_handler_t() : binary_intrinsic_handler_t("permutexvar") {} -}; - -struct insert_handler_t : public trinary_intrinsic_handler_t { - void on_initialize(intrin_call_node &node) override { - assert(node.args_.size() == 2 || node.args_.size() == 3); - node.dtype_ = node.args_[0]->dtype_; - } - insert_handler_t() : trinary_intrinsic_handler_t("insert") {} -}; - -struct extract_handler_t : public intrinsic_handler_t { - void on_initialize(intrin_call_node &node) override { - assert(node.args_.size() == 1 || node.args_.size() == 2); - if (node.args_.size() == 1) { - node.dtype_ = sc_data_type_t(node.args_[0]->dtype_.type_code_); - auto lanes = node.intrin_attrs_->get("lanes"); - if (lanes > 1) { - node.dtype_.lanes_ = lanes; - } - } else { - auto rows = node.intrin_attrs_->get("rows"); - auto cols = node.intrin_attrs_->get("cols"); - auto lanes = rows > 0 ? (rows * cols) : cols; - node.dtype_ = - sc_data_type_t(node.args_[0]->dtype_.type_code_, lanes, rows); - } - } - extract_handler_t() : intrinsic_handler_t("extract") {} -}; - -struct gather_handler_t : public binary_intrinsic_handler_t { - gather_handler_t() : binary_intrinsic_handler_t("gather") {} - void on_initialize(intrin_call_node &node) override { - assert(node.args_[1]->dtype_.is_etype(sc_data_etype::S32)); - node.dtype_ = node.args_[0]->dtype_.get_pointer_element(); - node.dtype_.lanes_ = node.args_[1]->dtype_.lanes_; - } -}; - -struct round_and_cast_handler_t : public intrinsic_handler_t { - void on_initialize(intrin_call_node &node) override { - assert(node.args_.size() == 1); - node.dtype_ = - node.intrin_attrs_->get(intrin_attr::out_dtype); - COMPILE_ASSERT(node.dtype_.lanes_ == node.args_[0]->dtype_.lanes_ && - (node.dtype_.type_code_ == sc_data_etype::S32 || - (node.dtype_.type_code_ == sc_data_etype::U32 && - node.dtype_.lanes_ > 1)) && - node.args_[0]->dtype_.type_code_ == sc_data_etype::F32, - "round_and_cast cannot handle " << node.args_[0]->dtype_ - << "->" << node.dtype_); - } - round_and_cast_handler_t() : intrinsic_handler_t("round_and_cast") {} -}; - -struct isnan_handler_t : public intrinsic_handler_t { - void on_initialize(intrin_call_node &node) override { - assert(node.args_.size() == 1); - node.dtype_ = sc_data_type_t::boolean(node.dtype_.lanes_); - } - isnan_handler_t() : intrinsic_handler_t("isnan") {} -}; - -struct saturated_cast_handler_t : public intrinsic_handler_t { - void on_initialize(intrin_call_node &node) override { - assert(node.args_.size() == 1); - node.dtype_ = node.intrin_attrs_->get("out_dtype"); - } - saturated_cast_handler_t() : intrinsic_handler_t("saturated_cast") {} -}; - -struct shl_handler_t : public intrinsic_handler_t { - void on_initialize(intrin_call_node &node) override { - assert(node.args_.size() == 2); - node.dtype_ = node.args_[0]->dtype_; - } - shl_handler_t() : intrinsic_handler_t("shl") {} -}; - -struct shr_handler_t : public intrinsic_handler_t { - void on_initialize(intrin_call_node &node) override { - assert(node.args_.size() == 2); - node.dtype_ = node.args_[0]->dtype_; - } - shr_handler_t() : intrinsic_handler_t("shr") {} -}; - -struct int_and_handler_t : public binary_intrinsic_handler_t { - int_and_handler_t() : binary_intrinsic_handler_t("int_and") {} -}; - -struct int_or_handler_t : public binary_intrinsic_handler_t { - int_or_handler_t() : binary_intrinsic_handler_t("int_or") {} -}; - -struct int_xor_handler_t : public binary_intrinsic_handler_t { - int_xor_handler_t() : binary_intrinsic_handler_t("int_xor") {} -}; - -sc_data_type_t get_dtype_from_struct_and_field(const std::string &in, - int field) { - if (in == dyn_tsr_struct_t::name) { - return dyn_tsr_struct_t::dtypes[field]; - } else { - COMPILE_ASSERT(false, "struct " << in << " has not been supported!"); - } - return sc_data_type_t(); -} - -struct read_struct_handler_t : public intrinsic_handler_t { - void on_initialize(intrin_call_node &node) override { - assert(node.args_.size() == 1); - assert(node.intrin_attrs_->has_key(intrin_attr::struct_name) && - node.intrin_attrs_->has_key(intrin_attr::struct_field)); - node.dtype_ = get_dtype_from_struct_and_field( - node.intrin_attrs_->get(intrin_attr::struct_name), - node.intrin_attrs_->get(intrin_attr::struct_field)); - } - read_struct_handler_t() : intrinsic_handler_t("read_struct") {} -}; - -struct write_struct_handler_t : public binary_intrinsic_handler_t { - void on_initialize(intrin_call_node &node) override { - assert(node.intrin_attrs_->has_key(intrin_attr::struct_name) && - node.intrin_attrs_->has_key(intrin_attr::struct_field)); - node.dtype_ = datatypes::void_t; - } - write_struct_handler_t() : binary_intrinsic_handler_t("write_struct") {} -}; - -struct brgemm_handler_t : public intrinsic_handler_t { - size_t arg_cnt_; - void on_initialize(intrin_call_node &node) override { - assert(node.check_brgemm_arg_size(arg_cnt_)); - node.dtype_ = datatypes::void_t; - } - brgemm_handler_t(size_t arg_cnt, const char *name) - : intrinsic_handler_t(name), arg_cnt_(arg_cnt) {} -}; - -struct set_thread_idle_func_handler_t : public intrinsic_handler_t { - void on_initialize(intrin_call_node &node) override { - node.dtype_ = datatypes::void_t; - COMPILE_ASSERT(node.args_.size() >= 2UL, - "set_thread_idle_func requires more than 2 args"); - COMPILE_ASSERT(node.args_[0]->dtype_ == datatypes::pointer, - "The first arg of set_thread_idle_func should be pointer"); - } - set_thread_idle_func_handler_t() - : intrinsic_handler_t("set_thread_idle_func") {} -}; - -struct prefetch_handler_t : public intrinsic_handler_t { - void on_initialize(intrin_call_node &node) override { - node.dtype_ = datatypes::void_t; - COMPILE_ASSERT(node.args_.size() == 1, "prefetch requires 1 arg"); - COMPILE_ASSERT(node.args_[0]->dtype_.is_pointer(), - "The first arg of prefetch should be pointer"); - auto locality = node.intrin_attrs_->get_or_else("locality", -1); - COMPILE_ASSERT(locality >= 0 && locality <= 3, - "locality attr of prefetch must be between 0 to 3"); - } - prefetch_handler_t() : intrinsic_handler_t("prefetch") {} -}; - -struct constant_load_handler_t : public intrinsic_handler_t { - void on_initialize(intrin_call_node &node) override { - assert(node.args_.size() == 1); - node.dtype_ = node.args_[0]->dtype_; - } - constant_load_handler_t() : intrinsic_handler_t("constant_load") {} -}; - -struct volatile_load_handler_t : public intrinsic_handler_t { - void on_initialize(intrin_call_node &node) override { - assert(node.args_.size() == 1); - node.dtype_ = node.args_[0]->dtype_; - } - volatile_load_handler_t() : intrinsic_handler_t("volatile_load") {} -}; - -struct get_group_id_handler_t : public intrinsic_handler_t { - void on_initialize(intrin_call_node &node) override { - assert(node.args_.size() == 1); - node.dtype_ = datatypes::u32; - } - get_group_id_handler_t() : intrinsic_handler_t("get_group_id") {} -}; - -struct get_group_thread_id_handler_t : public intrinsic_handler_t { - void on_initialize(intrin_call_node &node) override { - assert(node.args_.size() == 1); - node.dtype_ = datatypes::s32; - } - get_group_thread_id_handler_t() - : intrinsic_handler_t("get_group_thread_id") {} -}; - -struct avx_broadcast_idx_handler_t : public x86_intrinsic_handler_t { - void on_initialize(low_level_intrin_node &node) override { - assert(node.args_.size() == 3); - assert(node.args_[0]->dtype_.is_pointer()); - auto lanes = node.intrin_attrs_->get("lanes"); - node.dtype_ = sc_data_type_t( - etypes::get_pointer_element(node.args_[0]->dtype_.type_code_), lanes); - } - avx_broadcast_idx_handler_t() - : x86_intrinsic_handler_t("avx_broadcast_idx") {} -}; - -struct avx_mask_cast_handler_t : public x86_intrinsic_handler_t { - void on_initialize(low_level_intrin_node &node) override { - assert(node.args_.size() == 1); - node.dtype_ = node.intrin_attrs_->get("dtype"); - } - avx_mask_cast_handler_t() : x86_intrinsic_handler_t("avx_mask_cast") {} -}; - -struct avx_compare_handler_t : public x86_intrinsic_handler_t { - void on_initialize(low_level_intrin_node &node) override { - assert(node.args_.size() == 3); - assert(node.args_[2].isa()); - assert(node.args_[0]->dtype_ == node.args_[1]->dtype_); - node.dtype_ = node.args_[0]->dtype_; - } - avx_compare_handler_t() : x86_intrinsic_handler_t("avx_compare") {} -}; - -namespace brgemm_args { -sc_data_type_t arg_types[NUM_FULL_ARGS_STRIDE] = { - datatypes::pointer, // A (overloaded) - datatypes::pointer, // B - datatypes::pointer, // C - datatypes::s32, // num - datatypes::s32, // M - datatypes::s32, // N - datatypes::s32, // K - datatypes::s32, // LDA - datatypes::s32, // LDB - datatypes::s32, // LDC - datatypes::s32, // stride_a - datatypes::s32, // stride_b - datatypes::pointer, // bias - datatypes::pointer, // scales - datatypes::pointer, // binary_post_ops_rhs - datatypes::index, // oc_logical_off - datatypes::index, // dst_row_logical_off - datatypes::pointer, // data_C_ptr - datatypes::index, // first_mb_matrix_addr_off - datatypes::pointer, // a_zp_compensations - datatypes::pointer, // b_zp_compensations - datatypes::pointer, // c_zp_values - datatypes::boolean, // skip_accumulation - datatypes::s32, // zp_a_val - datatypes::boolean, // do_only_comp - datatypes::boolean, // do_only_zp_a_val - datatypes::pointer, // c_buf - datatypes::index, // bdmask_idx - datatypes::pointer, // top_pad - datatypes::pointer // bottom_pad -}; - -sc_data_type_t list_arg_types[NUM_FULL_ARGS_LIST] = { - datatypes::pointer, // A - datatypes::pointer, // B - datatypes::pointer, // C - datatypes::s32, // num - datatypes::s32, // M - datatypes::s32, // N - datatypes::s32, // K - datatypes::s32, // LDA - datatypes::s32, // LDB - datatypes::s32, // LDC - datatypes::s32, // stride_a - datatypes::s32, // stride_b - datatypes::s32, // len - datatypes::pointer, // bias - datatypes::pointer, // scales - datatypes::pointer, // binary_post_ops_rhs - datatypes::index, // oc_logical_off - datatypes::index, // dst_row_logical_off - datatypes::pointer, // data_C_ptr - datatypes::index, // first_mb_matrix_addr_off - datatypes::pointer, // a_zp_compensations - datatypes::pointer, // b_zp_compensations - datatypes::pointer, // c_zp_values - datatypes::boolean, // skip_accumulation - datatypes::s32, // zp_a_val - datatypes::boolean, // do_only_comp - datatypes::boolean, // do_only_zp_a_val - datatypes::pointer, // c_buf - datatypes::index, // bdmask_idx - datatypes::pointer, // top_pad - datatypes::pointer // bottom_pad -}; -} // namespace brgemm_args - -static std::unique_ptr handlers[] = { - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), - utils::make_unique(brgemm_args::NUM_FULL_ARGS_STRIDE, - "brgemm"), - utils::make_unique(brgemm_args::NUM_FULL_ARGS_LIST, - "list_brgemm"), -}; - -static_assert(sizeof(handlers) / sizeof(handlers[0]) == - int(intrin_type::NUM_INTRINSICS), - "Not all intrinsics are filled in handlers"); - -intrinsic_handler_t &get_intrinsic_handler(intrin_type intrin) { - return *handlers[static_cast(intrin)]; -} - -static std::unique_ptr x86_handlers[] = { - utils::make_unique(), - utils::make_unique(), - utils::make_unique(), -}; -x86_intrinsic_handler_t &get_x86_intrinsic_handler(int64_t intrin) { - return *x86_handlers[intrin]; -} - -static_assert(sizeof(x86_handlers) / sizeof(x86_handlers[0]) == - int(x86_intrin_type::NUM_INTRINSICS), - "Not all intrinsics are filled in x86 handlers"); - -#define OFFSET(struct_type, field) \ - (size_t)(&(((struct_type *)nullptr)->field)) // NOLINT - -const sc_data_type_t dyn_tsr_struct_t::dtypes[5] = { - datatypes::pointer, datatypes::pointer, datatypes::s32, datatypes::u32, - datatypes::u8}; -const size_t dyn_tsr_struct_t::offsets[5] = { - OFFSET(runtime::dynamic_tensor_t, data_), - OFFSET(runtime::dynamic_tensor_t, dims_), - OFFSET(runtime::dynamic_tensor_t, ndims_), - OFFSET(runtime::dynamic_tensor_t, dtype_), - OFFSET(runtime::dynamic_tensor_t, dyn_mask_)}; - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/intrinsics.hpp b/legacy/core/src/compiler/ir/intrinsics.hpp deleted file mode 100644 index 58a1be8ac..000000000 --- a/legacy/core/src/compiler/ir/intrinsics.hpp +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Copyright (C) 2025 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions - * and limitations under the License. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_INTRINSICS_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_INTRINSICS_HPP - -#include "sc_expr.hpp" -#include -#include -#include -#include -#include -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -class ir_visitor_t; -class ir_printer_t; -struct intrinsic_handler_t { - std::string name_; - virtual void on_initialize(intrin_call_node &node) = 0; - intrinsic_handler_t(const std::string &name); - virtual void to_string(const intrin_call_c &v, ir_printer_t *printer) const; - virtual ~intrinsic_handler_t() = default; -}; - -struct x86_intrinsic_handler_t : public intrinsic_handler_t { - virtual void on_initialize(intrin_call_node &node) {}; - virtual void on_initialize(low_level_intrin_node &node) = 0; - x86_intrinsic_handler_t(const std::string &name); - virtual ~x86_intrinsic_handler_t() = default; -}; - -// user defined struct for reading/writing at runtime. -struct dyn_tsr_struct_t { - static constexpr const char *name = "dyn_tsr"; - enum fields : int { data_ptr = 0, dim_ptr, ndims, dtype, dyn_mask }; - static const sc_data_type_t dtypes[5]; - static const size_t offsets[5]; -}; - -sc_data_type_t get_dtype_from_struct_and_field(const std::string &in, - int field); - -// the indices of arguments of the brgemm intrinsices -namespace brgemm_args { -constexpr int A = 0; -constexpr int B = 1; -constexpr int C = 2; -constexpr int NUM = 3; -constexpr int M = 4; -constexpr int N = 5; -constexpr int K = 6; -constexpr int LDA = 7; -constexpr int LDB = 8; -constexpr int LDC = 9; -constexpr int STRIDE_A = 10; -constexpr int STRIDE_B = 11; -constexpr int NUM_BASIC_ARGS_STRIDE = STRIDE_B + 1; -constexpr int LEN = 12; -constexpr int NUM_BASIC_ARGS_LIST = LEN + 1; -// extra +4 for c_buf, bdmask_idx, top_pad, bottom_pad -constexpr int NUM_FULL_ARGS_STRIDE = - NUM_BASIC_ARGS_STRIDE + brgemm::postops_data_init_func_nargs + 4; -// extra +4 for c_buf, bdmask_idx, top_pad, bottom_pad -constexpr int NUM_FULL_ARGS_LIST = - NUM_BASIC_ARGS_LIST + brgemm::postops_data_init_func_nargs + 4; - -struct cpu_t { - // use init_update or update - bool init_; - bool operator==(const cpu_t &other) const { return init_ == other.init_; } - bool operator!=(const cpu_t &other) const { return init_ != other.init_; } -}; - -namespace extra_args_offset { -constexpr int dtypeA = 0; -constexpr int dtypeB = 1; -constexpr int brg_attrs = 2; -constexpr int bd_mask = 3; -constexpr int postops_setting = 4; -constexpr int cache_nargs = postops_setting + 1; -constexpr int top_pad = 5; -constexpr int bottom_pad = 6; -constexpr int postops_data = 7; -constexpr int c_buf = 8; -constexpr int nargs = c_buf + 1; -} // namespace extra_args_offset - -struct extra_args_t { - bool is_cpu_; - sc_data_type_t dtype_A_ = datatypes::undef; // element dtype of mat A - sc_data_type_t dtype_B_ = datatypes::undef; // element dtype of mat B - sc_data_type_t dtype_C_ = datatypes::undef; // element dtype of mat C - sc_brgemm_attrs_t brg_attrs_; // brgemm attrs - sc_brgemm_bd_mask_t bd_mask_; // bd mask - int bd_mask_set_num_; // num of bd_mask set - sc_brgemm_postops_setting_t postops_setting_; // post ops setting - - union { - cpu_t cpu_; - }; - extra_args_t(const cpu_t &g, sc_data_type_t dtypeA, - sc_data_type_t dtypeB = datatypes::undef, - sc_data_type_t dtypeC = datatypes::undef, - const sc_brgemm_attrs_t &brg_attrs = sc_brgemm_attrs_t(), - const sc_brgemm_bd_mask_t &bd_mask = sc_brgemm_bd_mask_t(), - const int bd_mask_set_num = 1, - const sc_brgemm_postops_setting_t &brg_postops = - sc_brgemm_postops_setting_t()) - : is_cpu_(true), dtype_A_(dtypeA), - dtype_B_(dtypeB == datatypes::undef ? dtypeA : dtypeB), - dtype_C_(dtypeC == datatypes::undef ? dtypeA : dtypeC), - brg_attrs_(brg_attrs), bd_mask_(bd_mask), - bd_mask_set_num_(bd_mask_set_num), postops_setting_(brg_postops), - cpu_(g) {} - bool operator==(const extra_args_t &other) const { - return is_cpu_ == other.is_cpu_ && dtype_A_ == other.dtype_A_ && - dtype_B_ == other.dtype_B_ && dtype_C_ == other.dtype_C_ && - brg_attrs_ == other.brg_attrs_ && bd_mask_ == other.bd_mask_ && - bd_mask_set_num_ == other.bd_mask_set_num_ && - postops_setting_ == other.postops_setting_ && cpu_ == other.cpu_; - } - bool operator!=(const extra_args_t &other) const { return !(*this == other); } -}; - -extern sc_data_type_t arg_types[NUM_FULL_ARGS_STRIDE]; -extern sc_data_type_t list_arg_types[NUM_FULL_ARGS_LIST]; -} // namespace brgemm_args - -intrinsic_handler_t &get_intrinsic_handler(intrin_type intrin); -x86_intrinsic_handler_t &get_x86_intrinsic_handler(int64_t intrin); - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/ir_comparer.cpp b/legacy/core/src/compiler/ir/ir_comparer.cpp deleted file mode 100644 index f0dc66774..000000000 --- a/legacy/core/src/compiler/ir/ir_comparer.cpp +++ /dev/null @@ -1,157 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "builder.hpp" - -#include "ir_comparer.hpp" - -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -void ir_comparer::reset() { - same_ = true; - if (diff) - diff = utils::make_unique(); - var_mapping_.clear(); -} -ir_comparer::ir_comparer(bool needs_diff, bool cmp_names, bool cmp_var_ref, - bool cmp_callee, bool cmp_commutative) - : cmp_names_(cmp_names), cmp_callee_(cmp_callee), cmp_var_ref_(cmp_var_ref), - cmp_commutative_(cmp_commutative), same_(true) { - if (needs_diff) - diff = utils::make_unique(); -} - -bool ir_comparer::set_result(func_c l, func_c r, bool cond) { - if (same_ && !cond) { - same_ = false; - if (diff) { - diff->first_diff_func_.first = std::move(l); - diff->first_diff_func_.second = std::move(r); - } - } - return cond; -} - -bool ir_comparer::set_result(expr_c l, expr_c r, bool cond) { - if (same_ && !cond) { - same_ = false; - if (diff) { - diff->first_diff_expr_.first = std::move(l); - diff->first_diff_expr_.second = std::move(r); - } - } - return cond; -} - -bool ir_comparer::set_result(stmt_c l, stmt_c r, bool cond) { - if (same_ && !cond) { - same_ = false; - if (diff) { - diff->first_diff_stmt_.first = std::move(l); - diff->first_diff_stmt_.second = std::move(r); - } - } - return cond; -} - -bool ir_comparer::expr_arr_equals(const std::vector &a, - const std::vector &b) { - if (a.size() != b.size()) { - return false; - } - for (size_t i = 0; i < a.size(); i++) { - if (!a.at(i)->equals(b.at(i), *this)) { - return false; - } - } - return true; -} - -bool ir_comparer::compare(const func_c &l, const func_c &r, bool auto_reset) { - bool ret = l->equals(r, *this); - if (auto_reset) - reset(); - return ret; -} - -bool ir_comparer::compare(const expr_c &l, expr_c r, bool auto_reset) { - bool ret = l->equals(std::move(r), *this); - if (auto_reset) - reset(); - return ret; -} - -bool ir_comparer::compare(const stmt_c &l, stmt_c r, bool auto_reset) { - bool ret = l->equals(std::move(r), *this); - if (auto_reset) - reset(); - return ret; -} - -bool ir_comparer::check_or_set_expr_mapping(const expr_c &l, const expr_c &r) { - auto f = var_mapping_.find(l.get()); - if (f != var_mapping_.end()) { - return f->second == r.get(); - } - var_mapping_.insert(std::make_pair(l.get(), r.get())); - return true; -} - -void ir_comparer::set_expr_mapping(const expr_c &l, const expr_c &r) { - var_mapping_.insert(std::make_pair(l.get(), r.get())); -} - -bool ir_comparer::get_expr_mapping(const expr_c &l, const expr_c &r) { - auto f = var_mapping_.find(l.get()); - if (f != var_mapping_.end()) { - return f->second == r.get(); - } - return false; -} - -std::ostream &operator<<(std::ostream &os, ir_comparer &cmper) { - if (cmper.same_) { - os << "same"; - } else { - os << "not same: "; - if (cmper.diff) { - if (cmper.diff->first_diff_expr_.first.defined() || - cmper.diff->first_diff_expr_.second.defined()) { - os << "diff expr = " << cmper.diff->first_diff_expr_.first << " v.s. " - << cmper.diff->first_diff_expr_.second; - } else if (cmper.diff->first_diff_func_.first || - cmper.diff->first_diff_func_.second) { - os << "diff func = " << cmper.diff->first_diff_func_.first << " v.s. " - << cmper.diff->first_diff_func_.second; - } else if (cmper.diff->first_diff_stmt_.first.defined() || - cmper.diff->first_diff_stmt_.second.defined()) { - os << "diff stmt = " << cmper.diff->first_diff_stmt_.first << " v.s. " - << cmper.diff->first_diff_stmt_.second; - } - } - } - os << '\n'; - return os; -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/ir_comparer.hpp b/legacy/core/src/compiler/ir/ir_comparer.hpp deleted file mode 100644 index 1c661b712..000000000 --- a/legacy/core/src/compiler/ir/ir_comparer.hpp +++ /dev/null @@ -1,127 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_IR_COMPARER_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_IR_COMPARER_HPP - -#include "sc_stmt.hpp" -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -struct ir_comparer_diff_t { - std::pair first_diff_func_; - std::pair first_diff_expr_; - std::pair first_diff_stmt_; -}; - -class ir_comparer { - std::unordered_map var_mapping_; - -public: - std::unique_ptr diff; - bool cmp_names_; - bool cmp_callee_; - bool cmp_var_ref_; - bool cmp_commutative_; - bool same_; - - void reset(); - /** - * The IR comparison context. - * @param needs_diff true if need to return the first met different IR in - * "diff" field - * @param cmp_names if false, will ignore the names of func/var/tensor. - * i.e. nodes with different "name_" may be treated as the same - * @param cmp_var_ref if true, the comparison of var/tensor will be - * comparing vars/tensors by their pointers instead of comparing the names - * (decided by cmp_names), shapes and dtypes. Note: you can use - * set_expr_mapping() to map one var/tensor to another even if this flag is - * true. If the LHS var/tensor is in the mapping, IR comparison will check - * whether the pointer of the LHS expr is mapped in the mapping instead - * @param cmp_callee similar to cmp_var_ref. if false, will compare callee - * functions in call_nodes by their pointer instead of comparing using - * func_base::equals() - * @param cmp_commutative if the expr is commutative, try to match both - * orders. DO NOT enable this when the expr is very complex - * */ - ir_comparer(bool needs_diff = false, bool cmp_names = false, - bool cmp_var_ref = false, bool cmp_callee = false, - bool cmp_commutative = false); - bool set_result(func_c l, func_c r, bool cond); - bool set_result(expr_c l, expr_c r, bool cond); - bool set_result(stmt_c l, stmt_c r, bool cond); - bool expr_arr_equals(const std::vector &a, const std::vector &b); - - bool compare(const func_c &l, const func_c &r, bool auto_reset = true); - bool compare(const expr_c &l, expr_c r, bool auto_reset = true); - bool compare(const stmt_c &l, stmt_c r, bool auto_reset = true); - - /** - * Checks if the var/tensor definition mapping is correct. - * Will set a mapping l=>r if the mapping of l is not found - * It will find the difference of: - * var a:int - * var b:int - * a=b - * ----------------------- and: - * var a:int - * var b:int - * b=a - * */ - bool check_or_set_expr_mapping(const expr_c &l, const expr_c &r); - - /** - * Sets the expr mapping from l to r. - * l and r should be var/tensor - * The result of l->equals(r, *this) will be true - * */ - void set_expr_mapping(const expr_c &l, const expr_c &r); - - /** - * Gets the expr mapping from l to r. - * l and r should be var/tensor - * If l is mapped to r, returns true - * */ - bool get_expr_mapping(const expr_c &l, const expr_c &r); - - template bool check_equals_may_null(T a, T b) { - if (!a.defined()) { - return set_result(a, b, !b.defined()); - } - if (!b.defined()) { - return set_result(a, b, false); - } - return a->equals(b, *this); - } -}; - -std::ostream &operator<<(std::ostream &, ir_comparer &); - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/ir_module.cpp b/legacy/core/src/compiler/ir/ir_module.cpp deleted file mode 100644 index d34ed114e..000000000 --- a/legacy/core/src/compiler/ir/ir_module.cpp +++ /dev/null @@ -1,342 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "ir_module.hpp" -#include "builder.hpp" -#include "pass/func_dependency.hpp" -#include "visitor.hpp" -#include -#include -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -static std::atomic rename_cnt = {0}; -// this pass will -// break the link from call_nodes to function nodes with bodies -class func_unlinker_t : public ir_inplace_visitor_t { -public: - using ir_inplace_visitor_t::dispatch_impl; - expr visit_impl(call c) override { - auto ret = ir_inplace_visitor_t::visit_impl(std::move(c)).as(); - func_t callee = std::dynamic_pointer_cast(ret->func_); - if (!callee) { - return ret; - } - // if the callee has a function body, unlink the call_node with the - // function with body. Use the decl_ instead - if (callee->body_.defined()) { - assert(callee->decl_); - callee = callee->decl_; - return builder::remake_call(callee, ret->args_, ret); - } else { - assert(!callee->decl_); - } - // if the call is to a decl_, no changes - return ret; - } - - expr visit_impl(func_addr c) override { - auto ret = ir_inplace_visitor_t::visit_impl(std::move(c)).as(); - func_t callee = std::dynamic_pointer_cast(ret->func_); - if (!callee) { - return ret; - } - // if the callee has a function body, unlink the call_node with the - // function with body. Use the decl_ instead - if (callee->body_.defined()) { - assert(callee->decl_); - ret->func_ = callee->decl_; - return ret; - } else { - assert(!callee->decl_); - } - // if the call is to a decl_, no changes - return ret; - } - - func_t dispatch_impl(func_t f) override { - if (f->decl_) { - if (f->decl_->name_ != f->name_) { - f->decl_->name_ = f->name_; - } - } - return ir_inplace_visitor_t::dispatch_impl(std::move(f)); - } -}; - -std::shared_ptr ir_module_t::from_entry_func(context_ptr ctx, - func_t funct) { - auto ret = - from_entry_func(std::move(ctx), std::vector{std::move(funct)}); - ret->entry_func_idx_ = 0; - return ret; -} - -// resolve function dependency, returns funcs with their dependency -static std::vector resolve_func(const std::vector &funcs) { - std::vector dep; - std::unordered_set set; - func_dependency_finder_t finder(dep); - for (auto &f : funcs) { - if (set.find(f) != set.end()) { - continue; - } - dep.emplace_back(f); - set.insert(f); - size_t old_size = 0; - do { - size_t last_size = old_size; - old_size = set.size(); - // run dependency_finder on newly added functions - for (size_t i = last_size; i < dep.size(); i++) { - finder(dep[i], set); - } - assert(set.size() == dep.size()); - // if old_size == set.size(), no dependency added, can exit - } while (old_size != set.size()); - } - return dep; -} - -std::shared_ptr -ir_module_t::from_entry_func(context_ptr ctx, - const std::vector &funcs) { - auto ret = std::make_shared(std::move(ctx)); - ret->add_resolved_func(resolve_func(funcs)); - return ret; -} - -ir_module_t *ir_module_t::merge(const ir_module_t &m) { - assert(m.ctx_ == ctx_); - add_resolved_func(m.get_contents()); - for (auto &v : m.module_vars_) { - add_global_var(v); - } - for (auto &kv : m.op_table_map_) { - add_op_table(kv); - } - return this; -} - -ir_module_t *ir_module_t::merge(const std::vector &list) { - for (auto &m : list) { - merge(*m); - } - return this; -} - -func_t ir_module_t::get_func(const std::string &name) const { - auto itr = symbols_.find(name); - if (itr != symbols_.end()) { - return contents_.at(itr->second); - } - return func_t(); -} - -ir_module_ptr ir_module_t::copy() const { - return std::make_shared(*this); -} - -ir_module_ptr -ir_module_t::copy_and_remove_funcs(const std::vector &mask) const { - auto ret = std::make_shared(*this); - auto entry_func = ret->entry_func_idx_; - ret->entry_func_idx_ = -1; - ret->symbols_ = {}; - std::vector newcontents; - for (size_t old_id = 0; old_id < ret->contents_.size(); old_id++) { - if (mask.at(old_id)) { - auto new_id = newcontents.size(); - auto the_func = ret->contents_[old_id]; - newcontents.emplace_back(the_func); - ret->symbols_[the_func->name_] = new_id; - if ((int)old_id == entry_func) { - ret->entry_func_idx_ = new_id; - } - } - } - ret->contents_ = std::move(newcontents); - return ret; -} - -ir_module_ptr ir_module_t::deep_copy() const { - auto ret = copy(); - std::unordered_map replacer; - for (auto &kv : ret->var_symbols_) { - ir_copier_t cpy{replacer, true}; - kv.second = cpy(kv.second).remove_const().checked_as(); - } - for (auto &def : ret->module_vars_) { - def = ret->var_symbols_[get_node_name(def->var_)]; - } - for (auto &f : ret->contents_) { - ir_copier_t cpy{replacer, true}; - f = std::const_pointer_cast(cpy(f)); - } - for (auto &kv : ret->op_table_map_) { - kv.second = std::make_shared(*kv.second); - } - return ret; -} - -void ir_module_t::add_func(const std::vector &funcs) { - add_resolved_func(resolve_func(funcs)); -} - -var ir_module_t::make_global_var(sc_data_type_t dtype, const std::string &name, - linkage linkage, expr init) { - var ret = builder::make_var(dtype, name).static_as(); - auto def = - builder::make_var_tensor_def_unattached(ret, linkage, std::move(init)) - .static_as(); - add_global_var(std::move(def)); - return ret; -} - -tensor ir_module_t::make_global_tensor(sc_data_type_t dtype, - const std::string &name, - const std::vector &dims, - linkage linkage) { - tensor ret = builder::make_tensor(name, dims, dtype).static_as(); - auto def = - builder::make_var_tensor_def_unattached(ret, linkage).static_as(); - add_global_var(std::move(def)); - return ret; -} - -tensor ir_module_t::make_global_stensor(sc_data_type_t dtype, - const std::string &name, - const std::vector &dims, - const std::vector &strides, - linkage linkage, stmt *out_def_node) { - tensor ret = - builder::make_stensor(name, dims, strides, dtype).static_as(); - auto def = - builder::make_var_tensor_def_unattached(ret, linkage).static_as(); - if (out_def_node) { - *out_def_node = def; - } - add_global_var(std::move(def)); - return ret; -} - -void ir_module_t::add_global_var(define def) { - auto &name = def->var_.isa() ? def->var_.static_as()->name_ - : def->var_.checked_as()->name_; - auto itr = var_symbols_.find(name); - if (itr != var_symbols_.end()) { - COMPILE_ASSERT(def->linkage_ != linkage::public_global, - "Global var redefinition: " << name); - std::string new_name; - do { - new_name = name + '_' + std::to_string(++rename_cnt); - itr = var_symbols_.find(new_name); - } while (itr != var_symbols_.end()); - name = new_name; - } - var_symbols_.insert(std::make_pair(name, def)); - module_vars_.emplace_back(std::move(def)); -} - -void ir_module_t::add_resolved_func(const std::vector &funcs) { - func_unlinker_t replacer; - // all dependencies found, now check if any function name duplications - for (auto &f : funcs) { - // skip function decl_ - if (!f->body_.defined()) { - continue; - } - auto itr = symbols_.find(f->name_); - if (itr != symbols_.end()) { - // if the function is already added to the module, skip - if (contents_.at(itr->second) == f) { - continue; - } - - // try to rename the function if we are allowed to - COMPILE_ASSERT(!f->attr_ || !f->attr_->has_key("entry_func"), - "The function " - << f->name_ - << " is duplicated and is marked \'entry_func\'.") - std::string name; - do { - name = f->name_ + '_' + std::to_string(++rename_cnt); - itr = symbols_.find(name); - } while (itr != symbols_.end()); - f->name_ = name; - assert(f->decl_); - } - symbols_.insert(std::make_pair(f->name_, contents_.size())); - replacer.dispatch_impl(f); - contents_.emplace_back(f); - } -} - -void ir_module_t::add_op_table( - const std::pair &tb) { - op_table_map_.insert(tb); -} - -void ir_module_t::run_pass(function_pass_t &pass) { - for (auto &f : contents_) { - f = std::const_pointer_cast(pass(f)); - } -} - -func_t ir_module_t::make_init_func() const { - if (!module_vars_.empty()) { - stmts seq = make_stmt(std::vector()); - for (auto &v : module_vars_) { - assert(v->linkage_ == linkage::private_global || - v->linkage_ == linkage::public_global); - if (v->init_.defined()) { - seq->seq_.emplace_back( - builder::make_assign_unattached(v->var_, v->init_)); - } - } - if (seq->seq_.empty()) - return func_t(); - auto ret = builder::make_func("__sc_init__", std::vector(), - std::move(seq), datatypes::void_t); - return ret; - } - return func_t(); -} -ostream &operator<<(ostream &os, const ir_module_t &m) { - ir_printer_t p{os}; - return p.do_dispatch(m); -} -ostream &operator<<(ostream &os, const const_ir_module_ptr &m) { - return (os << *m); -} - -ostream &operator<<(ostream &os, const ir_module_ptr &m) { - return (os << const_ir_module_ptr(m)); -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/ir_module.hpp b/legacy/core/src/compiler/ir/ir_module.hpp deleted file mode 100644 index c109b94b7..000000000 --- a/legacy/core/src/compiler/ir/ir_module.hpp +++ /dev/null @@ -1,212 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_IR_MODULE_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_IR_MODULE_HPP - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -struct brg_range_handle_t; -using brg_range_handle_ptr = std::shared_ptr; -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -struct op_dispatch_tables_t; -using op_dispatch_tables_ptr = std::shared_ptr; -using dispatch_table_map_t = - std::unordered_map; - -class function_pass_t; - -class SC_INTERNAL_API ir_module_t { - // Items to appear in the module. - // Might be useful at some point to be able to include data, - // initializations, finalizations, etc. - std::vector contents_; - // the function name -> func_idx mapping - std::unordered_map symbols_; - // the "main" function of the module, <0 if no entry func_t is set - int entry_func_idx_; - // the global variables - std::vector module_vars_; - std::unordered_map var_symbols_; - // hold all op tables in ir module and pass to jit module for dynamic - // dispatch. the op function name -> op tables ptr - dispatch_table_map_t op_table_map_; - // vector of brgemm range handler. - std::vector brg_handles_; - -public: - // the attr keys for ir_module - struct attr_key_t { - // the statics_table_t for the global data buffer - static constexpr const char *MODULE_DATA_BUFFERS = "MODULE_DATA_BUFFERS"; - // float, the estimated number of FP operations in the module (GFLOPS) - static constexpr const char *GFLOP = "gflop"; - // bool, whether to use managed thread pool - static constexpr const char *MANAGED_THREAD_POOL = "MANAGED_THREAD_POOL"; - // string, the name of the module - static constexpr const char *NAME = "name"; - // bool, default=false. whether the addresses of global tensors and - // variables will be hardcoded in the JIT'd code - static constexpr const char *STATIC_GLOBALS = "static_globals"; - // vector>. The list of base tensors - // for shared consts - static constexpr const char *SHARED_CONST_BASES = "shared_const_bases"; - }; - - context_ptr ctx_; - any_map_t attr_; - ir_module_t(context_ptr ctx) : ctx_(std::move(ctx)) { entry_func_idx_ = -1; } - - // creates an ir_module_t with given functions. If entry_func_idx >=0, will - // set entry_func_ = contents[entry_func_idx]. Otherwise, will leave - // entry_func_ as empty - ir_module_t(context_ptr ctx, const std::vector &contents, - int entry_func_idx = -1) - : contents_(contents), ctx_(std::move(ctx)) { - entry_func_idx_ = entry_func_idx; - } - - int get_entry_func_idx() const { return entry_func_idx_; } - void set_entry_func_idx(int entry_func_idx) { - COMPILE_ASSERT( - entry_func_idx == -1 || - (entry_func_idx >= 0 && size_t(entry_func_idx) < contents_.size()), - "Invalid entry_func_idx"); - entry_func_idx_ = entry_func_idx; - } - - std::vector &get_contents() { return contents_; } - const std::vector &get_contents() const { return contents_; } - - const std::vector &get_module_vars() const { return module_vars_; } - std::vector &get_module_vars() { return module_vars_; } - - const dispatch_table_map_t &get_op_table_map() const { return op_table_map_; } - dispatch_table_map_t &get_op_table_map() { return op_table_map_; } - const std::vector &get_brg_range_handle_vec() const { - return brg_handles_; - } - std::vector &get_brg_range_handle_vec() { - return brg_handles_; - } - - // run the pass on all functions in this module - void run_pass(function_pass_t &pass); - - func_t make_init_func() const; - - var make_global_var(sc_data_type_t dtype, const std::string &name, - linkage linkage = linkage::private_global, - expr init = expr()); - tensor make_global_tensor(sc_data_type_t dtype, const std::string &name, - const std::vector &dims, - linkage linkage = linkage::private_global); - // make a global tensor with strides - tensor make_global_stensor(sc_data_type_t dtype, const std::string &name, - const std::vector &dims, - const std::vector &strides, - linkage linkage = linkage::private_global, - stmt *out_def_node = nullptr); - // adds a global var def, handles renaming - void add_global_var(define def); - // adds a pair of name and op table for dynamic dispatch - void add_op_table(const std::pair &tb); - // gets the entry func_t. nullable - func_t get_entry_func() const { - return entry_func_idx_ >= 0 ? contents_[entry_func_idx_] : func_t(); - } - // get var define node from input symbol, if could not find, return nullptr. - define get_var_def_from_symbol(const std::string &symbol) const { - auto it = var_symbols_.find(symbol); - if (it != var_symbols_.end()) { - return it->second; - } - return define(); - } - - // adds a list of functions to the module, resolves dependencies and handles - // name duplications - void add_func(const std::vector &f); - - // adds the functions of another module to this module. Will handle the - // renaming of the functions returns this - ir_module_t *merge(const ir_module_t &m); - - // adds the function of another module list to this module. - ir_module_t *merge(const std::vector> &list); - - // gets a function from the module. If the name is not in the symbol table, - // returns null - func_t get_func(const std::string &name) const; - - // copies the module - std::shared_ptr copy() const; - // deep copies the module - std::shared_ptr deep_copy() const; - - // copies the module and remove the specified funcs by mask. A function will - // be copied to the returned IR module only if it is get_contents()[i] and - // mask[i] is true. - std::shared_ptr - copy_and_remove_funcs(const std::vector &mask) const; - - /** - * Creates an IR module from a list of functions. Finds the direct and - * indirect dependent functions of the given functions. Also rename the - * functions with duplicated names to "XXX_1", "XXX_1_1", etc. - * @param ctx the context - * @param f the list of functions - * @return the `ir_module_t` containing the given list of functions and all - * dependent functions. The names of the functions in the module are unique - * */ - static std::shared_ptr - from_entry_func(context_ptr ctx, const std::vector &f); - - /** - * @see from_entry_func overloaded function - * */ - static std::shared_ptr from_entry_func(context_ptr ctx, - func_t f); - -private: - // adds a list of functions to the module, dependency already resolved - void add_resolved_func(const std::vector &f); -}; - -using ir_module_ptr = std::shared_ptr; -using const_ir_module_ptr = std::shared_ptr; - -extern ostream &operator<<(ostream &os, const const_ir_module_ptr &); -SC_API extern ostream &operator<<(ostream &os, const ir_module_ptr &); -SC_API extern ostream &operator<<(ostream &os, const ir_module_t &); - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl -#endif diff --git a/legacy/core/src/compiler/ir/ir_node_names.hpp b/legacy/core/src/compiler/ir/ir_node_names.hpp deleted file mode 100644 index 02dd2d28c..000000000 --- a/legacy/core/src/compiler/ir/ir_node_names.hpp +++ /dev/null @@ -1,63 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_IR_NODE_NAMES_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_IR_NODE_NAMES_HPP - -#define SC_EXPAND(x) x - -#define FOR_EACH_EXPR_IR_TYPE(F, ...) \ - SC_EXPAND(F(constant, __VA_ARGS__)) \ - SC_EXPAND(F(var, __VA_ARGS__)) \ - SC_EXPAND(F(cast, __VA_ARGS__)) \ - SC_EXPAND(F(add, __VA_ARGS__)) \ - SC_EXPAND(F(sub, __VA_ARGS__)) \ - SC_EXPAND(F(mul, __VA_ARGS__)) \ - SC_EXPAND(F(div, __VA_ARGS__)) \ - SC_EXPAND(F(mod, __VA_ARGS__)) \ - SC_EXPAND(F(cmp_eq, __VA_ARGS__)) \ - SC_EXPAND(F(cmp_ne, __VA_ARGS__)) \ - SC_EXPAND(F(cmp_lt, __VA_ARGS__)) \ - SC_EXPAND(F(cmp_le, __VA_ARGS__)) \ - SC_EXPAND(F(cmp_gt, __VA_ARGS__)) \ - SC_EXPAND(F(cmp_ge, __VA_ARGS__)) \ - SC_EXPAND(F(logic_and, __VA_ARGS__)) \ - SC_EXPAND(F(logic_or, __VA_ARGS__)) \ - SC_EXPAND(F(logic_not, __VA_ARGS__)) \ - SC_EXPAND(F(select, __VA_ARGS__)) \ - SC_EXPAND(F(indexing, __VA_ARGS__)) \ - SC_EXPAND(F(call, __VA_ARGS__)) \ - SC_EXPAND(F(tensor, __VA_ARGS__)) \ - SC_EXPAND(F(tensorptr, __VA_ARGS__)) \ - SC_EXPAND(F(intrin_call, __VA_ARGS__)) \ - SC_EXPAND(F(ssa_phi, __VA_ARGS__)) \ - SC_EXPAND(F(func_addr, __VA_ARGS__)) \ - SC_EXPAND(F(low_level_intrin, __VA_ARGS__)) - -#define FOR_EACH_BASE_EXPR_IR_TYPE(F, ...) \ - SC_EXPAND(F(binary, __VA_ARGS__)) \ - SC_EXPAND(F(logic, __VA_ARGS__)) \ - SC_EXPAND(F(cmp, __VA_ARGS__)) - -#define FOR_EACH_STMT_IR_TYPE(F, ...) \ - SC_EXPAND(F(assign, __VA_ARGS__)) \ - SC_EXPAND(F(stmts, __VA_ARGS__)) \ - SC_EXPAND(F(if_else, __VA_ARGS__)) \ - SC_EXPAND(F(evaluate, __VA_ARGS__)) \ - SC_EXPAND(F(for_loop, __VA_ARGS__)) \ - SC_EXPAND(F(returns, __VA_ARGS__)) \ - SC_EXPAND(F(define, __VA_ARGS__)) - -#endif diff --git a/legacy/core/src/compiler/ir/ir_utils.cpp b/legacy/core/src/compiler/ir/ir_utils.cpp deleted file mode 100644 index fd0d838de..000000000 --- a/legacy/core/src/compiler/ir/ir_utils.cpp +++ /dev/null @@ -1,148 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#include "ir_utils.hpp" -#include "sc_expr.hpp" -#include "sc_stmt.hpp" -#include "ssa_data.hpp" -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -void get_direct_dependency_of_expr( - const expr &v, const std::function)> &callback) { - switch (v->node_type_) { - case sc_expr_type::undef: - assert(0 && "Unreachable"); - break; - case sc_expr_type::constant: - case sc_expr_type::func_addr: - break; - case sc_expr_type::tensor: - case sc_expr_type::var: { - if (utils::is_uninitialized_weakptr(v->ssa_data_->owner_)) { - // a var/tensor can be attached none of the define_node, which - // means it is a parameter - break; - } - stmt owner{v->ssa_data_->owner_.lock()}; - assert(owner.defined()); - auto init = owner.static_as()->init_; - if (owner.isa() && init.defined()) { - if (init.isa() || init.isa()) { - callback({init}); - } else { - get_direct_dependency_of_expr(init, callback); - } - } - } break; - case sc_expr_type::cast: - callback({v.static_as()->in_}); - break; - case sc_expr_type::add: - case sc_expr_type::sub: - case sc_expr_type::mul: - case sc_expr_type::div: - case sc_expr_type::mod: { - auto val = v.static_as(); - callback({val->l_, val->r_}); - } break; - case sc_expr_type::cmp_eq: - case sc_expr_type::cmp_ne: - case sc_expr_type::cmp_lt: - case sc_expr_type::cmp_le: - case sc_expr_type::cmp_gt: - case sc_expr_type::cmp_ge: { - auto val = v.static_as(); - callback({val->l_, val->r_}); - } break; - case sc_expr_type::logic_and: - case sc_expr_type::logic_or: { - auto val = v.static_as(); - callback({val->l_, val->r_}); - } break; - case sc_expr_type::logic_not: - callback(&v.static_as()->in_); - break; - case sc_expr_type::select: { - auto val = v.static_as()) { - auto node = ret.static_as(); - const auto &cond = node->cond_; - const auto &left = node->l_; - const auto &right = node->r_; - const auto &cond_vals = cond.dyn_as()->value_; - auto count = left->dtype_.lanes_; - if (left.isa() && right.isa() && - (cond.isa())) { - const auto &right_vals = right.dyn_as()->value_; - const auto &left_vals = left.dyn_as()->value_; - std::vector vals(count); - auto value_size_select = [](const std::vector &vls, - size_t itr) { - return vls.size() > 1 ? vls[itr] : vls[0]; - }; - auto cond_select = [](bool cond, const union_val &left, - const union_val &right) { - return cond ? left : right; - }; - for (size_t itr = 0; itr < count; itr++) { - // Our IR is usually written in the form of - // make_expr(1.f,f32(8)), and its - // constant value size will be 1. - auto cond_value = cond_vals[0].u64; - bool boolean_cond = value_size_select(cond_vals, itr).u64 > 0UL; - bool uint_cond = ((1ULL << itr) & cond_value); - union_val sel_val; - if (cond->dtype_.type_code_ == sc_data_etype::BOOLEAN) { - sel_val = - cond_select(boolean_cond, value_size_select(left_vals, itr), - value_size_select(right_vals, itr)); - } else { - sel_val = cond_select(uint_cond, value_size_select(left_vals, itr), - value_size_select(right_vals, itr)); - } - vals[itr] = sel_val; - } - return builder::make_constant(vals, ret->dtype_); - } else if (cond.isa() && - cond->dtype_.type_code_ != sc_data_etype::BOOLEAN) { - assert(left->dtype_.lanes_ <= utils::get_sizeof_type(cond->dtype_) * 8); - bool check_true = true; - while (count-- > 0) { - check_true &= (cond_vals[0].u64 >> count) & 1; - } - bool check_false = cond_vals[0].u64 == 0UL; - if (check_true) { - return node->l_; - } - if (check_false) { - return node->r_; - } - return ret; - } else if (cond->dtype_.type_code_ == sc_data_etype::BOOLEAN && - cond.isa()) { - bool check_true = std::all_of(cond_vals.begin(), cond_vals.end(), - [](union_val x) { return x.u64 > 0UL; }); - bool check_false = - std::all_of(cond_vals.begin(), cond_vals.end(), - [](union_val x) { return x.u64 == 0UL; }); - if (check_true) { - return node->l_; - } - if (check_false) { - return node->r_; - } - return ret; - } else if (node->l_.ptr_same(node->r_) || - (node->l_.isa() && node->l_->equals(node->r_))) { - return node->l_; - } - } - return ret; - } - - stmt_c visit(define_c v) override { - auto ret = ir_consistent_visitor_t::visit(v).checked_as(); - if (fast_) { - return ret; - } - if (v->var_.isa() && v->var_->dtype_.lanes_ == 1 && - ret->init_.defined() && !ret.ptr_same(v) && - ret->init_.isa()) { - var_rvalue_changed_ = true; - } - return ret; - } - - stmt_c visit(assign_c v) override { - auto ret = ir_consistent_visitor_t::visit(v).checked_as(); - if (fast_) { - return ret; - } - if (v->var_.isa() && v->var_->dtype_.lanes_ == 1 && - !v->value_.ptr_same(ret->value_) && ret->value_.isa()) { - var_rvalue_changed_ = true; - } - return ret; - } - - stmt_c visit(for_loop_c v) override { - auto old_fast = fast_; - fast_ |= - (v->attr_ && v->attr_->get_or_else("bypass_complex_const_fold", false)); - // don't fold range for the var - auto var = dispatch(v->var_); - auto begin = fold_range_dispatch(v->iter_begin_); - auto end = fold_range_dispatch(v->iter_end_); - auto step = fold_range_dispatch(v->step_); - - mark_range_for_const(run_idx_, begin, fast_); - mark_range_for_const(run_idx_, end, fast_); - int64_t loop_len_hint = -1; - if (!fast_) { - if (step.isa()) { - auto stepc = get_const_as_int(step.static_as()); - auto begin_r = get_range_of_expr(run_idx_, begin, fast_); - auto end_r = get_range_of_expr(run_idx_, end, fast_); - if (stepc > 0 && begin_r && end_r) { - int64_t max_loop_len = end_r->end.s64 - begin_r->start.s64; - int64_t real_loop_len = max_loop_len / stepc * stepc; - if (max_loop_len > 0 && real_loop_len > 0) { - get_analysis_for_edit(var.get(), run_idx_) - ->set_range(const_range_t{ - get_type_category(var->dtype_), begin_r->start, - begin_r->start.s64 + (real_loop_len - 1) * stepc}); - } - } - } - if (v->attr_) { - loop_len_hint = v->attr_->get_or_else("loop_len_hint", INT64_C(-1)); - } - // try to fold the for range like for(i = A to A+1 step 1) {} - if (loop_len_hint == -1) { - if (!begin.isa() && !end.isa()) { - expr_c real_begin = - get_assigned_expr(run_idx_, begin.get())->node_ptr_from_this(); - expr_c real_end = - get_assigned_expr(run_idx_, end.get())->node_ptr_from_this(); - auto ths = this; - auto try_fold = [ths, &loop_len_hint](const expr_c &beg_v, - const expr_c &end_v, - const expr_c &step_v) -> bool { - auto loop_len = ths->fold_range_dispatch((end_v - beg_v) / step_v); - if (loop_len.isa()) { - loop_len_hint = get_expr_as_int(loop_len); - return true; - } - return false; - }; - if (try_fold(real_begin, real_end, step)) { - // fall through - } else if (!real_end.ptr_same(end) && - try_fold(real_begin, end, step)) { - // fall through - } else if (!real_begin.ptr_same(begin) && - try_fold(begin, real_end, step)) { - // fall through - } - } - } - } - loop_depth_++; - if (!old_fast) { - get_analysis_for_edit(var.get(), run_idx_)->loop_depth = loop_depth_; - } - auto body = dispatch(v->body_); - loop_depth_--; - - bool changed = !(var.ptr_same(v->var_) && begin.ptr_same(v->iter_begin_) && - end.ptr_same(v->iter_end_) && step.ptr_same(v->step_) && - body.ptr_same(v->body_)); - stmt ret; - if (changed) { - ret = copy_attr(*v, builder::make_for_loop_unattached( - var, begin, end, step, body, v->incremental_, - v->kind_, v->num_threads_)); - } else { - ret = std::move(v).remove_const(); - } - if (loop_len_hint >= 0) { - ret->attr()["loop_len_hint"] = loop_len_hint; - } - fast_ = old_fast; - return ret; - } - - stmt_c visit(if_else_c v) override { - auto cond = fold_range_dispatch(v->condition_); - auto thencase = dispatch(v->then_case_); - - stmt_c elsecase; - if (v->else_case_.defined()) - elsecase = dispatch(v->else_case_); - bool changed = !cond.ptr_same(v->condition_) || - !elsecase.ptr_same(v->else_case_) || - !thencase.ptr_same(v->then_case_); - if (cond.isa()) { - assert(!cond.as()->is_vector()); - COMPILE_ASSERT( - cond->dtype_ == datatypes::boolean, - "IfElse node expects an boolean expr as the condition, got " - << cond->dtype_ << " expr = " << v); - bool val = cond.as()->value_[0].u64; - if (val) { - return thencase; - } else { - if (v->else_case_.defined()) { - return elsecase; - } - return make_stmt(std::vector()); - } - } - if (changed) { - return copy_attr( - *v, builder::make_if_else_unattached(cond, thencase, elsecase)); - } - return v; - } - - func_c dispatch(func_c v) override { - if (!fast_) { - func_c cur_f = v; - func_c ret; - for (;;) { - var_rvalue_changed_ = false; - constant_fold_analysis_t ana{run_idx_}; - ana.dispatch(cur_f); - // keep the exprs alive to make sure the raw pointers in - // analysis result is valid - auto keep_alive = std::move(ana.single_assign_); - ret = ir_visitor_t::dispatch(cur_f); - if (!var_rvalue_changed_ || ret == cur_f) { - for (auto &kv : keep_alive) { - if (kv.first->temp_data_) { - kv.first->temp_data_->clear(); - } - } - return ret; - } - cur_f = ret; - sub_run_idx_++; - } - } - return ir_visitor_t::dispatch(v); - } -}; - -func_c constant_folder_t::operator()(func_c f) const { - constant_fold_t pass{fast_}; - return pass.dispatch(std::move(f)); -} - -stmt_c constant_folder_t::operator()(stmt_c f) const { - constant_fold_t pass{fast_}; - return pass.dispatch(std::move(f)); -} - -expr_c constant_folder_t::operator()(expr_c f) const { - constant_fold_t pass{fast_}; - return pass.dispatch(std::move(f)); -} - -/** - * this feature is currently used to fold the index of reshape/reorder output, - * so additional folding is added before and after expand_polynomial. TODO: move - * this feature to constant folding pass - * @param f: original polynomial expr. - * @param max_iter: maximum iteration time, default is one. - * @param skip_mod: skip int mod expand. - * */ -expr_c constant_folder_t::expand_polynomial(expr_c f, int max_iter, - bool skip_mod) { - constant_fold_t pass{true, skip_mod}; - auto ret = pass.dispatch(std::move(f)); - for (int i = 0; i < max_iter; i++) { - auto old = ret; - ret = pass.expand_polynomial(old); - if (ret.ptr_same(old)) { - break; - } - } - constant_fold_t pass2{true, skip_mod}; - return pass2.dispatch(ret); -} - -const_ir_module_ptr constant_folder_t::operator()(const_ir_module_ptr f) { - constant_fold_t pass{fast_}; - return dispatch_module_on_visitor(&pass, f); -} - -expr do_cast_and_fold(const expr &in) { - static auto_caster_t caster; - constant_folder_t folder{true}; - return folder(caster(in)).remove_const(); -} - -expr_c do_cast_and_fold(const expr_c &in) { - return do_cast_and_fold(in.remove_const()); -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/transform/constant_fold.hpp b/legacy/core/src/compiler/ir/transform/constant_fold.hpp deleted file mode 100644 index daa45251d..000000000 --- a/legacy/core/src/compiler/ir/transform/constant_fold.hpp +++ /dev/null @@ -1,73 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_CONSTANT_FOLD_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_CONSTANT_FOLD_HPP - -#include "../module_pass.hpp" -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -namespace constant_folding { -std::pair get_operand_from_binary(const expr_c &a); -bool is_op_commutative_and_associative(const expr_c &v); -} // namespace constant_folding - -/** - * Fold the constants. - * Supported nodes: - * binary, cmp, logic, logic_not, cast, if_else - * - * It will do the following (c as constant, "+" as an example): - * c1 + c2 => c3 - * c + x => x + c - * (x + c1) + c2 => x + (c1 + c2) - * (x + c) + y => (x + y) + c - * x + (y + c) => (x + y) + c - * (x + c1) + (y + c2) => (x + y) + (c1 + c2) - * - * Also fold special expr: - * a (+ - * && ||) 0/false - * a (* / % && ||) 1/true - * a (- / % && || max min > >= < <= == !=) a - * - * @param fast whether check if varaibles are actually constants. If true, will - * skip this check and keep the variables - * */ -class constant_folder_t : public module_pass_t { -public: - bool fast_; - constant_folder_t(bool fast = true) : fast_(fast) {} - func_c operator()(func_c f) const; - stmt_c operator()(stmt_c f) const; - expr_c operator()(expr_c f) const; - const_ir_module_ptr operator()(const_ir_module_ptr f) override; - expr_c expand_polynomial(expr_c f, int max_iter = 1, bool skip_mod = false); - SC_DECL_PASS_INFO_FUNC(); -}; -// do auto cast and constant fold for input expr. -expr do_cast_and_fold(const expr &in); -expr_c do_cast_and_fold(const expr_c &in); -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/transform/cpu/closurize.cpp b/legacy/core/src/compiler/ir/transform/cpu/closurize.cpp deleted file mode 100644 index 493daccb7..000000000 --- a/legacy/core/src/compiler/ir/transform/cpu/closurize.cpp +++ /dev/null @@ -1,520 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#include "closurize.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -SC_MODULE(pass.closurize) -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -SC_DECL_PASS_INFO(closurizer_cpu, - SC_PASS_DEPENDS_ON(nested_parallel_flattener, - parallel_workload_dispatcher, validator, - trace_inserter, parallel_merge, - tensor_init), - SC_PASS_REQUIRE_STATE(), SC_PASS_REQUIRE_NOT_STATE(), - SC_PASS_SET_STATE(), SC_PASS_UNSET_STATE()); - -func_t get_parallel_call_with_env_func(bool managed) { - static func_t f = builder::_decl_func( - "sc_parallel_call_cpu_with_env", datatypes::void_t, - {_arg_("func", datatypes::pointer), _arg_("flags", datatypes::index), - _arg_("stream", datatypes::pointer), - _arg_("env", datatypes::s8.get_pointerof()), - _arg_("begin", datatypes::index), _arg_("end", datatypes::index), - _arg_("step", datatypes::index), - _arg_("args", datatypes::generic.get_pointerof())}); - static func_t f_managed = builder::_decl_func( - "sc_parallel_call_managed", datatypes::void_t, - {_arg_("func", datatypes::pointer), _arg_("flags", datatypes::index), - _arg_("stream", datatypes::pointer), - _arg_("env", datatypes::s8.get_pointerof()), - _arg_("begin", datatypes::index), _arg_("end", datatypes::index), - _arg_("step", datatypes::index), - _arg_("args", datatypes::generic.get_pointerof())}); - return managed ? f_managed : f; -} - -// the sub-pass to collect the tensor definitions in a function. It is used in -// last-barrier safe removal to find the captured base tensors -class tensor_def_collector_t : public ir_viewer_t { -public: - using ir_viewer_t::dispatch; - struct tensor_def_info_t { - expr_c init_; - bool is_arg_; - }; - using result_t = std::unordered_map; - static expr_c find_base_tensor(const expr_c &v, - const tensor_def_collector_t::result_t &scope, - bool &out_is_arg) { - expr_c cur = v; - for (;;) { - auto base = utils::find_map_value(scope, cur).get_or_else(nullptr); - // global tensor - if (!base) { - return cur; - } - if (base->is_arg_) { - // it is arg tensor - out_is_arg = true; - return v; - } - if (!base->init_.defined()) { - // itself is a base tensor - out_is_arg = false; - return cur; - } - if (!base->init_.isa()) { - // base is not a tensor, failed - out_is_arg = false; - return expr_c(); - } - cur = base->init_; - } - } - - result_t result_; - - expr_c dispatch(expr_c v) override { return v; } - func_c dispatch(func_c v) override { - for (auto &arg : v->params_) { - if (arg.isa()) { - result_[arg] = tensor_def_info_t{expr_c(), true}; - } - } - return ir_viewer_t::dispatch(v); - } - - void view(define_c v) override { - if (v->var_.isa()) { - expr_c base; - if (v->init_.defined()) { - auto tsr = get_base_tensor_of(v->init_); - if (tsr.defined()) { - base = tsr; - } else { - base = v->init_; - } - } - result_[v->var_] = tensor_def_info_t{base, false}; - } - } - - // link the args of a function into the current function's tensor info. The - // call should happen in the current func body - void link_call_args(const call &thecall, const func_t &callee) { - for (size_t i = 0; i < callee->params_.size(); i++) { - auto ¶m = callee->params_[i]; - auto &arg = thecall->args_[i]; - if (param.isa()) { - result_[param] = tensor_def_info_t{get_base_tensor_of(arg), false}; - } - } - } -}; - -namespace parallel_call_args { -enum { ADDR = 0, FLAGS, STREAM, MODULE_DATA, BEGIN, END, STEP, ARGS }; -} - -class closurize_cpu_impl_t : public closurize_impl_t { - int rename_counter_ = 0; - bool use_managed_thread_pool_; - std::vector out_calls; - func_t the_last_op_; - // we use pointers instead of reference to bypass a g++4.8 bug. If we pass - // result_t by reference, the address will be wrong in g++4.8 - const tensor_def_collector_t::result_t *main_tensor_def_info_; - const tensor_def_collector_t::result_t *lastop_tensor_def_info_; - bool has_idle_func_ = false; - // makes the closure function and its generic wrapper - func_t make_closure_func( - const std::string &name, std::vector &¶ms, stmt_c body, - const std::vector ¶_attr) override { - bool need_trace = runtime_config_t::get().trace_mode_ == - runtime_config_t::trace_mode_t::MULTI_THREAD; - COMPILE_ASSERT(para_attr.size() == 1, - "CPU does not support grouped parallel"); - func_t closure = builder::make_func(name, params, body, datatypes::void_t); - // make the wrapper function - stmts seq = make_stmt(std::vector()); - int func_id = -1; - if (need_trace) { - func_id = register_traced_func(name); - seq->seq_.insert(seq->seq_.begin(), - builder::make_evaluate_unattached( - builtin::make_trace(func_id, 0, 0))); - } - expr itervar = builder::make_var(datatypes::index, "i"); - expr args = builder::make_tensor("args", {(int)params.size() - 1}, - datatypes::generic); - func_t ret = - builder::make_func(name + "_0wrapper", std::vector{itervar, args}, - seq, datatypes::void_t); - assert(!modu_->get_func(ret->name_)); - assert(!modu_->get_func(closure->name_)); - - closure->attr()[function_attrs::private_] = true; - ret->attr()[function_attrs::private_] = true; - ret->decl_->attr() = ret->attr(); - closure->decl_->attr() = closure->attr(); - std::vector fargs; - fargs.reserve(params.size()); - // params[0] is the itervar - fargs.emplace_back(itervar); - for (uint64_t idx = 1; idx < params.size(); idx++) { - auto ¶m = closure->params_[idx]; - assert(param->dtype_.lanes_ == 1); - if (param->dtype_ == datatypes::generic) { - fargs.emplace_back(args[idx - 1]); - } else { - fargs.emplace_back(builder::make_cast(param->dtype_, args[idx - 1])); - } - } - seq->seq_.emplace_back(builder::make_evaluate_unattached( - builder::make_call(closure->decl_, fargs))); - if (need_trace) { - seq->seq_.emplace_back(builder::make_evaluate_unattached( - builtin::make_trace(func_id, 1, 0))); - } - modu_->add_func({closure, ret}); - return ret; - } - stmt make_parallel_call( - func_t target, std::vector &captures, - std::vector &¶_attr) override { - // we now need to prepare the arguments. - // 1. Allcate a generic value array - // 2. For each captured variable except the first one, put them - // in the array. The first capture is the itervar, we should - // ignore it - stmts seq = make_stmt(std::vector()); - std::string argname = "__tempargs"; - argname += std::to_string(rename_counter_++); - auto argsbuf = builder::make_tensor(argname, {captures.size() - 1}, - datatypes::generic); - seq->seq_.emplace_back(builder::make_var_tensor_def_unattached(argsbuf)); - - for (uint64_t argidx = 1; argidx < captures.size(); argidx++) { - // push "argsbuf[i] = (generic)captures_i" - seq->seq_.emplace_back(builder::make_assign_unattached( - argsbuf[argidx - 1], - builder::make_cast(datatypes::generic, captures[argidx]))); - } - assert(para_attr.size() == 1); - constant_folder_t folder{}; - expr_c begin_v = para_attr[0].begin_; - cast_to(begin_v, datatypes::index, begin_v); - begin_v = folder(begin_v); - - expr_c end_v = para_attr[0].end_; - cast_to(end_v, datatypes::index, end_v); - end_v = folder(end_v); - - expr_c step_v = para_attr[0].step_; - cast_to(step_v, datatypes::index, step_v); - step_v = folder(step_v); - uint64_t flag = has_idle_func_ - ? runtime::thread_pool_flags::THREAD_POOL_RUN_IDLE_FUNC - : 0; - auto ret_call = builder::make_call( - get_parallel_call_with_env_func(use_managed_thread_pool_), - std::vector{ - builder::make_func_addr(std::move(target)), - /*flags*/ make_expr(flag), - /*stream*/ - make_expr(UINT64_C(0), datatypes::pointer), - /*env*/ - make_expr(UINT64_C(0), - datatypes::s8.get_pointerof()), - begin_v, end_v, step_v, argsbuf}); - ret_call->temp_data() = captures; - out_calls.emplace_back(ret_call.static_as()); - seq->seq_.emplace_back(make_stmt(std::move(ret_call))); - has_idle_func_ = false; - return seq; - } - -public: - using closurize_impl_t::dispatch; - using closurize_impl_t::visit; - - expr_c visit(call_c v) override { - if (v->func_ == builtin::get_set_idle_func_managed_func() && - !in_parallel_for) { - has_idle_func_ = true; - } - return closurize_impl_t::visit(std::move(v)); - } - - uint64_t &get_last_parallel_call_flag() { - return out_calls.back() - ->args_.at(parallel_call_args::FLAGS) - .checked_as() - ->value_.at(0) - .u64; - } - - optional> - collect_captured_base_tensors(const std::vector &captured) { - std::vector ret; - for (auto &v : captured) { - if (!v.isa()) { - if (v.isa() && v->dtype_.is_etype_pointer()) { - return none_opt(); - } - continue; - } - if (v.static_as()->elem_dtype_.is_pointer()) { - return none_opt(); - } - bool is_arg = false; - expr_c base = tensor_def_collector_t::find_base_tensor( - v, *lastop_tensor_def_info_, is_arg); - if (!base.defined()) { - // failed to find the captured base tensor - return none_opt(); - } - if (is_arg) { - base = tensor_def_collector_t::find_base_tensor( - base, *main_tensor_def_info_, is_arg); - if (!base.defined()) { - // failed to find the captured base tensor - return none_opt(); - } - } - ret.emplace_back(base); - } - return ret; - } - - func_c dispatch(func_c f) override { - out_calls.clear(); - has_idle_func_ = false; - auto ret = closurize_impl_t::dispatch(f); - if (!out_calls.empty() && f->attr_ && - f->attr_->get_or_else(function_attrs::has_idle_func, false)) { - auto &the_flag = get_last_parallel_call_flag(); - the_flag |= runtime::thread_pool_flags::THREAD_POOL_RUN_IDLE_FUNC; - the_flag |= runtime::thread_pool_flags::THREAD_POOL_DISABLE_ROLLING; - } - if (f == the_last_op_ && !out_calls.empty()) { - // try to remove the last barrier - auto &seq = f->body_.checked_as()->seq_; - for (auto itr = seq.rbegin(); itr != seq.rend(); ++itr) { - auto &s = *itr; - // allow return const after the last parallel call - if (s.cast() - .filter( - [](const returns &v) { return v->value_.isa(); }) - .has_value()) { - continue; - } - // if the statement is a parallel-for - if (s.cast() - .filter([](const for_loop &v) { - return v->kind_ == for_type::PARALLEL; - }) - .has_value()) { - // we need to pin the captured tensors to runtime stack, we - // first collect the base tensors - auto captured_base = collect_captured_base_tensors( - out_calls.back()->get_temp_data().get>()); - // if the captured tensors are based on complex - // tensors/pointers, we can't optimize it - if (!captured_base.has_value()) { - SC_MODULE_WARN << "Cannot optimize the last barrier in " - "function " - << f->name_ - << " because it captures complex pointers."; - break; - } - for (auto &base : captured_base.get()) { - base.remove_const()->attr()[attr_keys::runtime_stack_alloc] = true; - } - auto &the_flag = get_last_parallel_call_flag(); - the_flag |= runtime::thread_pool_flags::THREAD_POOL_EXIT; - // the closure args buffer must be allocated via runtime - // allocator instead of the native heap. So that when the - // main thread exits from the kernel and waiting for the - // worker threads, the args are still valid. - auto closure_arg = - out_calls.back()->args_.at(parallel_call_args::ARGS).as(); - COMPILE_ASSERT(closure_arg.defined(), "Bad closure arg"); - closure_arg->attr()[attr_keys::runtime_stack_alloc] = true; - break; - } - - // else, there are some complex statements after the last - // parallel-for, do not optimize - SC_MODULE_WARN << "Cannot optimize the last barrier in function " - << f->name_ - << " because there are complex code after the last " - "parallel-for."; - break; - } - } - return ret; - } - closurize_cpu_impl_t( - const ir_module_ptr &m, bool use_managed_thread_pool, - const func_t &the_last_op, - const tensor_def_collector_t::result_t &main_tensor_def_info, - const tensor_def_collector_t::result_t &lastop_tensor_def_info) - : closurize_impl_t(m->get_module_vars(), m), - use_managed_thread_pool_(use_managed_thread_pool), - the_last_op_{the_last_op}, main_tensor_def_info_{&main_tensor_def_info}, - lastop_tensor_def_info_{&lastop_tensor_def_info} {} -}; - -class single_core_remove_parallel_t : public ir_visitor_t { -public: - using ir_visitor_t::dispatch; - expr_c dispatch(expr_c v) override { return v; } - stmt_c visit(for_loop_c v) override { - auto ret = ir_visitor_t::visit(v).checked_as(); - if (ret->kind_ == for_type::PARALLEL) { - if (ret.get() != v.get()) { - // the for loop is remade, just change it - ret->kind_ = for_type::NORMAL; - ret->attr()[attr_keys::buf_sched_top_scope] = true; - } else { - // remake a new IR node - auto retnode = copy_attr(*v, builder::make_for_loop_unattached( - ret->var_, ret->iter_begin_, - ret->iter_end_, ret->step_, ret->body_, - ret->incremental_, for_type::NORMAL)); - retnode->attr()[attr_keys::buf_sched_top_scope] = true; - return retnode; - } - } - return ret; - } -}; - -func_c remove_parallel_on_func(const func_c &f) { - single_core_remove_parallel_t pass; - return pass.dispatch(f); -} - -const_ir_module_ptr closurizer_cpu_t::operator()(const_ir_module_ptr inmod) { - float gflop = inmod->attr_.get_or_else(ir_module_t::attr_key_t::GFLOP, 0.0f); - auto tp_mod = inmod->attr_.get_or_else( - ir_module_t::attr_key_t::MANAGED_THREAD_POOL, thread_pool_mode_t::DIRECT); - bool use_managed_thread_pool = tp_mod == thread_pool_mode_t::MANAGED; - - SC_MODULE_INFO << "Use managed thread pool? " << use_managed_thread_pool - << ". Module gflops = " << gflop; - - func_t the_last_op; - tensor_def_collector_t::result_t main_tensor_def_info; - tensor_def_collector_t::result_t lastop_tensor_def_info; - if (!single_core_ && use_managed_thread_pool) { - // find the last op in main entry - auto main_entry_func = inmod->get_entry_func(); - if (main_entry_func && main_entry_func->attr_ && - main_entry_func->attr_->get_or_else(function_attrs::is_main, false)) { - auto &seq = main_entry_func->body_.checked_as()->seq_; - for (auto itr = seq.rbegin(); itr != seq.rend(); ++itr) { - auto &s = *itr; - // allow return const after the last op call - if (s.cast() - .filter( - [](const returns &v) { return v->value_.isa(); }) - .has_value()) { - continue; - } - auto f = - s.cast() - .map([](const evaluate &v) { return v->value_.as(); }) - .map([](const call &v) { - return std::dynamic_pointer_cast(v->func_); - }); - if (f.has_value()) { - the_last_op = inmod->get_func(f.get()->name_); - if (the_last_op) { - assert(the_last_op->name_ == f.get()->name_); - tensor_def_collector_t collector; - collector.dispatch(main_entry_func); - auto last_call = s.static_as()->value_.static_as(); - collector.link_call_args(last_call, the_last_op); - main_tensor_def_info = std::move(collector.result_); - - tensor_def_collector_t collector2; - collector2.dispatch(the_last_op); - lastop_tensor_def_info = std::move(collector2.result_); - } - break; - } - // else, there are some complex statements after the last - // parallel-for, do not optimize - SC_MODULE_WARN << "Cannot optimize the last barrier in main function " - << main_entry_func->name_ - << " because there are complex code after the last " - "parallel-for."; - break; - } - } - } - - auto ret = inmod->copy(); - ir_visitor_t *the_pass; - closurize_cpu_impl_t pass(ret, use_managed_thread_pool, the_last_op, - main_tensor_def_info, lastop_tensor_def_info); - single_core_remove_parallel_t singlepass{}; - if (single_core_) { - the_pass = &singlepass; - } else { - the_pass = &pass; - } - auto &funcs = ret->get_contents(); - auto sz = funcs.size(); - for (unsigned i = 0; i < sz; i++) { - auto f = std::const_pointer_cast(the_pass->dispatch(funcs[i])); - funcs[i] = std::move(f); - } - - ret->attr_[ir_module_t::attr_key_t::MANAGED_THREAD_POOL] = tp_mod; - return ret; -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/transform/cpu/closurize.hpp b/legacy/core/src/compiler/ir/transform/cpu/closurize.hpp deleted file mode 100644 index 53bc866f0..000000000 --- a/legacy/core/src/compiler/ir/transform/cpu/closurize.hpp +++ /dev/null @@ -1,73 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_CPU_CLOSURIZE_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_CPU_CLOSURIZE_HPP - -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -/** - * Base class for closurizer. Different targets should extend the - * class to make target-specific parallel functions and parallel calls. Replaces - * parallel for nodes with call_parallel nodes. Also moves the bodies of the - * parallel for nodes to new closure functions. The closure functions have the - * interface like `void closure_name(uint64_t i, T1 capture1, T2 capture2, - * ...);`, where `i` is the loop variable, and captureX is the captured - * variables that may be used in the body of the original for-loop. - * - * The target specific subclass should at least override `make_closure_func` and - * `make_parallel_call`. See comments below. - * - * */ - -/** - * Replaces parallel for nodes with call_parallel nodes for CPU - * @see closurize_impl_t - * This pass will also generate a wrapper function for each closure on CPU - * backend, which has signature `void closure_name_wrapper(uint64_t i, - * generic_val* args);`. The wrapper will extract the arguments from `args` and - * call the closure. - * - * the for-loop will finally replaced by: - * { - * tensor argbuf: [generic * X] - * argbuf[0] = capture_0; - * ... - * argbuf[X-1] = capture_X_1; - * parallel_call(closure_wrapper_func, argbuf, parallel_attr = {...}) - * } - * The backend should lower parallel_call into a call to `sc_parallel_call_cpu` - * */ -class closurizer_cpu_t : public module_pass_t { -public: - bool single_core_; - closurizer_cpu_t(bool single_core) : single_core_(single_core) {} - const_ir_module_ptr operator()(const_ir_module_ptr m) override; - SC_DECL_PASS_INFO_FUNC(); -}; - -func_t get_parallel_call_with_env_func(bool managed); -func_c remove_parallel_on_func(const func_c &f); -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/transform/cpu/kernel_lower.cpp b/legacy/core/src/compiler/ir/transform/cpu/kernel_lower.cpp deleted file mode 100644 index 88897b014..000000000 --- a/legacy/core/src/compiler/ir/transform/cpu/kernel_lower.cpp +++ /dev/null @@ -1,949 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#include - -#include "../../builder.hpp" -#include "../../content_hash.hpp" -#include "../../visitor.hpp" -#include "../buffer_schedule.hpp" -#include "../index_flatten.hpp" -#include "kernel_lower.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -SC_MODULE(pass.kernel_lowering_cpu) -namespace std { -template <> struct hash { - std::size_t - operator()(const dnnl::impl::graph::gc::brgemm::postop_setting_t &k) const { - size_t ret = 0; - dnnl::impl::graph::gc::hash_combine(ret, k.pack_info_[0]); - dnnl::impl::graph::gc::hash_combine(ret, k.pack_info_[1]); - return ret; - } -}; - -template <> -struct hash> { - std::size_t - operator()(const std::pair - &bdmask_pair) const { - size_t ret = 0; - for (auto &m : bdmask_pair.first) { - dnnl::impl::graph::gc::hash_combine(ret, m); - } - dnnl::impl::graph::gc::hash_combine(ret, bdmask_pair.second); - return ret; - } -}; -} // namespace std -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -SC_DECL_PASS_INFO(kernel_lowering_cpu, - SC_PASS_DEPENDS_ON(constant_folder, buffer_scheduler), - SC_PASS_REQUIRE_STATE(), SC_PASS_REQUIRE_NOT_STATE(), - SC_PASS_SET_STATE(), SC_PASS_UNSET_STATE()); - -using namespace builtin; -using namespace brgemm; - -static bool check_arg_range(const std::vector &args, - std::vector &cached_args, int start, - int end) { - cached_args.reserve(cached_args.size() + end - start); - // check the parameters, if they are all constants, - // we can cache the kernel - for (int i = start; i < end; i++) { - if (!args[i].isa() && !args[i].isa() && - !args[i].isa()) { - // not const, don't cache - return false; - } - cached_args.emplace_back(args[i]); - } - return true; -} - -static std::vector get_brgemm_attrs_key(const sc_brgemm_attrs_t &attrs, - size_t &valid_sz, - bool &range_optimize) { - std::vector key(attr_key::nkeys, 0); - valid_sz = 0; - for (auto &attr : attrs) { - if (attr.first < brgemm::attr_key::nkeys) { - valid_sz++; - key[attr.first] = attr.second; - } else if (utils::is_one_of(attr.first, - brgemm::attr_key::M_range_upper_bound, - brgemm::attr_key::N_range_upper_bound, - brgemm::attr_key::K_range_upper_bound)) { - range_optimize = true; - } - } - return key; -} - -static void validate_brgemm_attrs(const sc_brgemm_attrs_t &brg_attrs, - const sc_brgemm_bd_mask_t &bd_mask, - const int bd_mask_set_num, - bool &use_bd_mask) { - size_t dummy; - bool dummy1; - auto keys = get_brgemm_attrs_key(brg_attrs, dummy, dummy1); - if (keys[attr_key::use_uker]) { - COMPILE_ASSERT(keys[attr_key::max_bs] > 0, - "max_bs should be >0 and valid number for the bs used by " - "brgemm when use_uker=true, but got " - << keys[attr_key::max_bs] << "."); - } - use_bd_mask = false; - if (keys[attr_key::bd_mask_level] > 0) { - COMPILE_ASSERT(!bd_mask.empty(), - "bd_mask should be specified when bd_mask_level>0."); - COMPILE_ASSERT(bd_mask_set_num > 0, - "bd_mask_set_num should be >0 when bd_mask_level>0, but got " - << bd_mask_set_num << "."); - use_bd_mask = true; - } -} - -static expr -get_brgemm_attrs_arg(const ir_module_ptr &mod, const sc_brgemm_attrs_t &attrs, - std::unordered_map, expr> &cache, - bool &range_optimize) { - size_t sz; - std::vector key = get_brgemm_attrs_key(attrs, sz, range_optimize); - if (sz == 0) { - return get_ir_null(); - } - COMPILE_ASSERT(sz <= attr_key::nkeys, - "Size of user defined attributes should not exceed nkeys!"); - if (cache.find(key) != cache.end()) { - return cache[key]; - } - size_t tsr_sz = sz * sizeof(attrs_setting_t::attrs_map_t) + sizeof(int64_t); - std::vector data(tsr_sz, 0); - attrs_setting_t *setting = reinterpret_cast(data.data()); - setting->num_ = sz; - int c = 0; - for (auto &it : attrs) { - if (it.first < brgemm::attr_key::nkeys) { - setting->map_[c++] = it; - } - } - auto init = std::make_shared(data.data(), tsr_sz); - auto tsr = builder::make_tensor("__brgemm_attrs", {tsr_sz}, datatypes::u8, - address_space::automatic, init); - mod->add_global_var( - builder::make_var_tensor_def_unattached(tsr, linkage::private_global) - .static_as()); - cache[key] = tsr; - return tsr; -} - -static std::pair get_brgemm_bd_mask_arg( - const ir_module_ptr &mod, const sc_brgemm_bd_mask_t &bd_mask, - const expr &bd_mask_idx, const expr &bd_mask_len, const int bd_mask_set_num, - std::unordered_map, - std::pair> &cur_cache, - std::unordered_map &full_cached, - std::vector> &brg_bd_mask_arr_assignment) { - size_t sz = bd_mask.size(); - auto bd_mask_pair = - std::pair(bd_mask, bd_mask_set_num); - if (sz == 0) { - return std::pair(get_ir_null(), get_ir_null()); - } - if (cur_cache.find(bd_mask_pair) != cur_cache.end()) { - return cur_cache[bd_mask_pair]; - } - - expr full_bd_mask; - if (full_cached.find(bd_mask) == full_cached.end()) { - size_t tsr_sz = sz; - auto init = std::make_shared(bd_mask); - expr full_bd_mask_ = - builder::make_tensor("__brgemm_full_bd_mask", {tsr_sz}, datatypes::u8, - address_space::automatic, init); - mod->add_global_var(builder::make_var_tensor_def_unattached( - full_bd_mask_, linkage::private_global) - .static_as()); - full_bd_mask = full_bd_mask_; - full_cached[bd_mask] = full_bd_mask; - } else { - full_bd_mask = full_cached[bd_mask]; - } - - expr bd_mask_arr = - mod->make_global_tensor(datatypes::pointer, "__brgemm_bd_mask_arr", - {bd_mask_set_num}, linkage::private_global); - std::vector bd_mask_arr_assignment(bd_mask_set_num); - for (int i = 0; i < bd_mask_set_num; ++i) { - bd_mask_arr_assignment.at(i) = builder::make_assign_unattached( - builder::make_indexing(bd_mask_arr, {i}), - builder::tensor_ptr(full_bd_mask, {i * bd_mask_len})); - } - brg_bd_mask_arr_assignment.push_back(bd_mask_arr_assignment); - auto ret = std::pair(bd_mask_arr, bd_mask_arr[bd_mask_idx]); - cur_cache[bd_mask_pair] = ret; - - return ret; -} - -static expr get_brgemm_postops_setting_arg( - const ir_module_ptr &mod, const sc_brgemm_postops_setting_t &postops, - std::unordered_map &cache) { - if (postops.empty()) { - return get_ir_null(); - } - if (cache.find(postops) != cache.end()) { - return cache[postops]; - } - size_t sz = postops.size(); - size_t tsr_sz = sz * sizeof(postop_setting_t) + sizeof(int64_t); - std::vector data(tsr_sz, 0); - postops_setting_t *setting = - reinterpret_cast(data.data()); - setting->num_ = sz; - int c = 0; - for (auto &op : postops) { - setting->ops_[c++] = op; - } - auto init = std::make_shared(data.data(), tsr_sz); - expr tsr = - builder::make_tensor("__brgemm_postops_setting", {tsr_sz}, datatypes::u8, - address_space::automatic, init); - mod->add_global_var( - builder::make_var_tensor_def_unattached(tsr, linkage::private_global) - .static_as()); - cache[postops] = tsr; - return tsr; -} - -// has post ops -static expr get_brgemm_postops_data_arg(std::vector &ret, - const std::vector &ops_data) { - assert(ops_data.size() == brgemm::postops_data_init_func_nargs); - auto &in_bufs = ops_data; - auto postop_data = builder::make_tensor("__brgemm_postops_data", - {postops_data_size}, datatypes::u8); - - expr bin_ptr = in_bufs[2]; - if (!in_bufs[2]->equals(get_ir_null())) { - bin_ptr = builder::make_tensor("__binary_rhs_ptr", {1}, datatypes::pointer); - ret.emplace_back(builder::make_var_tensor_def_unattached(bin_ptr)); - ret.back().remove_const()->attr().set(attr_keys::tsr_dont_buf_sched, true); - ret.emplace_back( - builder::make_assign_unattached(bin_ptr[UINT64_C(0)], in_bufs[2])); - } - ret.emplace_back(builder::make_var_tensor_def_unattached(postop_data)); - ret.emplace_back(builder::make_evaluate_unattached(builder::make_call( - builtin::get_brgemm_postops_data_init_func(), - {postop_data, in_bufs[0], in_bufs[1], bin_ptr, in_bufs[3], in_bufs[4], - in_bufs[5], in_bufs[6], in_bufs[7], in_bufs[8], in_bufs[9], in_bufs[10], - in_bufs[11], in_bufs[12], in_bufs[13]}))); - - return postop_data; -} - -static sc_data_type_t infer_out_dtype(const sc_data_type_t &in_dtype) { - if (utils::is_one_of(in_dtype, datatypes::u8, datatypes::s8)) { - return datatypes::s32; - } - return datatypes::f32; -} - -static expr brgemm_init_kernel_cache(brgemm_mode mode, - scflags_t::brgemm_backend_t backend, - const std::vector &args, - std::vector &cached_args, - float beta, bool has_postop) { - if (mode == brgemm_mode::stride) { - // cache basic args - const int expected_cache_org_args = brgemm_args::NUM_BASIC_ARGS_STRIDE; - const int expected_cache_extra_start = brgemm_args::NUM_FULL_ARGS_STRIDE; - const int expected_cache_extra_args = - brgemm_args::extra_args_offset::cache_nargs; - // +1 for context pointer - assert(args.size() == brgemm_args::NUM_FULL_ARGS_STRIDE + - brgemm_args::extra_args_offset::nargs + 1); - if (!check_arg_range(args, cached_args, brgemm_args::M, - expected_cache_org_args)) { - return expr(); - } - if (!check_arg_range(args, cached_args, expected_cache_extra_start, - expected_cache_extra_start + - expected_cache_extra_args)) { - return expr(); - } - return get_brgemm_creator_and_call_func(mode, backend, has_postop) - .first( - args[brgemm_args::M], args[brgemm_args::N], args[brgemm_args::K], - args[brgemm_args::LDA], args[brgemm_args::LDB], - args[brgemm_args::LDC], args[brgemm_args::STRIDE_A], - args[brgemm_args::STRIDE_B], beta, - args[brgemm_args::NUM_FULL_ARGS_STRIDE + - brgemm_args::extra_args_offset::dtypeA], // dtypeA - args[brgemm_args::NUM_FULL_ARGS_STRIDE + - brgemm_args::extra_args_offset::dtypeB], // dtypeB - args[brgemm_args::NUM_FULL_ARGS_STRIDE + - brgemm_args::extra_args_offset::brg_attrs], // attrs - args[brgemm_args::NUM_FULL_ARGS_STRIDE + - brgemm_args::extra_args_offset::bd_mask], // bd_mask - args[brgemm_args::NUM_FULL_ARGS_STRIDE + - brgemm_args::extra_args_offset::postops_setting]); // postops - // set - } else { - // cache basic args and bdmask - // brgemm_args::LEN does not require to be cached - // so we use NUM_BASIC_ARGS_STRIDE rather than NUM_BASIC_ARGS_LIST here - const int expected_cache_org_args = brgemm_args::NUM_BASIC_ARGS_STRIDE; - const int expected_cache_extra_start = brgemm_args::NUM_FULL_ARGS_LIST; - const int expected_cache_extra_args = - brgemm_args::extra_args_offset::cache_nargs; - // +1 for context pointer - assert(args.size() == brgemm_args::NUM_FULL_ARGS_LIST + - brgemm_args::extra_args_offset::nargs + 1); - if (!check_arg_range(args, cached_args, brgemm_args::M, - expected_cache_org_args)) { - return expr(); - } - if (!check_arg_range(args, cached_args, expected_cache_extra_start, - expected_cache_extra_start + - expected_cache_extra_args)) { - return expr(); - } - return get_brgemm_creator_and_call_func(mode, backend, has_postop) - .first( - args[brgemm_args::M], args[brgemm_args::N], args[brgemm_args::K], - args[brgemm_args::LDA], args[brgemm_args::LDB], - args[brgemm_args::LDC], beta, - args[brgemm_args::NUM_FULL_ARGS_LIST + - brgemm_args::extra_args_offset::dtypeA], // dtypeA - args[brgemm_args::NUM_FULL_ARGS_LIST + - brgemm_args::extra_args_offset::dtypeB], // dtypeB - args[brgemm_args::NUM_FULL_ARGS_LIST + - brgemm_args::extra_args_offset::brg_attrs], // attrs - args[brgemm_args::NUM_FULL_ARGS_LIST + - brgemm_args::extra_args_offset::bd_mask], // bd_mask - args[brgemm_args::NUM_FULL_ARGS_LIST + - brgemm_args::extra_args_offset::postops_setting]); // postops - // set - } -} - -static expr brgemm_run(brgemm_mode mode, scflags_t::brgemm_backend_t backend, - const expr &cache, const std::vector &args, - bool has_postop) { - if (mode == brgemm_mode::stride) { - const int expected_num_args = brgemm_args::NUM_FULL_ARGS_STRIDE + - brgemm_args::extra_args_offset::nargs; - assert(args.size() == expected_num_args + 1); - auto run_func = - get_brgemm_creator_and_call_func(mode, backend, has_postop).second; - if (has_postop) { - return run_func(cache, args[brgemm_args::A], args[brgemm_args::B], - args[brgemm_args::C], args[brgemm_args::NUM], - /*postop data*/ - args[brgemm_args::NUM_FULL_ARGS_STRIDE + - brgemm_args::extra_args_offset::postops_data], - /*c_buf*/ - args[brgemm_args::NUM_FULL_ARGS_STRIDE + - brgemm_args::extra_args_offset::c_buf], - args[brgemm_args::NUM_FULL_ARGS_STRIDE + - brgemm_args::extra_args_offset::top_pad], - args[brgemm_args::NUM_FULL_ARGS_STRIDE + - brgemm_args::extra_args_offset::bottom_pad], - /*ctx*/ args.back()); - } - return run_func(cache, args[brgemm_args::A], args[brgemm_args::B], - args[brgemm_args::C], args[brgemm_args::NUM], - args[brgemm_args::NUM_FULL_ARGS_STRIDE + - brgemm_args::extra_args_offset::top_pad], - args[brgemm_args::NUM_FULL_ARGS_STRIDE + - brgemm_args::extra_args_offset::bottom_pad], - /*ctx*/ args.back()); - } else { - const int expected_num_args = - brgemm_args::NUM_FULL_ARGS_LIST + brgemm_args::extra_args_offset::nargs; - assert(args.size() == expected_num_args + 1); - auto run_func = - get_brgemm_creator_and_call_func(mode, backend, has_postop).second; - if (has_postop) { - return run_func(cache, args[brgemm_args::A], args[brgemm_args::B], - args[brgemm_args::C], args[brgemm_args::NUM], - args[brgemm_args::STRIDE_A], args[brgemm_args::STRIDE_B], - args[brgemm_args::LEN], - args[brgemm_args::NUM_FULL_ARGS_LIST + - brgemm_args::extra_args_offset::dtypeA], - args[brgemm_args::NUM_FULL_ARGS_LIST + - brgemm_args::extra_args_offset::dtypeB], - /*postop data*/ - args[brgemm_args::NUM_FULL_ARGS_LIST + - brgemm_args::extra_args_offset::postops_data], - /*c_buf*/ - args[brgemm_args::NUM_FULL_ARGS_LIST + - brgemm_args::extra_args_offset::c_buf], - args[brgemm_args::NUM_FULL_ARGS_LIST + - brgemm_args::extra_args_offset::top_pad], - args[brgemm_args::NUM_FULL_ARGS_LIST + - brgemm_args::extra_args_offset::bottom_pad], - /*ctx*/ args.back()); - } - return run_func(cache, args[brgemm_args::A], args[brgemm_args::B], - args[brgemm_args::C], args[brgemm_args::NUM], - args[brgemm_args::STRIDE_A], args[brgemm_args::STRIDE_B], - args[brgemm_args::LEN], - args[brgemm_args::NUM_FULL_ARGS_LIST + - brgemm_args::extra_args_offset::dtypeA], - args[brgemm_args::NUM_FULL_ARGS_LIST + - brgemm_args::extra_args_offset::dtypeB], - args[brgemm_args::NUM_FULL_ARGS_LIST + - brgemm_args::extra_args_offset::top_pad], - args[brgemm_args::NUM_FULL_ARGS_LIST + - brgemm_args::extra_args_offset::bottom_pad], - /*ctx*/ args.back()); - } -} - -static expr range_brgemm_run(brgemm_mode mode, - scflags_t::brgemm_backend_t backend, - const expr &cache, const std::vector &args, - bool has_postop) { - if (mode == brgemm_mode::stride) { - const int expected_num_args = brgemm_args::NUM_FULL_ARGS_STRIDE + - brgemm_args::extra_args_offset::nargs; - assert(args.size() == expected_num_args + 1); - auto run_func = get_brgemm_call_range_func(mode); - return run_func(cache, args[brgemm_args::M], args[brgemm_args::N], - args[brgemm_args::K], args[brgemm_args::A], - args[brgemm_args::B], args[brgemm_args::C], - args[brgemm_args::NUM], - args[brgemm_args::NUM_FULL_ARGS_STRIDE + - brgemm_args::extra_args_offset::top_pad], - args[brgemm_args::NUM_FULL_ARGS_STRIDE + - brgemm_args::extra_args_offset::bottom_pad], - /*ctx*/ args.back()); - } else { - const int expected_num_args = - brgemm_args::NUM_FULL_ARGS_LIST + brgemm_args::extra_args_offset::nargs; - assert(args.size() == expected_num_args + 1); - auto run_func = get_brgemm_call_range_func(mode); - return run_func(cache, args[brgemm_args::M], args[brgemm_args::N], - args[brgemm_args::K], args[brgemm_args::A], - args[brgemm_args::B], args[brgemm_args::C], - args[brgemm_args::NUM], args[brgemm_args::STRIDE_A], - args[brgemm_args::STRIDE_B], args[brgemm_args::LEN], - args[brgemm_args::NUM_FULL_ARGS_LIST + - brgemm_args::extra_args_offset::dtypeA], - args[brgemm_args::NUM_FULL_ARGS_LIST + - brgemm_args::extra_args_offset::dtypeB], - args[brgemm_args::NUM_FULL_ARGS_LIST + - brgemm_args::extra_args_offset::top_pad], - args[brgemm_args::NUM_FULL_ARGS_LIST + - brgemm_args::extra_args_offset::bottom_pad], - /*ctx*/ args.back()); - } -} - -class kernel_lower_impl_t : public ir_visitor_t { -public: - using ir_visitor_t::dispatch; - using ir_visitor_t::visit; - // the kernel parameters => kernel pointer mapping - using param_cache_table = content_hash_map, expr>; - ir_module_ptr mod_; - int optimize_; - int brg_bdmask_set_num_ = 0; - bool brg_use_bdmask_ = false; - // brgemm init stmts includes postops_data/c_buf - std::vector brg_postop_init_; - std::unordered_map, expr> attrs_cache_; - std::unordered_map, std::pair> - cur_bd_mask_cache_; - std::unordered_map full_bd_mask_cache_; - std::unordered_map postop_set_cache_; - std::vector> sc_kernel_cache_assignment_; - std::vector> brg_bd_mask_arr_assignment_; - // the kernel name => param_cache_table mapping - std::unordered_map kernel_cache; - typedef expr (*init_func_t)(brgemm_mode mode, - scflags_t::brgemm_backend_t backend, - const std::vector &args, - std::vector &cached_args, float beta, - bool has_postop); - using run_func_t = expr (*)(brgemm_mode, scflags_t::brgemm_backend_t, - const expr &, const std::vector &, - bool has_postop); - - expr_c optimize_range_kernel_call( - brgemm_mode mode, scflags_t::brgemm_backend_t backend, expr_c v, - const std::vector &args, const std::string &name, - run_func_t run_func, const sc_brgemm_attrs_t &brg_attrs, float beta, - bool has_postop, bool use_bdmask) { - std::vector cached_args; - if (backend != scflags_t::brgemm_backend_t::dnnl || has_postop || - use_bdmask) { - SC_MODULE_INFO << "Cannot optimize the range kernel call: " << v; - return v; - } - auto M_iter = brg_attrs.find(brgemm::attr_key::M_range_upper_bound); - auto N_iter = brg_attrs.find(brgemm::attr_key::N_range_upper_bound); - auto K_iter = brg_attrs.find(brgemm::attr_key::K_range_upper_bound); - if ((M_iter == brg_attrs.end() && !args[brgemm_args::M].isa()) || - (N_iter == brg_attrs.end() && !args[brgemm_args::N].isa()) || - (K_iter == brg_attrs.end() && !args[brgemm_args::K].isa())) { - SC_MODULE_INFO << "Cannot optimize the range kernel call: " << v; - return v; - } - auto M_tail_iter = brg_attrs.find(brgemm::attr_key::M_range_tail_value); - auto N_tail_iter = brg_attrs.find(brgemm::attr_key::N_range_tail_value); - auto K_tail_iter = brg_attrs.find(brgemm::attr_key::K_range_tail_value); - auto get_tail_value = [](const sc_brgemm_attrs_t &m, - const sc_brgemm_attrs_t::const_iterator &tail_it, - bool is_range) { - if (is_range) { - if (tail_it != m.end()) { - return static_cast(tail_it->second); - } - return brg_range_tail_value::dyn_tail; - } - return brg_range_tail_value::no_tail; - }; - bool is_M_range = M_iter != brg_attrs.end(); - bool is_N_range = N_iter != brg_attrs.end(); - bool is_K_range = K_iter != brg_attrs.end(); - int M_tail_value = get_tail_value(brg_attrs, M_tail_iter, is_M_range); - int N_tail_value = get_tail_value(brg_attrs, N_tail_iter, is_N_range); - int K_tail_value = get_tail_value(brg_attrs, K_tail_iter, is_K_range); - int M_upper_bound = static_cast( - is_M_range ? M_iter->second : get_expr_as_int(args[brgemm_args::M])); - int N_upper_bound = static_cast( - is_N_range ? N_iter->second : get_expr_as_int(args[brgemm_args::N])); - int K_upper_bound = static_cast( - is_K_range ? K_iter->second : get_expr_as_int(args[brgemm_args::K])); - const int expected_cache_org_args = brgemm_args::NUM_BASIC_ARGS_STRIDE; - const int expected_cache_extra_start = - mode == brgemm_mode::stride ? brgemm_args::NUM_FULL_ARGS_STRIDE - : brgemm_args::NUM_FULL_ARGS_LIST; - const int expected_cache_extra_args = - brgemm_args::extra_args_offset::cache_nargs; - // start from lda - if (!check_arg_range(args, cached_args, brgemm_args::LDA, - expected_cache_org_args)) { - return v; - } - if (!check_arg_range(args, cached_args, expected_cache_extra_start, - expected_cache_extra_start + - expected_cache_extra_args)) { - return v; - } - int LDA = static_cast(get_expr_as_int(args[brgemm_args::LDA])); - int LDB = static_cast(get_expr_as_int(args[brgemm_args::LDB])); - int LDC = static_cast(get_expr_as_int(args[brgemm_args::LDC])); - int stride_a = - static_cast(get_expr_as_int(args[brgemm_args::STRIDE_A])); - int stride_b = - static_cast(get_expr_as_int(args[brgemm_args::STRIDE_B])); - int dtypeA = static_cast( - get_expr_as_int(args[expected_cache_extra_start + - brgemm_args::extra_args_offset::dtypeA])); - int dtypeB = static_cast( - get_expr_as_int(args[expected_cache_extra_start + - brgemm_args::extra_args_offset::dtypeB])); - void *attr_ptr = nullptr; - auto attr_arg = args[expected_cache_extra_start + - brgemm_args::extra_args_offset::brg_attrs]; - if (attr_arg.isa()) { - attr_arg = attr_arg.static_as()->init_value_->data_; - } - auto handle = (mode == brgemm_mode::stride - ? std::make_shared( - M_upper_bound, N_upper_bound, K_upper_bound, LDA, - LDB, LDC, stride_a, stride_b, beta, dtypeA, dtypeB, - attr_ptr, M_tail_value, N_tail_value, K_tail_value) - : std::make_shared( - M_upper_bound, N_upper_bound, K_upper_bound, LDA, - LDB, LDC, beta, dtypeA, dtypeB, attr_ptr, - M_tail_value, N_tail_value, K_tail_value)); - mod_->get_brg_range_handle_vec().emplace_back(handle); - // Make kernel pointer global var. will be auto-renamed - auto init = make_expr( - reinterpret_cast(handle.get()), datatypes::pointer); - auto handlev = - mod_->make_global_var(datatypes::pointer, "__sc_range_kernel_cache", - linkage::private_global, init); - // todo: add cache for range handle. - expr result = run_func(mode, backend, handlev, args, has_postop); - assert(result.defined()); - return result; - } - - expr_c optimize_kernel_call(brgemm_mode mode, - scflags_t::brgemm_backend_t backend, expr_c v, - const std::vector &args, - const std::string &name, init_func_t init_func, - run_func_t run_func, float beta, bool has_postop, - bool use_bdmask) { - std::vector cached_args; - - if (use_bdmask) { - int num_full_args = (mode == brgemm_mode::stride) - ? brgemm_args::NUM_FULL_ARGS_STRIDE - : brgemm_args::NUM_FULL_ARGS_LIST; - int bd_mask_arg_offset = - num_full_args + +brgemm_args::extra_args_offset::bd_mask; - expr bd_mask = args[bd_mask_arg_offset]; - expr bd_mask_idx = args[num_full_args - 3]; - assert(bd_mask.defined() && bd_mask_idx.defined()); - - expr result = get_ir_null(); - std::vector brg_args = args; - std::vector sc_kernel_cache; - // cached_args contains the original bd_mask with new index - std::vector cached_args_iter; - - // cached_args contains the bd_mask_arr - expr cachev = - init_func(mode, backend, args, cached_args, beta, has_postop); - if (!cachev.defined()) { - SC_MODULE_INFO << "Cannot optimize the kernel call: " << v; - return v; - } - - // find the param_cache_table in the kernel cache - auto first_itr = kernel_cache.find(name); - if (first_itr != kernel_cache.end()) { - auto &entry = first_itr->second; - auto second_itr = entry.find(cached_args); - if (second_itr != entry.end()) { - // if the same parameters are cached in the - // kernel_cache, reuse the cached kernel pointer - return run_func(mode, backend, second_itr->second[bd_mask_idx], args, - has_postop); - } - } - - expr sc_kernel_cache_arr = mod_->make_global_tensor( - datatypes::pointer, "__sc_kernel_cache_arr", {brg_bdmask_set_num_}, - linkage::private_global); - std::vector kernel_cache_assignment; - for (int i = 0; i < brg_bdmask_set_num_; ++i) { - brg_args[bd_mask_arg_offset] = bd_mask[i]; - expr cachev = init_func(mode, backend, brg_args, cached_args_iter, beta, - has_postop); - - if (!cachev.defined()) { - SC_MODULE_INFO << "Cannot optimize the kernel call: " << v; - return v; - } - kernel_cache_assignment.emplace_back(builder::make_assign_unattached( - builder::make_indexing(sc_kernel_cache_arr, {i}), cachev)); - } - sc_kernel_cache_assignment_.push_back(kernel_cache_assignment); - // put the var to the kernel_cache - kernel_cache[name][cached_args] = sc_kernel_cache_arr; - brg_args[bd_mask_arg_offset] = bd_mask[bd_mask_idx]; - result = run_func(mode, backend, sc_kernel_cache_arr[bd_mask_idx], - brg_args, has_postop); - assert(result.defined()); - - return result; - } else { - expr cachev = - init_func(mode, backend, args, cached_args, beta, has_postop); - - // check if the kernel lowerer agrees to optimize the kernel call - if (!cachev.defined()) { - SC_MODULE_INFO << "Cannot optimize the kernel call: " << v; - return v; - } - // find the param_cache_table in the kernel cache - auto first_itr = kernel_cache.find(name); - if (first_itr != kernel_cache.end()) { - auto &entry = first_itr->second; - auto second_itr = entry.find(cached_args); - if (second_itr != entry.end()) { - // if the same parameters are cached in the kernel_cache, - // reuse the cached kernel pointer - return run_func(mode, backend, second_itr->second, args, has_postop); - } - } - // Make kernel pointer global var. will be auto-renamed - expr cache = mod_->make_global_var(cachev->dtype_, "__sc_kernel_cache", - linkage::private_global, cachev); - // put the var to the kernel_cache - kernel_cache[name][cached_args] = cache; - expr result = run_func(mode, backend, cache, args, has_postop); - assert(result.defined()); - return result; - } - } - - expr_c visit(intrin_call_c v) override { - brgemm_args::extra_args_t *extras; - sc_data_type_t dtypeA, dtypeB; - brgemm_mode mode; - v = ir_visitor_t::visit(std::move(v)).checked_as(); - if (v->type_ == intrin_type::brgemm) { - mode = brgemm_mode::stride; - } else if (v->type_ == intrin_type::list_brgemm) { - mode = brgemm_mode::addr_list; - } else { - return v; - } - - extras = &v->intrin_attrs_->get( - intrin_attr::brgemm_extras); - COMPILE_ASSERT(extras->is_cpu_, "Found non-CPU brgemm: " << v); - dtypeA = extras->dtype_A_; - dtypeB = extras->dtype_B_; - sc_brgemm_attrs_t &brg_attrs = extras->brg_attrs_; - sc_brgemm_bd_mask_t &bd_mask = extras->bd_mask_; - brg_bdmask_set_num_ = extras->bd_mask_set_num_; - sc_brgemm_postops_setting_t &brg_postops_setting = extras->postops_setting_; - assert(mode == brgemm_mode::stride || mode == brgemm_mode::addr_list); - int num_basic_args = (mode == brgemm_mode::stride) - ? brgemm_args::NUM_BASIC_ARGS_STRIDE - : brgemm_args::NUM_BASIC_ARGS_LIST; - int num_full_args = (mode == brgemm_mode::stride) - ? brgemm_args::NUM_FULL_ARGS_STRIDE - : brgemm_args::NUM_FULL_ARGS_LIST; - COMPILE_ASSERT(v->args_.size() == static_cast(num_full_args), - "invalid number of brgemm args, expected to be " - << num_full_args << ", but got " << v->args_.size() - << "."); - - // layout of v->args (full args): - // | basic_args | postops_data list(11 elems) | c_buf | bdmask_idx | - // top_pad | bottom_pad - std::vector brg_postops_data = - std::vector(v->args_.begin() + num_basic_args, - v->args_.begin() + num_full_args - 4); - COMPILE_ASSERT(brg_postops_data.size() == - brgemm::postops_data_init_func_nargs, - "brg_postops_data.size() is expected to be " - << brgemm::postops_data_init_func_nargs << ", but got " - << brg_postops_data.size()); - bool use_bdmask = false; - validate_brgemm_attrs(brg_attrs, bd_mask, brg_bdmask_set_num_, use_bdmask); - brg_use_bdmask_ |= use_bdmask; - - // layout of opt_args: - // | basic_args | postops_data list(11 elems) | c_buf | bdmask_idx - // | dtypeA | dtypeB | attrs | bd_mask | postops setting | top_pad | - // bottom_pad | postops data | c_buf | stream | - // The 1st c_buf will be replaced by 2nd c_buf - std::vector opt_args{v->args_.begin(), v->args_.end()}; - opt_args.emplace_back(dtypeA.as_etype_int()); - opt_args.emplace_back(dtypeB.as_etype_int()); - // brgemm attrs - bool try_range_optimize = false; - expr brg_attrs_arg = - get_brgemm_attrs_arg(mod_, brg_attrs, attrs_cache_, try_range_optimize); - opt_args.emplace_back(brg_attrs_arg); - // bd mask - expr bd_mask_arr = get_ir_null(), cur_bd_mask = get_ir_null(); - if (use_bdmask) { - expr bd_mask_idx = v->args_[num_full_args - 3]; - expr bd_mask_len = v->args_[brgemm_args::M]; - auto bd_mask_arg = get_brgemm_bd_mask_arg( - mod_, bd_mask, bd_mask_idx, bd_mask_len, brg_bdmask_set_num_, - cur_bd_mask_cache_, full_bd_mask_cache_, brg_bd_mask_arr_assignment_); - bd_mask_arr = bd_mask_arg.first; - cur_bd_mask = bd_mask_arg.second; - } - opt_args.emplace_back(bd_mask_arr); - // brgemm postops setting - expr brg_setting_arg = get_brgemm_postops_setting_arg( - mod_, brg_postops_setting, postop_set_cache_); - opt_args.emplace_back(brg_setting_arg); - - // top and bottom padding - // TODO(zhicong): optimize the brgemm call with vpad in compilation - // phase - expr top_pad = v->args_[num_full_args - 2], - bottom_pad = v->args_[num_full_args - 1]; - opt_args.emplace_back(top_pad); - opt_args.emplace_back(bottom_pad); - - // brgemm postops data - std::vector &ret = brg_postop_init_; - expr brg_data_arg; - if (!brg_postops_setting.empty()) { - brg_data_arg = get_brgemm_postops_data_arg(ret, brg_postops_data); - } else { - auto ref = create_initialed_postops_data(); - for (size_t i = 0; i < brg_postops_data.size(); i++) { - COMPILE_ASSERT(brg_postops_data[i]->equals(ref[i]), - "Postops data is not empty when setting is empty."); - } - brg_data_arg = get_ir_null(); - } - opt_args.emplace_back(brg_data_arg); - - // brgemm c buf, currently we create a local buffer with M*N size - expr brg_c_buf = v->args_[num_full_args - 4]; - assert(brg_c_buf.defined()); - if (brg_c_buf->equals(get_ir_null())) { - if (!brg_postops_setting.empty()) { - brg_c_buf = builder::make_tensor( - "__brgemm_c_buf", - {opt_args[brgemm_args::M] * opt_args[brgemm_args::N]}, - infer_out_dtype(dtypeA)); - ret.emplace_back(builder::make_var_tensor_def_unattached(brg_c_buf)); - } - } - opt_args.emplace_back(brg_c_buf); - - // placeholder for the context, required by brgemm with AMX. - opt_args.emplace_back(get_ir_null()); - brg_postop_init_ = ret; - - // layout of no_opt_args: - // | basic_args | dtypeA | dtypeB | attrs | bd_mask | postops setting - // | top_pad | bottom_pad | postops data | c_buf | stream | - std::vector no_opt_args(opt_args.begin(), - opt_args.begin() + num_basic_args); - // +2 for old c_buf and bdmask_idx - no_opt_args.insert(no_opt_args.end(), - opt_args.begin() + num_basic_args + - brgemm::postops_data_init_func_nargs + 4, - opt_args.end()); - no_opt_args[num_basic_args + 3] = cur_bd_mask; - - bool optimized = optimize_ >= 1; - scflags_t::brgemm_backend_t backend = mod_->ctx_->flags_.brgemm_backend_; - auto fpair = get_brgemm_update_funcs(mode, backend); - func_t f = extras->cpu_.init_ ? fpair.second : fpair.first; - assert(f); - - if (!optimized) { - return builder::make_call(f, no_opt_args); - } else { - if (try_range_optimize) { - auto ret = optimize_range_kernel_call( - mode, backend, v, opt_args, f->name_, range_brgemm_run, brg_attrs, - extras->cpu_.init_ ? 0.0f : 1.0f, !brg_postops_setting.empty(), - use_bdmask); - if (!ret.ptr_same(v)) { - return ret; - } - } - // try general optimization again for bd mask and postop. - auto ret = optimize_kernel_call(mode, backend, v, opt_args, f->name_, - brgemm_init_kernel_cache, brgemm_run, - extras->cpu_.init_ ? 0.0f : 1.0f, - !brg_postops_setting.empty(), use_bdmask); - if (ret.ptr_same(v)) { - return builder::make_call(f, no_opt_args); - } - return ret; - } - } - - stmt_c visit(stmts_c v) override { - bool changed = false; - std::vector seq; - for (auto &st : v->seq_) { - brg_postop_init_.clear(); - auto new_st = dispatch(st); - changed |= !new_st.ptr_same(st); - if (new_st.isa() && - new_st.static_as()->value_.isa() && - !brg_postop_init_.empty()) { - seq.insert(seq.end(), brg_postop_init_.begin(), brg_postop_init_.end()); - changed = true; - } - seq.emplace_back(new_st); - } - if (changed) { - return copy_attr(*v, builder::make_stmts_unattached(seq)); - } - return v; - } - - kernel_lower_impl_t(ir_module_ptr mod, int optimize) - : mod_(std::move(mod)), optimize_(optimize) {} -}; - -const_ir_module_ptr kernel_lowering_cpu_t::operator()(const_ir_module_ptr m) { - auto ret = m->copy(); - kernel_lower_impl_t pass(ret, optimize_); - auto old_gval_size = ret->get_module_vars().size(); - for (auto &f : ret->get_contents()) { - f = std::const_pointer_cast(pass.dispatch(f)); - } - if (auto initf = ret->get_func("__sc_init__")) { - for (size_t i = old_gval_size; i < ret->get_module_vars().size(); i++) { - auto pvar = ret->get_module_vars()[i]; - // attrs/bdmask/set are tensors. - if (pvar->var_.isa()) { - auto name = pvar->var_.static_as()->name_; - // only kernel cache needs init define. - if (name.find("kernel_cache") != std::string::npos) { - initf->body_.checked_as()->seq_.emplace_back( - builder::make_assign_unattached(pvar->var_, pvar->init_)); - } - } - } - } else { - initf = ret->make_init_func(); - if (initf) - ret->add_func({initf}); - } - - if (pass.brg_use_bdmask_) { - auto initf = ret->get_func("__sc_init__"); - if (!initf) { - stmts seq = make_stmt(std::vector()); - initf = builder::make_func("__sc_init__", std::vector(), - std::move(seq), datatypes::void_t); - ret->add_func({initf}); - } - assert(initf && "__sc_init__ func is expected be presented in \ - the current ir module, but not."); - - for (auto &sts : pass.brg_bd_mask_arr_assignment_) { - for (auto &st : sts) { - initf->body_.checked_as()->seq_.push_back(st); - } - } - for (auto &sts : pass.sc_kernel_cache_assignment_) { - for (auto &st : sts) { - initf->body_.checked_as()->seq_.push_back(st); - } - } - } - - return ret; -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/transform/cpu/kernel_lower.hpp b/legacy/core/src/compiler/ir/transform/cpu/kernel_lower.hpp deleted file mode 100644 index 8d68573ae..000000000 --- a/legacy/core/src/compiler/ir/transform/cpu/kernel_lower.hpp +++ /dev/null @@ -1,44 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_CPU_KERNEL_LOWER_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_CPU_KERNEL_LOWER_HPP - -#include "../../module_pass.hpp" - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -/** - * Lower the micro-kernel calls. Try to replace them with optimal versions of - * APIs - * */ -class kernel_lowering_cpu_t : public module_pass_t { -public: - int optimize_; - kernel_lowering_cpu_t(int optimize) : optimize_(optimize) {} - const_ir_module_ptr operator()(const_ir_module_ptr m) override; - SC_DECL_PASS_INFO_FUNC(); -}; - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/transform/cpu/local_tensor_lower.cpp b/legacy/core/src/compiler/ir/transform/cpu/local_tensor_lower.cpp deleted file mode 100644 index b2b9c8c35..000000000 --- a/legacy/core/src/compiler/ir/transform/cpu/local_tensor_lower.cpp +++ /dev/null @@ -1,487 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#include "local_tensor_lower.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -SC_MODULE(pass.local_tensor_lower); - -SC_DECL_PASS_INFO(local_tensor_lowering_cpu, - SC_PASS_DEPENDS_ON(constant_folder, buffer_scheduler, - tensor_init, module_globals_resolver, - simple_loop_invariant_code_motion), - SC_PASS_REQUIRE_STATE(), SC_PASS_REQUIRE_NOT_STATE(), - SC_PASS_SET_STATE(), SC_PASS_UNSET_STATE()); - -static func_t set_noalias_function(func_t f) { - f->attr()[function_attrs::no_alias] = true; - return f; -} - -func_t get_acquire_const_cache_func() { - static func_t f_global = set_noalias_function(builder::_decl_func( - "sc_acquire_const_cache", datatypes::pointer, - {_arg_("stream", datatypes::pointer), _arg_("cacheptr", datatypes::index), - _arg_("size", datatypes::index), - _arg_("outinited", datatypes::s32, {1})})); - return f_global; -} - -func_t get_release_const_cache_func() { - static func_t f_global = builder::_decl_func( - "sc_release_const_cache", datatypes::void_t, - {_arg_("stream", datatypes::pointer), _arg_("cacheptr", datatypes::index), - _arg_("ptr", datatypes::pointer)}); - return f_global; -} - -func_t get_cpu_temp_malloc_func(bool is_thread_local) { - static func_t f_global = set_noalias_function(builder::_decl_func( - "sc_aligned_malloc", datatypes::pointer, - {_arg_("stream", datatypes::pointer), _arg_("size", datatypes::index)})); - static func_t f_local = set_noalias_function(builder::_decl_func( - "sc_thread_aligned_malloc", datatypes::pointer, - {_arg_("stream", datatypes::pointer), _arg_("size", datatypes::index)})); - return is_thread_local ? f_local : f_global; -} - -func_t get_cpu_temp_free_func(bool is_thread_local) { - static func_t f_global = builder::_decl_func( - "sc_aligned_free", datatypes::void_t, - {_arg_("stream", datatypes::pointer), _arg_("ptr", datatypes::pointer)}); - static func_t f_local = builder::_decl_func( - "sc_thread_aligned_free", datatypes::void_t, - {_arg_("stream", datatypes::pointer), _arg_("ptr", datatypes::pointer)}); - return is_thread_local ? f_local : f_global; -} - -namespace local_tsr_lower { -// record the base tensor and the offset of a tensor after buffer-scheduling -struct tensor_base_offset_t { - expr base_; - int64_t start_offset_; - int64_t end_offset_; // inclusive offset -}; - -bool tensor_less_than_by_name(const expr &v1, const expr &v2) { - return v1.checked_as()->name_ < v2.checked_as()->name_; -} - -// the "trace" for our pass to scan the positions in the base tensor to decide -// which tensors are alias -struct tensor_trace_t { - expr tsr_; - int64_t offset_; - bool is_end_; - bool operator<(const tensor_trace_t &v) const { - if (offset_ < v.offset_) { - return true; - } - if (offset_ > v.offset_) { - return false; - } - // offset == v.offset_ - if (is_end_ != v.is_end_) { - // let "start" trace < "end" trace to let them overlap - // if is_end, then v.is_end_ is false, and we should let v< *this - return !is_end_; - } - // offset == v.offset_ && is_end_ == v.is_end_ - // the same position trace, sort by tensor name - return tensor_less_than_by_name(tsr_, v.tsr_); - } -}; - -} // namespace local_tsr_lower - -static const char *shared_const_handle_name = "__shared_const_handle"; - -using namespace local_tsr_lower; - -class tensor_lower_impl_t : public ir_visitor_t { -public: - using ir_visitor_t::dispatch; - size_t threshold_; - bool is_in_parallel_; - expr cur_rtl_ctx_; - // the defined tensor stack. The first dimension is for nested stmts. The - // second is for ordering the tensors defined in the same scope - std::vector> defined_tsr_; - // tensor -> [base, start_offset, end_offset] - std::unordered_map scheduled_tensor_position_; - // the ordered tensors in scheduled_tensor_position_ - std::vector scheduled_tensors_; - // tenosr -> hoisted base tensor in outer loop - std::unordered_map hoisted_tensor_map_; - expr shared_const_handles, tsr_is_init; - - // not interested in expr - expr_c dispatch(expr_c v) override { return v; } - - int64_t get_tensor_size(const tensor &tsr) { - COMPILE_ASSERT(tsr->dims_.size() == 1 && tsr->dims_[0].isa(), - "Expecting 1D constant sized tensor"); - return get_const_as_int(tsr->dims_[0].static_as()) * - (int64_t)utils::get_sizeof_type(tsr->elem_dtype_); - } - - stmt_c visit(define_c v) override { - if (!v->var_.isa()) { - return v; - } - if (v->linkage_ == linkage::local && - v->var_.static_as()->name_ == shared_const_handle_name) { - shared_const_handles = v->var_; - } - if (v->linkage_ != linkage::local) { - return v; - } - // only interested in local tensors - auto tsr = v->var_.static_as(); - if (tsr->name_ == "__is_init") { - tsr_is_init = tsr; - } - if (v->init_.defined()) { - if (v->init_.isa()) { - auto tptr = v->init_.static_as(); - auto attr = tptr->base_->ptr_->attr_.get(); - if (attr && attr->get_or_else("hoisted", false)) { - // the tensor is a tensor based on a hoisted tensor - COMPILE_ASSERT(tptr->base_->ptr_.isa() && - !tptr->base_->ptr_.ptr_same(tsr), - "Expecting a tensor on the base of hoisted tensor"); - hoisted_tensor_map_[tsr] = tptr->base_->ptr_; - } else if (attr && - attr->get_or_else(attr_keys::can_be_scheduled, false)) { - auto base = tptr->base_->ptr_; - COMPILE_ASSERT(tptr->base_->idx_.size() == 1 && - tptr->base_->idx_[0].isa(), - "Expecting 1D constant sized tensor"); - auto offset = - get_const_as_int(tptr->base_->idx_[0].static_as()); - COMPILE_ASSERT(tsr->dims_.size() == 1 && - tsr->dims_[0].isa(), - "Expecting 1D constant sized tensor"); - auto tsr_size = get_tensor_size(tsr); - COMPILE_ASSERT(tsr_size > 0, "Bad size of tensor"); - // recursively find the base tensor - for (;;) { - auto itr = scheduled_tensor_position_.find(base); - if (itr != scheduled_tensor_position_.end()) { - base = itr->second.base_; - offset += itr->second.start_offset_; - } else { - break; - } - } - scheduled_tensor_position_.insert(std::make_pair( - tsr, tensor_base_offset_t{base, offset, offset + tsr_size - 1})); - scheduled_tensors_.emplace_back(tsr); - } - } - return v; - } - COMPILE_ASSERT(tsr->dims_.size() == 1, - "tensor_lower_impl needs 1D tensors"); - // check if it is staticaly-shaped and shape is small - size_t sz = utils::get_sizeof_type(tsr->elem_dtype_); - expr_c alloc_size; - - bool is_const = true; - const auto &dim = tsr->dims_[0]; - auto shared_cached_buffer = - any_map_t::fetch_or_else>( - v->var_->attr_.get(), attr_keys::shared_const, nullptr); - if (!dim.isa()) { - is_const = false; - } else { - sz *= get_const_as_int(dim.static_as()); - } - if (is_const && sz <= threshold_ && - !shared_cached_buffer && /*and the tensor is not marked - runtime_stack_alloc*/ - !(tsr->attr_ && - tsr->attr_->get_or_else(attr_keys::runtime_stack_alloc, false))) { - // if the tensor is small enough - return v; - } - alloc_size = - is_const ? expr(sz) - : auto_caster_t()(tsr->dims_[0] * - utils::get_sizeof_type(tsr->elem_dtype_)); - expr initv; - if (shared_cached_buffer) { - // handle special local tensors: shared cached const - auto handle_idx = any_map_t::fetch_or_null( - v->var_->attr_.get(), attr_keys::shared_const_base_idx); - COMPILE_ASSERT(handle_idx, "Expecting attr shared_const_base_idx"); - COMPILE_ASSERT(shared_const_handles.defined(), - "Expecting shared_const_handles defined"); - if (shared_cached_buffer->buf_base_->is_lazy_) { - COMPILE_ASSERT(tsr_is_init.defined(), "Expecting __is_init defined"); - initv = builder::make_call(get_acquire_const_cache_func(), - {cur_rtl_ctx_, - shared_const_handles[uint64_t(*handle_idx)], - alloc_size, tsr_is_init}); - } else { - initv = builder::make_reinterpret( - shared_const_handles[uint64_t(*handle_idx)], datatypes::pointer); - } - } else { - bool thread_loca = - is_in_parallel_ || - any_map_t::fetch_or_else(tsr->attr_.get(), "is_thread_buffer", false); - // a large local tensor/dynamic tensor - initv = builder::make_call(get_cpu_temp_malloc_func(thread_loca), - {cur_rtl_ctx_, alloc_size}); - } - defined_tsr_.back().emplace_back(tsr); - return copy_attr( - *v, builder::make_var_tensor_def_unattached(tsr, v->linkage_, initv)); - } - - stmt_c visit(stmts_c v) override { - defined_tsr_.emplace_back(); - auto ret = ir_visitor_t::visit(v); - auto ¤t_scope = defined_tsr_.back(); - if (!current_scope.empty()) { - assert(!ret.ptr_same(v)); - auto &seq = ret.checked_as()->seq_; - bool is_ret = !seq.empty() && seq.back().isa(); - for (auto itr = current_scope.rbegin(); itr != current_scope.rend(); - ++itr) { - stmt the_call; - if (auto cached_buffer = any_map_t::fetch_or_else< - std::shared_ptr>( - (*itr)->attr_.get(), attr_keys::shared_const, nullptr)) { - auto handle_idx = any_map_t::fetch_or_null( - (*itr)->attr_.get(), attr_keys::shared_const_base_idx); - COMPILE_ASSERT(handle_idx, "Expecting attr shared_const_base_idx"); - COMPILE_ASSERT(shared_const_handles.defined(), - "Expecting shared_const_handles defined"); - if (cached_buffer->buf_base_->is_lazy_) { - the_call = builder::make_evaluate_unattached(builder::make_call( - get_release_const_cache_func(), - {cur_rtl_ctx_, shared_const_handles[uint64_t(*handle_idx)], - *itr})); - } else { - // compile-time constants, do not need to call release - continue; - } - } else { - bool thread_loca = is_in_parallel_ || any_map_t::fetch_or_else( - (*itr)->attr_.get(), - "is_thread_buffer", false); - the_call = builder::make_evaluate_unattached(builder::make_call( - get_cpu_temp_free_func(thread_loca), {cur_rtl_ctx_, *itr})); - } - if ((*itr)->attr_ && - (*itr)->attr_->get_or_else("temp.may_inplace", false)) { - assert((*itr).isa()); - auto tsr = (*itr).static_as(); - the_call = builder::make_if_else_unattached( - tsr->dims_[0] > UINT64_C(0), - builder::make_stmts_unattached({the_call}), stmt()); - } - - // if the last stmt is ret, should insert before it. - // Otherwise, append to the last position - auto pos = is_ret ? (seq.end() - 1) : seq.end(); - seq.insert(pos, the_call); - } - } - defined_tsr_.pop_back(); - return ret; - } -}; - -static std::vector> -mark_alias_for_scheduled_tensors( - const std::unordered_map - &scheduled_tensor_position, - const std::unordered_map &hoisted_tensor_map, - const std::vector &scheduled_tensors) { - if (scheduled_tensor_position.empty()) - return {}; - std::vector> ret; - std::unordered_map> base_tsr_to_traces; - for (auto &kv : scheduled_tensor_position) { - auto &tsr = kv.first; - auto &base_info = kv.second; - base_tsr_to_traces[base_info.base_].emplace_back( - tensor_trace_t{tsr, base_info.start_offset_, false}); - base_tsr_to_traces[base_info.base_].emplace_back( - tensor_trace_t{tsr, base_info.end_offset_, true}); - ret.emplace_back(alias_info::get_or_create_alias_info(*tsr)); - } - // sort the base tensors by name to have stable result - using pair_expr_vec_trace = std::pair *>; - std::vector sorted_base_tsr_to_traces; - sorted_base_tsr_to_traces.reserve(base_tsr_to_traces.size()); - for (auto &kv : base_tsr_to_traces) { - sorted_base_tsr_to_traces.emplace_back(kv.first, &kv.second); - } - std::sort(sorted_base_tsr_to_traces.begin(), sorted_base_tsr_to_traces.end(), - [](const pair_expr_vec_trace &v1, const pair_expr_vec_trace &v2) { - return tensor_less_than_by_name(v1.first, v2.first); - }); - int64_t clique_id = 1; - for (auto &kv : sorted_base_tsr_to_traces) { - auto &base = kv.first; - auto &traces = *kv.second; - if (traces.empty()) { - continue; - } - // the traces are sorted by offset, then by is_end, then by tensor name - std::sort(traces.begin(), traces.end()); - // the algorithm: - // think there is a cursor pointing to an offset of the base tensor. We - // move the cursor from 0 offset to max offset of the tensor. The traces - // marks the start and end of scheduled tensors and are sorted by the - // offset. So when the cursor points to an offset, we can know which - // tensors contain the position. - - // the set of tensors that contain the current cursor - alias_info::alias_set_t cur_alias; - std::shared_ptr cur_clique; - for (auto &trace : traces) { - auto alias_id = alias_info::get_or_create_alias_info(*trace.tsr_); - if (!trace.is_end_) { - if (!cur_clique) { - cur_clique = cur_alias.copy(); - cur_clique->id_ = clique_id; - clique_id++; - } - cur_alias.set_.insert(alias_id); - alias_id->add_to_clique(cur_clique); - } else { - cur_alias.set_.erase(alias_id); - // a tensor leaves the clique, need to make a new clique when - // another tensor joins - cur_clique = nullptr; - } - } - } - - // if scheduled tensor is based on a hoisted tensor, it should have same - // alias with the hoisted base. Recurisvely find the hoisted base - std::vector> - to_add_hoisted_alias; - // the vector to_add_hoisted_alias is to remember which scheduled tensor - // need to copy which alias set of a base hoisted tensor - for (auto &tsr : scheduled_tensors) { - auto itr = scheduled_tensor_position.find(tsr); - auto &base_info = itr->second; - auto aid_tsr = alias_info::get_or_create_alias_info(*tsr); - expr base = base_info.base_; - for (;;) { - auto itr = hoisted_tensor_map.find(base); - if (itr != hoisted_tensor_map.end()) { - auto hoisted = itr->second; - auto sche_itr = scheduled_tensor_position.find(hoisted); - if (sche_itr != scheduled_tensor_position.end()) { - if (auto aid_base = alias_info::get_alias_info(*hoisted)) { - // the tensor "tsr" is based on hoisted tensor "base" - // and "tsr" should have same aliases of "base" - for (auto &aset : aid_base->alias_cliques_) { - if (!aset->set_.has(aid_tsr)) { - to_add_hoisted_alias.emplace_back(aid_tsr.get(), aset.get()); - } - } - } - auto &hoisted_base_info = sche_itr->second; - base = hoisted_base_info.base_; - } else { - break; - } - } else { - break; - } - } - } - for (auto &kv : to_add_hoisted_alias) { - auto &aid_tsr = kv.first; - auto &aset = kv.second; - auto cli_copy = aset->copy(); - aid_tsr->add_to_clique(cli_copy); - cli_copy->id_ = clique_id; - clique_id++; - } - // output to INFO log - if (auto sc_stream_temp = - runtime::get_info_logging_stream(__sc_module_name)) { - for (auto &kv : scheduled_tensor_position) { - (*sc_stream_temp.stream_) << kv.first << ':'; - if (auto aid = alias_info::get_alias_info(*kv.first)) { - for (auto &aset : aid->alias_cliques_) { - (*sc_stream_temp.stream_) << aset->id_ << ','; - } - } - (*sc_stream_temp.stream_) << '\n'; - } - } - return ret; -} - -func_c local_tensor_lowering_cpu_t::operator()(func_c m) { - if (m->attr_ && m->attr_->get_or_else(function_attrs::low_level, false)) { - return m; - } - tensor_lower_impl_t impl; - COMPILE_ASSERT( - m->params_.size() >= 2 && - m->params_.front()->dtype_ == datatypes::pointer, - "local_tensor_lowering_cpu_t expecting the first function arugment " - "as a pointer, got: " - << m); - impl.cur_rtl_ctx_ = m->params_.front(); - impl.threshold_ = size_threshold_; - impl.is_in_parallel_ = any_map_t::fetch_or_else( - m->attr_.get(), function_attrs::no_parallel, false); - auto ret = impl.dispatch(m); - if (!impl.scheduled_tensor_position_.empty()) { - auto alias_ids = mark_alias_for_scheduled_tensors( - impl.scheduled_tensor_position_, impl.hoisted_tensor_map_, - impl.scheduled_tensors_); - std::const_pointer_cast(ret)->attr()["alias_sets"] = - std::move(alias_ids); - } - return ret; -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/transform/cpu/local_tensor_lower.hpp b/legacy/core/src/compiler/ir/transform/cpu/local_tensor_lower.hpp deleted file mode 100644 index 44ae9e144..000000000 --- a/legacy/core/src/compiler/ir/transform/cpu/local_tensor_lower.hpp +++ /dev/null @@ -1,57 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_CPU_LOCAL_TENSOR_LOWER_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_CPU_LOCAL_TENSOR_LOWER_HPP - -#include "../../function_pass.hpp" - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -/** - * Insert allocation/deallocation calls for local tensor. The alloc/dealloc - * follows "first-alloc-last-dealloc" order to let the memory allocator use a - * stack for easier and faster implementation. This pass should be placed after - * module_globals_resolver_t - * */ -class local_tensor_lowering_cpu_t : public function_pass_t { -public: - // the threshold in bytes. If a local tensor is larger than the threshold, - // we should allocate it on heap (using alloc function). Otherwise, we - // should keep the tensor untouched (the backend should lower it a buffer on - // stack) - size_t size_threshold_; - func_c operator()(func_c m) override; - local_tensor_lowering_cpu_t(size_t size_threshold) - : size_threshold_(size_threshold) {} - SC_DECL_PASS_INFO_FUNC(); -}; - -func_t get_cpu_temp_malloc_func(bool is_thread_local); - -func_t get_cpu_temp_free_func(bool is_thread_local); -func_t get_acquire_const_cache_func(); -func_t get_release_const_cache_func(); - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/transform/cpu/target_specific_lower.cpp b/legacy/core/src/compiler/ir/transform/cpu/target_specific_lower.cpp deleted file mode 100644 index 51a8bc31c..000000000 --- a/legacy/core/src/compiler/ir/transform/cpu/target_specific_lower.cpp +++ /dev/null @@ -1,1157 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include -#include -#include -#include -#include - -#include "target_specific_lower.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -extern const uint32_t sc_log_const_int_vals_1[32]; -extern const uint32_t sc_log_const_int_vals_2[32]; -extern const uint32_t sc_erf_const_int_vals[6][32]; -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -SC_DECL_PASS_INFO(target_specific_lowering_cpu, - SC_PASS_DEPENDS_ON(bf16_fp16_legalizer, bf16_fp16_eliminator, - func_inliner, dyn_tensor_transformer), - SC_PASS_REQUIRE_STATE(), SC_PASS_REQUIRE_NOT_STATE(), - SC_PASS_SET_STATE(), SC_PASS_UNSET_STATE(FUNC_INLINED)); - -static expr gen_vec_const(uint32_t lanes, float f) { - return make_expr(f, sc_data_type_t::f32(lanes)); -} - -static expr gen_vec_const_int(uint32_t lanes, int64_t f) { - return make_expr(f, sc_data_type_t::s32(lanes)); -} - -static expr gen_vec_const_u32(uint32_t lanes, int64_t f) { - return make_expr(f, sc_data_type_t::u32(lanes)); -} - -static std::vector make_args_by_intrinsic(const intrin_call_c &node) { - std::vector ret; - int idx = 0; - for (auto &v : node->args_) { - ret.emplace_back( - builder::make_var(v->dtype_, "_intrin_v" + std::to_string(idx))); - idx += 1; - } - return ret; -} - -static std::string get_exp_func_name(const intrin_call_c &node) { - std::stringstream ss; - ss << "_should_inline_exp_" << node->dtype_; - return ss.str(); -} - -static std::string get_read_struct_func_name(const intrin_call_c &node) { - auto &name = node->intrin_attrs_->get(intrin_attr::struct_name); - auto &field = node->intrin_attrs_->get(intrin_attr::struct_field); - std::stringstream ss; - ss << "_should_inline_read_struct_" << name << "_" << field; - return ss.str(); -} -static std::string get_write_struct_func_name(const intrin_call_c &node) { - auto &name = node->intrin_attrs_->get(intrin_attr::struct_name); - auto &field = node->intrin_attrs_->get(intrin_attr::struct_field); - std::stringstream ss; - ss << "_should_inline_write_struct_" << name << "_" << field; - return ss.str(); -} - -static expr_c create_cast_bf16_to_f32(const context_ptr &ctx, const cast_c &v) { - builder::ir_builder_t builder; - auto &in = v->in_; - const auto count = - make_expr(16UL, sc_data_type_t::u32(in->dtype_.lanes_)); - - auto uint32_v = builder::make_cast( - sc_data_type_t::u32(in->dtype_.lanes_), - builder::make_reinterpret(in, sc_data_type_t::u16(in->dtype_.lanes_))); - auto tmpf32 = copy_attr( - *v, builder::make_reinterpret(uint32_v << count, - sc_data_type_t::f32(v->dtype_.lanes_))); - if (!v->dtype_.is_etype(sc_data_etype::F32)) { - tmpf32 = builder::make_cast(v->dtype_, tmpf32); - } - - return tmpf32; -} - -static expr_c create_cast_f32_to_bf16(const context_ptr &ctx, const cast_c &v) { - auto &in = v->in_; - COMPILE_ASSERT(in->dtype_.is_etype(sc_data_etype::F32), - "bf16 should be cast from f32."); - const auto count = - make_expr(16UL, sc_data_type_t::u32(in->dtype_.lanes_)); - - if (ctx->flags_.bf16_fast_trunc_) { - auto uint32_v = - builder::make_reinterpret(in, sc_data_type_t::u32(in->dtype_.lanes_)); - return copy_attr( - *v, builder::make_reinterpret( - builder::make_cast(sc_data_type_t::u16(in->dtype_.lanes_), - uint32_v >> count), - sc_data_type_t::bf16(in->dtype_.lanes_))); - } - if (ctx->machine_.device_type_ == runtime::target_machine_t::type::cpu && - ctx->machine_.cpu_flags_.fAVX512BF16) { - /* todo: add it to instrinsic for special tag */ - return copy_attr(*v, builder::make_cast(v->dtype_, in)); - } else { - auto uint32_v = - builder::make_reinterpret(in, sc_data_type_t::u32(in->dtype_.lanes_)); - - auto rounding_bias = - builder::make_int_and( - (uint32_v >> count), - builder::make_constant( - std::vector(in->dtype_.lanes_, (int64_t)1), - sc_data_type_t::u32(in->dtype_.lanes_))) + - builder::make_constant( - std::vector(in->dtype_.lanes_, (int64_t)0x7FFF), - sc_data_type_t::u32(in->dtype_.lanes_)); - // reinterpret to bf16 to inference wrapper node dtype e.g. - // select/intrin_call. - return copy_attr( - *v, builder::make_reinterpret( - builder::make_cast(sc_data_type_t::u16(in->dtype_.lanes_), - (uint32_v + rounding_bias) >> count), - sc_data_type_t::bf16(in->dtype_.lanes_))); - } -} - -static expr_c create_cast_fp16_to_f32(const context_ptr &ctx, const cast_c &v) { - if (ctx->machine_.device_type_ == runtime::target_machine_t::type::cpu && - (ctx->machine_.cpu_flags_.fAVX512FP16)) { - return v; - } else { - auto &in = v->in_; - const auto lanes = in->dtype_.lanes_; - auto u32_dtype = sc_data_type_t::u32(lanes); - - auto uint32_v = builder::make_cast( - u32_dtype, builder::make_reinterpret(in, sc_data_type_t::u16(lanes))); - // const uint32_t e = (x & 0x7C00) >> 10; - auto e = (builder::make_int_and( - uint32_v, gen_vec_const_u32(lanes, UINT64_C(0X7C00)))) >> - gen_vec_const_u32(lanes, UINT64_C(10)); - // const uint32_t m = (x & 0x03FF) << 13; - auto m = (builder::make_int_and(uint32_v, - gen_vec_const_u32(lanes, UINT64_C(0X03FF)))) - << gen_vec_const_u32(lanes, UINT64_C(13)); - // const uint32_t v = as_uint((float)m) >> 23; - auto l = - (builder::make_reinterpret( - builder::make_cast(sc_data_type_t::f32(lanes), m), u32_dtype)) >> - gen_vec_const_u32(lanes, UINT64_C(23)); - - // (x & 0x8000) << 16 - auto cond1 = (builder::make_int_and( - uint32_v, gen_vec_const_u32(lanes, UINT64_C(0x8000)))) - << gen_vec_const_u32(lanes, UINT64_C(16)); - // (e != 0) * ((e + 112) << 23 | m) - auto tmp_cond_2 = - builder::make_int_or((e + gen_vec_const_u32(lanes, UINT64_C(112))) - << gen_vec_const_u32(lanes, UINT64_C(23)), - m); - auto cond2 = - builder::make_select(e != gen_vec_const_u32(lanes, UINT64_C(0)), - tmp_cond_2, gen_vec_const_u32(lanes, UINT64_C(0))); - //((e == 0) & (m != 0)) * ((l - 37) << 23 | ((m << (150 - l)) & - // 0x007FE000)) - auto tmp_cond_3 = builder::make_int_or( - (l - gen_vec_const_u32(lanes, UINT64_C(37))) - << gen_vec_const_u32(lanes, UINT64_C(23)), - builder::make_int_and( - (m << (gen_vec_const_u32(lanes, UINT64_C(150)) - l)), - gen_vec_const_u32(lanes, UINT64_C(0X007FE000)))); - auto cond3 = builder::make_select( - e == gen_vec_const_u32(lanes, UINT64_C(0)), - builder::make_select(m != gen_vec_const_u32(lanes, UINT64_C(0)), - tmp_cond_3, gen_vec_const_u32(lanes, UINT64_C(0))), - gen_vec_const_u32(lanes, UINT64_C(0))); - /* - (x & 0x8000) << 16 | (e != 0) * ((e + 112) << 23 | m) - | ((e == 0) & (m != 0)) - * ((l - 37) << 23 | ((m << (150 - l)) & 0x007FE000)) - */ - auto f32_v = - builder::make_int_or(builder::make_int_or(cond1, cond2), cond3); - auto tmpf32 = copy_attr( - *v, builder::make_reinterpret(f32_v, sc_data_type_t::f32(lanes))); - if (!v->dtype_.is_etype(sc_data_etype::F32)) { - tmpf32 = builder::make_cast(v->dtype_, tmpf32); - } - return tmpf32; - } -} - -static expr_c create_cast_f32_to_fp16(const context_ptr &ctx, const cast_c &v) { - auto &in = v->in_; - if (ctx->machine_.device_type_ == runtime::target_machine_t::type::cpu && - (ctx->machine_.cpu_flags_.fAVX512FP16)) { - return v; - } else { - COMPILE_ASSERT(in->dtype_.is_etype(sc_data_etype::F32), - "fp16 should be cast from f32."); - const auto lanes = in->dtype_.lanes_; - auto u32_dtype = sc_data_type_t::u32(lanes); - - auto uint32_v = builder::make_reinterpret(in, u32_dtype); - auto b = uint32_v + gen_vec_const_u32(lanes, UINT64_C(0x00001000)); - auto e = builder::make_int_and( - b, gen_vec_const_u32(lanes, UINT64_C(0x7F800000))) >> - gen_vec_const_u32(lanes, UINT64_C(23)); - auto m = builder::make_int_and( - b, gen_vec_const_u32(lanes, UINT64_C(0x007FFFFF))); - // (b & 0x80000000) >> 16 - auto cond1 = builder::make_int_and( - b, gen_vec_const_u32(lanes, UINT64_C(0x80000000))) >> - gen_vec_const_u32(lanes, UINT64_C(16)); - // (e > 112) * (((e - 112) << 10) & 0x7C00) - auto tmp_cond_2 = builder::make_int_and( - (e - gen_vec_const_u32(lanes, UINT64_C(112))) - << gen_vec_const_u32(lanes, UINT64_C(10)), - builder::make_constant( - std::vector(in->dtype_.lanes_, (int64_t)0x7C00), - sc_data_type_t::u32(in->dtype_.lanes_))); - // (e > 112) * ((((e - 112) << 10) & 0x7C00) | m >> 13) - auto cond2 = builder::make_select( - e > gen_vec_const_u32(lanes, UINT64_C(112)), - builder::make_int_or(tmp_cond_2, - m >> gen_vec_const_u32(lanes, UINT64_C(13))), - gen_vec_const_u32(lanes, UINT64_C(0))); - // ((e < 113) & (e > 101)) * ((((0x007FF000 + m) >> (125 - e)) + 1) >> - // 1) - auto tmp_cond_3 = (((gen_vec_const_u32(lanes, UINT64_C(0x007FF000)) + m) >> - (gen_vec_const_u32(lanes, UINT64_C(125)) - e)) + - gen_vec_const_u32(lanes, UINT64_C(1))) >> - gen_vec_const_u32(lanes, UINT64_C(1)); - //((e < 113) & (e > 101)) * ((((0x007FF000 + m) >> (125 - e)) + 1) - //>> 1) - auto cond3 = builder::make_select( - e < gen_vec_const_u32(lanes, UINT64_C(113)), - builder::make_select(e > gen_vec_const_u32(lanes, UINT64_C(101)), - tmp_cond_3, gen_vec_const_u32(lanes, UINT64_C(0))), - gen_vec_const_u32(lanes, UINT64_C(0))); - // (e > 143) * 0x7FFF - auto cond4 = - builder::make_select(e > gen_vec_const_u32(lanes, UINT64_C(143)), - gen_vec_const_u32(lanes, UINT64_C(0x7FFF)), - gen_vec_const_u32(lanes, UINT64_C(0))); - /* - (b & 0x80000000) >> 16 - | (e > 112) * ((((e - 112) << 10) & 0x7C00) | m >> 13) - | ((e < 113) & (e > 101)) - * ((((0x007FF000 + m) >> (125 - e)) + 1) >> 1) - | (e > 143) * 0x7FFF - */ - auto fp16_v = builder::make_int_or( - builder::make_int_or(builder::make_int_or(cond1, cond2), cond3), cond4); - return copy_attr( - *v, - builder::make_reinterpret( - builder::make_cast(sc_data_type_t::u16(in->dtype_.lanes_), fp16_v), - sc_data_type_t::f16(in->dtype_.lanes_))); - } -} - -static std::string get_isnan_func_name(const intrin_call_c &node) { - std::stringstream ss; - ss << "_should_inline_isnan_" << node->dtype_; - return ss.str(); -} - -static std::string get_log_func_name(const intrin_call_c &node) { - std::stringstream ss; - ss << "_should_inline_log_" << node->dtype_; - return ss.str(); -} - -static std::string get_erf_func_name(const intrin_call_c &node) { - std::stringstream ss; - ss << "_should_inline_erf_" << node->dtype_; - return ss.str(); -} - -class target_specific_lower_cpu_impl_t; -static func_t create_isnan_func(const ir_module_ptr &mod, - const intrin_call_c &node, - target_specific_lower_cpu_impl_t *visitor) { - auto type = node->dtype_; - uint32_t elements = type.lanes_; - auto ZERO = - make_expr(INT64_C(0), sc_data_type_t::s32(elements)); - auto exponent_bits = make_expr(INT64_C(0x7F800000), - sc_data_type_t::s32(elements)); - auto rm_sign_bits = make_expr(INT64_C(0x7FFFFFFF), - sc_data_type_t::s32(elements)); - auto ty_epi_32 = sc_data_type_t::s32(elements); - builder::ir_builder_t builder; - _function_(type, the_sc_isnan_func, make_args_by_intrinsic(node)) { - _bind_(inval_f32); - _var_(inval, ty_epi_32); - inval = builder::make_reinterpret(inval_f32, ty_epi_32); - expr mask = (inval & exponent_bits) == exponent_bits; - expr temp = builder::make_select(mask, inval & rm_sign_bits, ZERO); - expr ret = exponent_bits < temp; - _return_(ret); - } - std::string fixed_name = get_isnan_func_name(node); - the_sc_isnan_func->name_ = fixed_name; - the_sc_isnan_func->decl_->name_ = fixed_name; - return the_sc_isnan_func; -} - -static func_t create_exp_func(const ir_module_ptr &mod, - const intrin_call_c &node, - target_specific_lower_cpu_impl_t *visitor) { - auto type = node->dtype_; - uint32_t elements = type.lanes_; - bool overflow_check = - !node->attr_ || node->attr_->get_or_else("overflow_check", true); - - auto ZERO = gen_vec_const(elements, 0.0f); - auto ln2 = gen_vec_const(elements, 0.693147181f); - auto minus_ln2 = gen_vec_const(elements, -0.693147181f); - auto one_over_ln2 = gen_vec_const(elements, 1.442695041f); - auto half_float = gen_vec_const(elements, 0.5f); - auto ONE_f = gen_vec_const(elements, 1.0f); - auto ONE_i = - make_expr(INT64_C(1), sc_data_type_t::s32(elements)); - auto ty_epi_32 = sc_data_type_t::s32(elements); - auto overflow_x = gen_vec_const(elements, 88.72283935f); - auto underflow_x = gen_vec_const(elements, -87.33654785f); - auto ret_infinity = - make_expr(std::numeric_limits::infinity(), type); - - builder::ir_builder_t builder; - _function_(type, the_exp_func, make_args_by_intrinsic(node)) { - assert(node->args_.size() == 1); - _bind_(inval); - _var_(res, type); - // to avoid overflow - _var_init_(a_, type, inval); - - // e^x = 2^k_int * e^r - _var_(k_float, type); - k_float = builder::make_floor( - a_ * one_over_ln2 + half_float); // k_float = floor(x / ln2 + 0.5f) - _var_(k_int, ty_epi_32); - k_int = builder::make_cast(ty_epi_32, k_float); // k_int = int(k_float) - - _var_(r, type); - // TODO(x): mabye utilize fnmadd, fmsub, etc. in the future - // r = a_ - k_float * ln2; - r = builder::make_fmadd(k_float, minus_ln2, a_); - - expr table[7]; - table[6] = gen_vec_const(elements, 0.142857143f); - table[5] = gen_vec_const(elements, 0.166666667f); - table[4] = gen_vec_const(elements, 0.2f); - table[3] = gen_vec_const(elements, 0.25f); - table[2] = gen_vec_const(elements, 0.333333333f); - table[1] = gen_vec_const(elements, 0.5f); - table[0] = ONE_f; - // Calculate e^r (Tn) - - const size_t top_idx = 5; - _var_(Tn, type); - Tn = builder::make_fmadd(r, table[top_idx], ONE_f); - for (auto loop = top_idx; loop > 0; loop--) { - // Tn = Tn * (r / i) + 1 - Tn = builder::make_fmadd(Tn, r * table[loop - 1], ONE_f); - } - - // 2^k_int, shift to exponent bits position - auto const_23 = - make_expr(INT64_C(23), sc_data_type_t::s32(elements)); - auto p = k_int << const_23; - - _var_(result, ty_epi_32); - result = p + builder::make_reinterpret(Tn, ty_epi_32); - - res = builder::make_reinterpret(result, type); - if (overflow_check) { - res = builder::make_select(inval > overflow_x, ret_infinity, res); - } - res = builder::make_select(inval < underflow_x, ZERO, res); - _return_(res); - } // namespace gc - - std::string fixed_name = get_exp_func_name(node); - the_exp_func->name_ = fixed_name; - the_exp_func->decl_->name_ = fixed_name; - return the_exp_func; -} - -static func_t create_log_func(const ir_module_ptr &mod, - const intrin_call_c &node, - target_specific_lower_cpu_impl_t *visitor); - -static func_t create_erf_func(const ir_module_ptr &mod, - const intrin_call_c &node, - target_specific_lower_cpu_impl_t *visitor); - -static size_t get_field_offset(const std::string &in, int field) { - if (in == dyn_tsr_struct_t::name) { - return dyn_tsr_struct_t::offsets[field]; - } else { - COMPILE_ASSERT(false, "struct " << in << " has not been supported!"); - } - return 0; -} - -static func_t -create_read_struct_func(const ir_module_ptr &mod, const intrin_call_c &node, - target_specific_lower_cpu_impl_t *visitor) { - builder::ir_builder_t builder; - auto name = node->intrin_attrs_->get(intrin_attr::struct_name); - auto field = node->intrin_attrs_->get(intrin_attr::struct_field); - auto dtype = get_dtype_from_struct_and_field(name, field); - _function_(dtype, read_struct, {node->args_[0]}) { - assert(node->args_.size() == 1); - expr util_tsr = builder::make_tensor("util_tsr", {1}, dtype); - util_tsr->attr().set(attr_keys::tsr_dont_buf_sched, true); - util_tsr->attr().set(attr_keys::no_dead_write, true); - util_tsr->attr().set(attr_keys::no_tensor2var, true); - size_t offset = get_field_offset(name, field); - builder::get_current_builder()->push_var_tensor_def( - util_tsr, linkage::local, - builder::tensor_ptr(node->args_[0], {offset})); - expr ret = builder::make_indexing(util_tsr, 0); - _return_(ret); - } - auto func_name = get_read_struct_func_name(node); - read_struct->name_ = func_name; - read_struct->decl_->name_ = func_name; - return read_struct; -} - -static func_t -create_write_struct_func(const ir_module_ptr &mod, const intrin_call_c &node, - target_specific_lower_cpu_impl_t *visitor) { - builder::ir_builder_t builder; - auto name = node->intrin_attrs_->get(intrin_attr::struct_name); - auto field = node->intrin_attrs_->get(intrin_attr::struct_field); - auto dtype = node->args_[1]->dtype_; - if (dtype.is_pointer()) { - dtype = datatypes::pointer; - } - _function_(datatypes::void_t, write_struct, {node->args_[0]}, - {builder::make_var(dtype, "_intrin_v")}) { - assert(node->args_.size() == 2); - assert(node->args_[1]->dtype_ == (dtype) || - node->args_[1]->dtype_.is_pointer()); - _bind_(dyn_tsr, inval); - expr util_tsr = builder::make_tensor( - "util_tsr", {1}, dtype.is_pointer() ? datatypes::pointer : dtype); - util_tsr->attr().set(attr_keys::tsr_dont_buf_sched, true); - util_tsr->attr().set(attr_keys::no_dead_write, true); - util_tsr->attr().set(attr_keys::no_tensor2var, true); - size_t offset = get_field_offset(name, field); - builder::get_current_builder()->push_var_tensor_def( - util_tsr, linkage::local, builder::tensor_ptr(dyn_tsr, {offset})); - builder::get_current_builder()->push_assign( - builder::make_indexing(util_tsr, 0), inval); - } - auto func_name = get_write_struct_func_name(node); - write_struct->name_ = func_name; - write_struct->decl_->name_ = func_name; - return write_struct; -} - -using intrin_func_creator = func_t (*)(const ir_module_ptr &mod, - const intrin_call_c &node, - target_specific_lower_cpu_impl_t *); -using intrin_func_namer = std::string (*)(const intrin_call_c &node); - -class target_specific_lower_cpu_impl_t : public ir_visitor_t { -public: - using ir_visitor_t::dispatch; - using ir_visitor_t::visit; - - context_ptr ctx_; - ir_module_ptr mod_; - int var_cnt_ = 0; - int parallel_depth_ = 0; - // use a var instead of complex arg expr - std::vector>> need_defs_; - const stmts_node_t *cur_scope_ = nullptr; - std::unordered_map> barrier_idle_func_args_; - - std::vector visit_need_def_args(const std::vector &args) { - std::vector new_args; - new_args.reserve(args.size()); - for (auto &arg : args) { - if (!arg.isa() && !arg.isa() && !arg.isa() && - !arg.isa() && !arg.isa()) { - expr_c cache_var = builder::make_var( - arg->dtype_, "_arg_cache_" + std::to_string(var_cnt_++)); - need_defs_.back().emplace_back(cache_var, arg); - new_args.emplace_back(cache_var); - } else { - new_args.emplace_back(arg); - } - } - return new_args; - } - - expr_c do_lower_saturated_cast(const intrin_call_c &v) { - assert(v->args_.size() == 1); - const auto &inval1 = v->args_[0]; - auto intype = v->args_[0]->dtype_; - auto outtype = v->dtype_; - auto ths = this; - bool is_s8 = outtype.type_code_ == sc_data_etype::S8; - float i8_min = is_s8 ? (float)std::numeric_limits::min() - : (float)std::numeric_limits::min(); - float i8_max = is_s8 ? (float)std::numeric_limits::max() - : (float)std::numeric_limits::max(); - auto i8_min_constant = gen_vec_const(intype.lanes_, i8_min); - auto i8_max_constant = gen_vec_const(intype.lanes_, i8_max); - bool use_avx512_path = true; - auto u8s8_valid_val = [&]() { - if (use_avx512_path) { - if (is_s8) { - return builder::make_min(i8_max_constant, inval1); - } else { - // u8 - return builder::make_max(i8_min_constant, inval1); - } - } - return builder::make_min(i8_max_constant, - builder::make_max(i8_min_constant, inval1)); - }; - auto u8s8_saturate_cast = [&]() { - auto real_in = u8s8_valid_val(); - auto cast_target_type = (is_s8 || !use_avx512_path) - ? sc_data_type_t::s32(intype.lanes_) - : sc_data_type_t::u32(intype.lanes_); - real_in = builder::make_round_and_cast(real_in, cast_target_type); - if (use_avx512_path) { - return builder::make_saturated_cast(real_in, v->dtype_); - } else { - return builder::make_cast(v->dtype_, real_in); - } - }; - if (mod_->ctx_->machine_.cpu_flags_.fAVX512F) { - // the fast path for AVX512 - if (v->dtype_ == sc_data_type_t::s8(16)) { - if (intype == sc_data_type_t::s32(16)) { - return v; - } else if (intype == sc_data_type_t::f32(16)) { - return u8s8_saturate_cast(); - } - } else if (v->dtype_ == sc_data_type_t::u8(16)) { - if (intype == sc_data_type_t::s32(16)) { - auto zero = - make_expr(UINT64_C(0), sc_data_type_t::s32(16)); - return builder::make_saturated_cast(builder::make_max(inval1, zero), - v->dtype_); - } else if (intype == sc_data_type_t::f32(16)) { - return u8s8_saturate_cast(); - } - } else if (v->dtype_ == sc_data_type_t::s32(16)) { - assert(intype == sc_data_type_t::f32(16)); - return builder::make_round_and_cast(inval1, sc_data_type_t::s32(16)); - } - } - auto cast_s32_u8s8 = [intype, outtype](const expr_c &v) { - int64_t max_val = outtype.type_code_ == sc_data_etype::U8 ? 255 : 127; - int64_t min_val = outtype.type_code_ == sc_data_etype::U8 ? 0 : -128; - expr val255 = - make_expr(max_val, sc_data_type_t::s32(intype.lanes_)); - expr val0 = - make_expr(min_val, sc_data_type_t::s32(intype.lanes_)); - // ret = min(v, 255) - auto ret = builder::make_min(v, val255); - // ret = max(ret, 0) - ret = builder::make_max(ret, val0); - return builder::make_cast(outtype, ret); - }; - - COMPILE_ASSERT((v->dtype_.type_code_ == sc_data_etype::S8 || - v->dtype_.type_code_ == sc_data_etype::U8) && - (intype.type_code_ == sc_data_etype::S32 || - intype.type_code_ == sc_data_etype::F32), - "saturated_cast cannot handle: " - << v << '(' << intype << "->" << v->dtype_ << ')'); - expr_c real_in = inval1; - if (intype.type_code_ == sc_data_etype::F32) { - use_avx512_path = false; - return u8s8_saturate_cast(); - } - return cast_s32_u8s8(real_in); - } - - stmt_c visit(for_loop_c v) override { - if (v->kind_ == for_type::PARALLEL) { - parallel_depth_++; - } - auto ret = ir_visitor_t::visit(v); - if (v->kind_ == for_type::PARALLEL) { - parallel_depth_--; - } - return ret; - } - - int get_barrier_id(const node_base *v) { - int barrier_id = -1; - if (v->attr_) { - barrier_id = v->attr_->get_or_else("post_barrier_id", -1); - } - return barrier_id; - } - - expr_c visit(call_c v) override { - auto ret = ir_visitor_t::visit(v); - if (v->func_ == builtin::get_barrier_arrive_func()) { - int bar_id = get_barrier_id(v.get()); - auto itr = barrier_idle_func_args_.find(bar_id); - if (bar_id >= 0 && itr != barrier_idle_func_args_.end()) { - auto ret2 = ret.static_as(); - // pass idle_func and idle_args to barrier func - return copy_attr( - *v, builtin::get_barrier_arrive_func()( - ret2->args_.at(0), itr->second.first, itr->second.second)); - } - } - return ret; - } - expr_c visit(intrin_call_c v) override { - auto ret = ir_visitor_t::visit(v); - if (v->type_ == intrin_type::set_thread_idle_func) { - int barrier_id = -1; - if (parallel_depth_ > 0) { - // get the barrier id of the set_thread_idle_func call, and find - // the barrier call of the same id - barrier_id = get_barrier_id(v.get()); - bool found_consumer = false; - for (auto &seq : cur_scope_->seq_) { - if (seq.isa()) { - auto &val = seq.static_as()->value_; - if (val.isa()) { - auto call_node = val.static_as(); - if (call_node->func_ == builtin::get_barrier_arrive_func()) { - auto call_id = get_barrier_id(call_node.get()); - if (call_id == barrier_id) { - found_consumer = true; - break; - } - } - } - } - } - if (!found_consumer) { - // remove the intrinsic call - return 0; - } - } - std::vector insert_before; - expr args_pack; - - if (v->args_.size() == 2UL) { - args_pack = v->args_[1]; - } else { - args_pack = builder::make_tensor( - std::string("__idle_args") + std::to_string(var_cnt_++), - {v->args_.size() - 1}, datatypes::generic); - auto def = builder::make_var_tensor_def_unattached(args_pack); - insert_before.emplace_back(def); - def->attr()[attr_keys::tsr_dont_buf_sched] = true; - - for (size_t i = 1; i < v->args_.size(); i++) { - insert_before.emplace_back(builder::make_assign_unattached( - args_pack[i - 1], - builder::make_cast(datatypes::generic, v->args_[i]))); - } - insert_seq_before_ = std::move(insert_before); - } - if (parallel_depth_ > 0) { - // inform the barrier call to set the idle - // functions - barrier_idle_func_args_[barrier_id] = {v->args_[0], args_pack}; - // remove the intrinsic call. The idle function will be passed - // in barrier call - return 0; - } - if (runtime_config_t::get().managed_thread_pool_ != - thread_pool_mode_t::MANAGED) { - return 0; - } - return builtin::get_set_idle_func_managed_func()(v->args_[0], args_pack); - } else if (v->type_ == intrin_type::get_group_thread_id) { - int64_t level_id = get_const_as_int(v->args_[0].checked_as()); - COMPILE_ASSERT(level_id < 0, - "get_group_thread_id is used outside of nested " - "parallel. It must has group level -1."); - return builtin::get_thread_id_func()(); - } else if (v->type_ == intrin_type::get_group_id) { - throw std::runtime_error( - "get_group_id cannot be used outside of nested parallel."); - return expr(); - } -#ifndef NDEBUG - else if (v->type_ == intrin_type::min || v->type_ == intrin_type::max) { - COMPILE_ASSERT(v->args_.size() == 2, - "Invalid arg size: " << v->args_.size() - << ", should be 2"); - auto l = ret.checked_as()->args_[0]; - auto r = ret.checked_as()->args_[1]; - if (!l.isa() && r.isa() && - get_etype_category_nothrow(r->dtype_) == type_category::CATE_FLOAT) { - // min(x, c) => min(c, x) - // max(x, c) => max(c, x) - return builder::remake_binary(r, l, v); - } - } -#endif - intrin_func_creator lower_func = nullptr; - intrin_func_namer namer_func = nullptr; - switch (v->type_) { - case intrin_type::exp: - COMPILE_ASSERT(ret->dtype_.is_etype(sc_data_etype::F32), - "Currently sc_exp only supports f32"); - lower_func = &create_exp_func; - namer_func = &get_exp_func_name; - break; - case intrin_type::isnan: - COMPILE_ASSERT(v->args_[0]->dtype_.is_etype(sc_data_etype::F32), - "Currently sc_isnan only supports f32"); - lower_func = &create_isnan_func; - namer_func = &get_isnan_func_name; - break; - case intrin_type::log: - COMPILE_ASSERT(v->args_[0]->dtype_.is_etype(sc_data_etype::F32), - "Currently sc_log only supports f32"); - lower_func = &create_log_func; - namer_func = &get_log_func_name; - break; - case intrin_type::erf: - COMPILE_ASSERT(v->args_[0]->dtype_.is_etype(sc_data_etype::F32), - "Currently erf only supports f32"); - lower_func = &create_erf_func; - namer_func = &get_erf_func_name; - break; - case intrin_type::saturated_cast: - return do_lower_saturated_cast(ret.checked_as()); - break; - case intrin_type::read_struct: - COMPILE_ASSERT(v->args_[0]->dtype_.get_pointer_element() == datatypes::u8, - "User defined struct tensor should be a u8 tensor in " - "IR."); - lower_func = &create_read_struct_func; - namer_func = &get_read_struct_func_name; - break; - case intrin_type::write_struct: - COMPILE_ASSERT(v->args_[0]->dtype_.get_pointer_element() == datatypes::u8, - "User defined struct tensor should be a u8 tensor in " - "IR."); - lower_func = &create_write_struct_func; - namer_func = &get_write_struct_func_name; - break; - - default: - break; - } - if (lower_func) { - auto new_args = - visit_need_def_args(ret.checked_as()->args_); - func_t f = mod_->get_func(namer_func(v)); - if (f) { - // if the function is found, check the signature - const char *prompt = "Bad signature found for intrinsic implementation " - "function"; - COMPILE_ASSERT(f->ret_type_ == v->dtype_ && - f->params_.size() == v->args_.size(), - prompt << f); - for (size_t i = 0; i < f->params_.size(); i++) { - COMPILE_ASSERT(f->params_[i]->dtype_ == v->args_[i]->dtype_ || - (f->params_[i]->dtype_ == datatypes::pointer && - v->args_[i]->dtype_.is_pointer()), - prompt << f); - } - } else { - f = lower_func(mod_, ret.checked_as(), this); - // private function, so that hopefully it can be removed - // after inlined - f->attr()[function_attrs::private_] = true; - mod_->add_func({f}); - } - auto r = copy_attr(*ret, builder::make_call(f, new_args)); - r->attr()["inline_level"] = 2; - return r; - } - return ret; - } - - expr_c visit(cast_c v) override { - auto ret = ir_visitor_t::visit(v).checked_as().remove_const(); - if (ret->in_->dtype_.is_etype(sc_data_etype::BF16) || - ret->in_->dtype_.is_etype(sc_data_etype::F16)) { - expr new_in = visit_need_def_args({ret->in_})[0].remove_const(); - if (!new_in.ptr_same(ret->in_)) { - ret = ret->remake().static_as(); - } - ret->in_ = new_in; - return ret->in_->dtype_.is_etype(sc_data_etype::BF16) - ? create_cast_bf16_to_f32(ctx_, ret) - : create_cast_fp16_to_f32(ctx_, ret); - } else if (ret->dtype_.is_etype(sc_data_etype::BF16) || - ret->dtype_.is_etype(sc_data_etype::F16)) { - expr new_in = visit_need_def_args({ret->in_})[0].remove_const(); - if (!new_in.ptr_same(ret->in_)) { - ret = ret->remake().static_as(); - } - ret->in_ = new_in; - return ret->dtype_.is_etype(sc_data_etype::BF16) - ? create_cast_f32_to_bf16(ctx_, ret) - : create_cast_f32_to_fp16(ctx_, ret); - } - - return ret; - } - - std::vector insert_seq_before_; - stmt_c visit(stmts_c v) override { - auto old_scope = cur_scope_; - cur_scope_ = v.get(); - bool changed = false; - need_defs_.emplace_back(); - std::vector seqs; - size_t def_sz = 0; - for (auto &st : v->seq_) { - auto new_st = dispatch(st); - auto &cur_defs = need_defs_.back(); - changed |= !new_st.ptr_same(st); - if (cur_defs.size() != def_sz) { - assert(cur_defs.size() > def_sz); - for (size_t i = def_sz; i < cur_defs.size(); i++) { - expr_c &cache_var = cur_defs[i].first; - seqs.emplace_back(builder::make_var_tensor_def_unattached( - cache_var, linkage::local, cur_defs[i].second)); - } - def_sz = cur_defs.size(); - } - for (auto &insert : insert_seq_before_) { - seqs.emplace_back(std::move(insert)); - } - insert_seq_before_.clear(); - seqs.emplace_back(new_st); - } - need_defs_.pop_back(); - changed |= seqs.size() != v->seq_.size(); - if (changed) { - stmt newv = copy_attr(*v, builder::make_stmts_unattached(seqs)); - cur_scope_ = old_scope; - return std::move(newv); - } - cur_scope_ = old_scope; - return v; - } - - target_specific_lower_cpu_impl_t(context_ptr ctx, const ir_module_ptr &m) - : ctx_(std::move(ctx)), mod_(m) {} -}; - -static func_t create_log_func(const ir_module_ptr &mod, - const intrin_call_c &node, - target_specific_lower_cpu_impl_t *visitor) { - auto type = node->dtype_; - uint32_t elements = type.lanes_; - - auto ZERO = gen_vec_const(elements, 0.0f); - auto ln2 = gen_vec_const(elements, 0.693147181f); - auto ONE_f = gen_vec_const(elements, 1.0f); - auto ONE_i = gen_vec_const_int(elements, 1); - auto ty_epi_32 = sc_data_type_t::s32(elements); - auto neg_inf_f = - gen_vec_const(elements, -std::numeric_limits::infinity()); - auto inf_f = gen_vec_const(elements, std::numeric_limits::infinity()); - auto qnan_f = - gen_vec_const(elements, std::numeric_limits::quiet_NaN()); - - builder::ir_builder_t builder; - std::string log_const_table_name_1 = "log_const_table_1", - log_const_table_name_2 = "log_const_table_2"; - auto global_log_consts_1 = - mod->get_var_def_from_symbol(log_const_table_name_1); - expr log_table_tsr_1, log_table_tsr_2; - if (!global_log_consts_1.defined()) { - log_table_tsr_1 = builder::make_tensor( - log_const_table_name_1, {32}, datatypes::f32, address_space::automatic, - std::make_shared(sc_log_const_int_vals_1, - sizeof(sc_log_const_int_vals_1))); - log_table_tsr_2 = builder::make_tensor( - log_const_table_name_2, {32}, datatypes::f32, address_space::automatic, - std::make_shared(sc_log_const_int_vals_2, - sizeof(sc_log_const_int_vals_2))); - log_table_tsr_1->attr().set(attr_keys::static_global, - (void *)sc_log_const_int_vals_1); - log_table_tsr_2->attr().set(attr_keys::static_global, - (void *)sc_log_const_int_vals_2); - mod->add_global_var(builder::make_var_tensor_def_unattached( - log_table_tsr_1, linkage::public_global) - .checked_as()); - mod->add_global_var(builder::make_var_tensor_def_unattached( - log_table_tsr_2, linkage::public_global) - .checked_as()); - } else { - log_table_tsr_1 = global_log_consts_1->var_; - log_table_tsr_2 = - mod->get_var_def_from_symbol(log_const_table_name_2)->var_; - } - const int approx_bits = 5; - const int mantissa_bits = 23; - _function_(type, the_log_func, make_args_by_intrinsic(node)) { - assert(node->args_.size() == 1); - // to avoid underflow - _bind_(inval); - _var_init_(inval_int, ty_epi_32, - builder::make_reinterpret(inval, ty_epi_32)); - _var_(aux1_int, ty_epi_32); - aux1_int = ((inval_int >> - gen_vec_const_int(elements, mantissa_bits - approx_bits)) & - gen_vec_const_int(elements, 0x0000001f)); - _var_init_(aux2_int, ty_epi_32, - aux1_int >> gen_vec_const_int(elements, approx_bits - 1)); - _var_init_(aux3_int, ty_epi_32, - (inval_int >> gen_vec_const_int(elements, mantissa_bits)) + - aux2_int); - _var_init_(aux3_f, type, builder::make_cast(type, aux3_int)); - aux2_int = - builder::make_int_xor(aux2_int, gen_vec_const_int(elements, 0x0000007f)) - << gen_vec_const_int(elements, 23); - inval_int = - (inval_int & gen_vec_const_int(elements, 0x007fffff)) | aux2_int; - _var_init_(aux2_f, type, builder::make_gather(log_table_tsr_1, aux1_int)); - aux2_f = aux2_f * builder::make_reinterpret(inval_int, type) - ONE_f; - _var_init_(poly_f, type, gen_vec_const(elements, 0.199984118f)); - poly_f = builder::make_fmadd(poly_f, aux2_f, - gen_vec_const(elements, -0.250035613f)); - poly_f = builder::make_fmadd(poly_f, aux2_f, - gen_vec_const(elements, 0.333333343f)); - poly_f = - builder::make_fmadd(poly_f, aux2_f, gen_vec_const(elements, -0.5f)); - poly_f = builder::make_fmadd(poly_f, aux2_f, ONE_f) * aux2_f; - aux2_f = builder::make_gather(log_table_tsr_2, aux1_int); - aux2_f = builder::make_fmadd(aux3_f, ln2, aux2_f); - // two sum algorithm - _var_init_(res_hi, type, poly_f + aux2_f); - _var_init_(res_lo, type, res_hi - aux2_f); - res_lo = res_lo - poly_f; - res_hi = res_hi + res_lo; - res_hi = builder::make_select(inval == ZERO, neg_inf_f, res_hi); - res_hi = builder::make_select(inval < ZERO, qnan_f, res_hi); - res_hi = builder::make_select(inval == inf_f, inf_f, res_hi); - auto isnan = visitor->dispatch(builder::make_isnan(inval)); - res_hi = builder::make_select(isnan, qnan_f, res_hi); - res_hi = builder::make_select(inval == ONE_f, ZERO, res_hi); - _return_(res_hi); - } - std::string fixed_name = get_log_func_name(node); - the_log_func->name_ = fixed_name; - the_log_func->decl_->name_ = fixed_name; - return the_log_func; -} - -const_ir_module_ptr -target_specific_lowering_cpu_t::operator()(const_ir_module_ptr m) { - auto ret = m->copy(); - target_specific_lower_cpu_impl_t pass{ctx_, ret}; - auto &contents = ret->get_contents(); - auto sz = contents.size(); - for (size_t i = 0; i < sz; i++) { - auto f = std::const_pointer_cast(pass.dispatch(contents[i])); - contents[i] = std::move(f); - pass.barrier_idle_func_args_.clear(); - } - return ret; -} - -static func_t create_erf_func(const ir_module_ptr &mod, - const intrin_call_c &node, - target_specific_lower_cpu_impl_t *visitor) { - auto type = node->dtype_; - uint32_t elements = type.lanes_; - - auto ONE_i = gen_vec_const_int(elements, 1); - auto ONE_f = gen_vec_const(elements, 1.0f); - expr ZERO_f = make_expr(0.0f, sc_data_type_t::f32(elements)); - auto half_f = gen_vec_const(elements, 0.5f); - auto INT_23 = gen_vec_const_int(elements, 23); - auto INT_24 = gen_vec_const_int(elements, 24); - auto positive_mask = gen_vec_const_int(elements, 0x7fffffff); - auto sign_mask = gen_vec_const_int(elements, 0x80000000); - auto index_bias = gen_vec_const_int(elements, 0xc21fffff); - union { - float v; - int v2; - } caster; - caster.v2 = 0x40b15cee; - auto rbound = gen_vec_const(elements, caster.v); - caster.v2 = 0x3FB504F3; - auto sqrt_2 = gen_vec_const(elements, caster.v); - auto ty_epi_32 = sc_data_type_t::s32(elements); - - builder::ir_builder_t builder; - if (mod->ctx_->machine_.cpu_flags_.fAVX512F && elements >= 4) { - std::vector erf_const_table_names = { - "erf_const_table_0", "erf_const_table_1", "erf_const_table_2", - "erf_const_table_3", "erf_const_table_4", "erf_const_table_5"}; - auto global_erf_const_0 = - mod->get_var_def_from_symbol(erf_const_table_names[0]); - std::vector erf_table_tsr_list(6); - if (!global_erf_const_0.defined()) { - for (int i = 0; i < 6; i++) { - erf_table_tsr_list[i] = builder::make_tensor( - erf_const_table_names[i], {32}, datatypes::f32, - address_space::automatic, - std::make_shared(sc_erf_const_int_vals[i], - sizeof(uint32_t) * 32)); - erf_table_tsr_list[i]->attr().set(attr_keys::static_global, - (void *)sc_erf_const_int_vals[i]); - mod->add_global_var(builder::make_var_tensor_def_unattached( - erf_table_tsr_list[i], linkage::public_global) - .checked_as()); - } - } else { - erf_table_tsr_list[0] = global_erf_const_0->var_; - for (int i = 1; i < 6; i++) { - erf_table_tsr_list[i] = - mod->get_var_def_from_symbol(erf_const_table_names[i])->var_; - } - } - _function_(type, the_erf_func, make_args_by_intrinsic(node)) { - assert(node->args_.size() == 1); - // to avoid underflow - _bind_(inval); - inval = inval * sqrt_2; - auto in_s32 = builder::make_reinterpret(inval, ty_epi_32); - _var_init_(src_pos_s32, ty_epi_32, in_s32 & positive_mask); - _var_init_(src_pos, inval->dtype_, - builder::make_reinterpret(src_pos_s32, inval->dtype_)); - _var_init_(indices, ty_epi_32, - builder::make_min( - builder::make_max((src_pos_s32 + index_bias) >> - gen_vec_const_int(elements, 21), - ONE_i), - INT_24)); - indices = builder::make_select(src_pos > rbound, INT_23, indices); - _var_init_(pol, inval->dtype_, - builder::make_indexing(erf_table_tsr_list[5], {0}, elements)); - pol = builder::make_permutex2var( - pol, indices, - builder::make_indexing(erf_table_tsr_list[5], {16}, elements)); - _var_(tmp, inval->dtype_); - for (int i = 4; i >= 0; i--) { - tmp = builder::make_indexing(erf_table_tsr_list[i], {0}, elements); - tmp = builder::make_permutex2var( - tmp, indices, - builder::make_indexing(erf_table_tsr_list[i], {16}, elements)); - pol = builder::make_fmadd(pol, src_pos, tmp); - } - auto tmp2 = in_s32 & sign_mask; - pol = builder::make_reinterpret( - builder::make_reinterpret(pol, ty_epi_32) ^ tmp2, inval->dtype_); - _return_(pol); - } - std::string fixed_name = get_erf_func_name(node); - the_erf_func->name_ = fixed_name; - the_erf_func->decl_->name_ = fixed_name; - return the_erf_func; - } else { - expr const_a1 = - make_expr(0.254829592f, sc_data_type_t::f32(elements)); - expr const_a2 = - make_expr(-0.284496736f, sc_data_type_t::f32(elements)); - expr const_a3 = - make_expr(1.421413741f, sc_data_type_t::f32(elements)); - expr const_a4 = - make_expr(-1.453152027f, sc_data_type_t::f32(elements)); - expr const_a5 = - make_expr(1.061405429f, sc_data_type_t::f32(elements)); - expr const_p = - make_expr(0.3275911f, sc_data_type_t::f32(elements)); - _function_(type, the_erf_func, make_args_by_intrinsic(node)) { - assert(node->args_.size() == 1); - // to avoid underflow - _bind_(inval); - _var_init_(temp, sc_data_type_t::f32(elements), builder::make_abs(inval)); - _var_init_(neg_square, sc_data_type_t::f32(elements), - ZERO_f - inval * inval); - _var_init_(Q, sc_data_type_t::f32(elements), - ZERO_f - visitor->dispatch(builder::make_exp(neg_square))); - _var_init_( - t, sc_data_type_t::f32(elements), - builder::make_div(ONE_f, builder::make_fmadd(const_p, temp, ONE_f))); - _var_init_(result, sc_data_type_t::f32(elements), const_a5); - _var_init_( - sign, sc_data_type_t::s32(elements), - builder::make_int_and( - builder::make_reinterpret(inval, sc_data_type_t::s32(elements)), - sign_mask)); - temp = builder::make_mul(Q, t); - result = builder::make_fmadd(result, t, const_a4); - result = builder::make_fmadd(result, t, const_a3); - result = builder::make_fmadd(result, t, const_a2); - result = builder::make_fmadd(result, t, const_a1); - result = builder::make_fmadd(result, temp, ONE_f); - result = builder::make_reinterpret( - builder::make_int_xor( - sign, - builder::make_reinterpret(result, sc_data_type_t::s32(elements))), - sc_data_type_t::f32(elements)); - _return_(result); - } - std::string fixed_name = get_erf_func_name(node); - the_erf_func->name_ = fixed_name; - the_erf_func->decl_->name_ = fixed_name; - return the_erf_func; - } -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/transform/cpu/target_specific_lower.hpp b/legacy/core/src/compiler/ir/transform/cpu/target_specific_lower.hpp deleted file mode 100644 index 7fc127632..000000000 --- a/legacy/core/src/compiler/ir/transform/cpu/target_specific_lower.hpp +++ /dev/null @@ -1,43 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_CPU_TARGET_SPECIFIC_LOWER_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_CPU_TARGET_SPECIFIC_LOWER_HPP - -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -/** - * Lower the target-specific IRs. - * */ -class target_specific_lowering_cpu_t : public module_pass_t { -public: - context_ptr ctx_; - target_specific_lowering_cpu_t(context_ptr ctx) : ctx_(ctx) {} - const_ir_module_ptr operator()(const_ir_module_ptr m) override; - SC_DECL_PASS_INFO_FUNC(); -}; - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/transform/dead_func_eliminate.cpp b/legacy/core/src/compiler/ir/transform/dead_func_eliminate.cpp deleted file mode 100644 index 81e1c2c6c..000000000 --- a/legacy/core/src/compiler/ir/transform/dead_func_eliminate.cpp +++ /dev/null @@ -1,118 +0,0 @@ -/******************************************************************************* - * Copyright 2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#include "dead_func_eliminate.hpp" -#include -#include -#include -#include -#include -#include -#include - -SC_MODULE(pass.dead_func_elim); - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -SC_DECL_PASS_INFO(dead_func_eliminate, SC_PASS_DEPENDS_ON(), - SC_PASS_REQUIRE_STATE(), SC_PASS_REQUIRE_NOT_STATE(), - SC_PASS_SET_STATE(), SC_PASS_UNSET_STATE()); - -class func_reference_finder_t : public ir_viewer_t { -public: - using ir_viewer_t::dispatch; - // use pointer to workaround a gcc4 bug - const ir_module_t *mod_; - std::unordered_set> *funcs_; - std::vector> *new_found_; - - func_reference_finder_t(const ir_module_t &mod, - std::unordered_set> &funcs, - std::vector> &new_found) - : mod_{&mod}, funcs_(&funcs), new_found_(&new_found) {} - - void view(call_c v) override { - ir_viewer_t::view(v); - auto funct = std::dynamic_pointer_cast(v->func_); - if (funct) { - auto funcdef = mod_->get_func(funct->name_); - if (funcdef) { - bool is_inserted = funcs_->insert(funcdef).second; - if (is_inserted) { - new_found_->emplace_back(funcdef); - } - } - } - } - - void view(func_addr_c v) override { - ir_viewer_t::view(v); - auto funcdef = mod_->get_func(v->func_->name_); - if (funcdef) { - bool is_inserted = funcs_->insert(funcdef).second; - if (is_inserted) { - new_found_->emplace_back(funcdef); - } - } - } -}; - -const_ir_module_ptr dead_func_eliminate_t::operator()(const_ir_module_ptr f) { - std::vector> gc_roots; - for (auto &func : f->get_contents()) { - if (!func->attr_ || - !func->attr_->get_or_else(function_attrs::private_, false)) { - gc_roots.emplace_back(func); - } - } - std::unordered_set> met{gc_roots.begin(), - gc_roots.end()}; - if (auto mainf = f->get_entry_func()) { - if (met.count(mainf) == 0) { - gc_roots.emplace_back(mainf); - met.insert(mainf); - } - } - while (!gc_roots.empty()) { - std::vector> new_found; - func_reference_finder_t finder{*f, met, new_found}; - for (auto &func : gc_roots) { - finder.dispatch(func); - } - gc_roots = std::move(new_found); - } - if (met.size() == f->get_contents().size()) { - return f; - } - std::vector mask; - mask.reserve(f->get_contents().size()); - for (auto &func : f->get_contents()) { - auto keep = - met.count(func) != 0 || - (func->attr_ && func->attr_->get_or_else(attr_keys::keep_func, false)); - mask.push_back(keep); - if (!keep) { - SC_MODULE_INFO << "removing func " << func->name_; - } - } - return f->copy_and_remove_funcs(mask); -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/transform/dead_func_eliminate.hpp b/legacy/core/src/compiler/ir/transform/dead_func_eliminate.hpp deleted file mode 100644 index 12c56b25c..000000000 --- a/legacy/core/src/compiler/ir/transform/dead_func_eliminate.hpp +++ /dev/null @@ -1,41 +0,0 @@ -/******************************************************************************* - * Copyright 2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_DEAD_FUNC_ELIMINATE_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_DEAD_FUNC_ELIMINATE_HPP - -#include "../module_pass.hpp" -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace attr_keys { -constexpr const char *keep_func = "pass.keep_func"; -} -class dead_func_eliminate_t : public module_pass_t { -public: - const_ir_module_ptr operator()(const_ir_module_ptr f) override; - SC_DECL_PASS_INFO_FUNC(); -}; - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/transform/dead_write_eliminate.cpp b/legacy/core/src/compiler/ir/transform/dead_write_eliminate.cpp deleted file mode 100644 index 9ef69accd..000000000 --- a/legacy/core/src/compiler/ir/transform/dead_write_eliminate.cpp +++ /dev/null @@ -1,98 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#include "dead_write_eliminate.hpp" -#include "../visitor.hpp" -#include "buffer_schedule.hpp" -#include -#include -#include -#include -#include -#include -#include - -SC_MODULE(pass.dead_write_elim) - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -SC_DECL_PASS_INFO(dead_write_eliminator, - SC_PASS_DEPENDS_ON(validator, index2var, tensor_inplace), - // need to remove redundant memory store for index2var - // need pointer alias info - SC_PASS_REQUIRE_STATE(CONST_FOLDED, IR_SIMPLIFIED), - SC_PASS_REQUIRE_NOT_STATE(), SC_PASS_SET_STATE(), - SC_PASS_UNSET_STATE(IR_SIMPLIFIED)); - -// dead write elimination implementation -class dwe_impl_t : public ir_visitor_t { -public: - using ir_visitor_t::dispatch; - using ir_visitor_t::visit; - std::unordered_set defined; - - // not interested in any exprs - expr_c dispatch(expr_c v) override { return v; } - - stmt_c visit(define_c v) override { - defined.insert(v->var_); - return ir_visitor_t::visit(v); - } - - stmt_c visit(assign_c v) override { - if (dependency_analysis::get_dep_info(v.get()).depended_by_.size() == 0 && - v->var_.isa()) { - auto tsr = v->var_.static_as()->ptr_.checked_as(); - bool no_dead_write = - tsr->attr_ && - (tsr->attr_->get_or_else(attr_keys::no_dead_write, false) || - tsr->attr_->get_or_else(dependency_analysis::attr_directly_accessed, - false)); - bool is_pointer_buffer = tsr->elem_dtype_.is_pointer(); - bool is_read_buffer = tsr->attr().get_or_else("read_buffer", false); - bool is_tensor_local = - (is_read_buffer || defined.find(tsr) != defined.end()) && - !is_pointer_buffer && !no_dead_write; - - // if it is a store to indexing, and the target tensor is local, and - // no other stmt depends on it, we can delete it - if (is_tensor_local) { - SC_MODULE_INFO << "Remove " << v; - if (is_read_buffer) { - SC_MODULE_WARN << "Writing on read-only buffer: " << v; - } - return make_stmt(std::vector()); - } - } - return v; - } -}; - -func_c dead_write_eliminator_t::operator()(func_c f) { - if (f->attr_ && f->attr_->get_or_else(attr_keys::already_buf_sched, false)) { - return f; - } - dependency_analyzer_t ana; - f = ana(f); - dwe_impl_t v; - return v.dispatch(f); -} -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/transform/dead_write_eliminate.hpp b/legacy/core/src/compiler/ir/transform/dead_write_eliminate.hpp deleted file mode 100644 index f02faab92..000000000 --- a/legacy/core/src/compiler/ir/transform/dead_write_eliminate.hpp +++ /dev/null @@ -1,44 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_DEAD_WRITE_ELIMINATE_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_DEAD_WRITE_ELIMINATE_HPP - -#include "../function_pass.hpp" - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace attr_keys { -constexpr const char *no_dead_write = "pass.no_dead_write"; -} -/** - * Removes unread writes on local var (this feature is not done yet) and local - * tensors - * */ -class dead_write_eliminator_t : public function_pass_t { -public: - func_c operator()(func_c f) override; - SC_DECL_PASS_INFO_FUNC(); -}; - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/transform/dessa_transform.cpp b/legacy/core/src/compiler/ir/transform/dessa_transform.cpp deleted file mode 100644 index e7ded877a..000000000 --- a/legacy/core/src/compiler/ir/transform/dessa_transform.cpp +++ /dev/null @@ -1,667 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#include "dessa_transform.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -SC_DECL_PASS_INFO(dessa_transform, SC_PASS_DEPENDS_ON(ssa_transform), - SC_PASS_REQUIRE_STATE(SSA_STAGE), SC_PASS_REQUIRE_NOT_STATE(), - SC_PASS_SET_STATE(), SC_PASS_UNSET_STATE(SSA_STAGE)); - -struct dessa_analysis_data_t { - std::vector uses_; - const stmt_base_t *parent_; - - // if this var-define node should be removed - bool should_remove_ = false; - // the stmt to be inserted at the begining of the current stmts. If the - // current stmt is not stmts, it should be empty - std::vector inserted_within_; - // the stmt to be inserted after the current stmts. - std::vector inserted_after_; - // the copy for the var for loop-phi to solve lost-copy and swap problem - expr shadow_phi_var_; - - dessa_analysis_data_t(const stmt_base_t *parent) : parent_(parent) {} -}; - -// Loop phi coalesce is achieved by checking the usage of a, b, c -// in a = phi(b, c loop) is not overlapped and there is no interference. -// ssa_analysis_viewer_t will set to check b, c usage after a is defined -// and check a, b usage after c is defined, so if there is interference -// between any vars in any sub loops, all vars will not be coalesced. -// This will coalesce all loop phi vars in reduce situations -struct loop_phi_coalesce_data_t { - // check for interference after this var is defined - std::weak_ptr check_after_define_; - // the final coalesced var - std::weak_ptr coalesced_var_; - // flag for var defined inside loop - bool from_loop_ = false; - // flag for interfered phi vars - bool interfered_ = false; - // flag for checking interference - // when check_ is true and the var is used - // interfered_ will be set to true - bool check_ = false; - // flag for preserved var, a copy has been made - // original var can be used in other node - bool preserved_ = false; - - loop_phi_coalesce_data_t(const expr &v = expr()) - : check_after_define_(v.impl) {} -}; - -// get loop_phi_coalesce_data_t or null -static inline loop_phi_coalesce_data_t *get_coalesce_data(const expr_c &v) { - if (!v->get_temp_data().isa()) { // - return nullptr; - } - return &(v->temp_data().get()); -} -// get loop_phi_coalesce_data_t or create new -static inline loop_phi_coalesce_data_t * -get_or_create_coalesce_data(const expr_c &v, const expr &c = expr()) { - if (!v->get_temp_data().isa()) { - v->temp_data() = loop_phi_coalesce_data_t(c); - } - return &(v->temp_data().get()); -} - -// Merge related loop phi var interfered data, -// if one interfered, mark all interfered -static inline bool merge_interfered_data(const expr_c &var, - const ssa_phi_c &phi) { - if (!get_coalesce_data(var)) { - return true; - } - bool interfered = get_coalesce_data(var)->interfered_; - for (auto &val : phi->values_) { - if (!get_coalesce_data(val)) { - return true; - } - interfered |= get_coalesce_data(val)->interfered_; - } - get_coalesce_data(var)->interfered_ = interfered; - for (auto &val : phi->values_) { - get_coalesce_data(val)->interfered_ = interfered; - } - return interfered; -} - -// Get final coalesced var -static inline expr get_coalesced_from_var(const expr_c &v) { - if (get_coalesce_data(v)) { - auto ptr = get_coalesce_data(v)->coalesced_var_; - if (!utils::is_uninitialized_weakptr(ptr)) { - assert(ptr.lock()); - return ptr.lock()->node_ptr_from_this(); - } - } - return expr(); -} -// Get if the coalesced var is preserved -static inline bool get_perservd_from_var(const expr_c &v) { - return get_coalesce_data(v) ? get_coalesce_data(v)->preserved_ : false; -} -// Get final coalesced var from phi node, if incoming coalesced ptr_same -static inline expr get_coalesced_from_phi(const ssa_phi_c &phi) { - assert(phi->values_.size() == 2); - auto c0 = get_coalesced_from_var(phi->values_[0]); - auto c1 = get_coalesced_from_var(phi->values_[1]); - return c0.defined() && c1.defined() && c0.ptr_same(c1) ? c0 : expr(); -} -// Merge final coalesced var, set to the first incoming phi var -static inline void merge_coalesced_var(const expr_c &thevar, - const ssa_phi_c &phi, - const stmt &toplevel) { - assert(phi->values_.size() == 2); - auto &val_0 = phi->values_[0]; - auto &val_1 = phi->values_[1]; - - expr coalesced = get_coalesced_from_var(val_0); - - if (!coalesced.defined()) { - // perserve the outer loop defined var - get_coalesce_data(val_0)->preserved_ = true; - // make a copy of outer loop var - coalesced = thevar->remake(); - coalesced.static_as()->name_ += "_coalesced"; - // insert define after original var - auto coalesced_def = builder::make_var_tensor_def_unattached( - coalesced, linkage::local, val_0); - auto valowner = val_0->ssa_data_->get_owner(); - if (valowner.defined()) { - auto &tempdata = valowner->temp_data().get(); - tempdata.inserted_after_.emplace_back(coalesced_def); - } else { - auto &tempdata = toplevel->temp_data().get(); - tempdata.inserted_within_.emplace_back(coalesced_def); - } - } - - get_coalesce_data(thevar)->coalesced_var_ = coalesced.impl; - get_coalesce_data(val_0)->coalesced_var_ = coalesced.impl; - get_coalesce_data(val_1)->coalesced_var_ = coalesced.impl; -} - -struct ssa_analysis_viewer_t : public ssa_viewer_t { - using ssa_viewer_t::dispatch; - using ssa_viewer_t::view; - const stmt_base_t *current_ = nullptr; - - std::vector defines_; - // all the phi node defined in the current loop - std::vector>> curr_loop_phi_; - // restore parent loop's phi vars check state after curr loop - std::vector>> curr_check_state_; - - void process_define(const expr &var, const expr &init = expr()) { - // process var data for interference check - if (get_coalesce_data(var)) { - auto data = get_coalesce_data(var); - auto ptr = data->check_after_define_; - if (!utils::is_uninitialized_weakptr(ptr)) { - assert(ptr.lock()); - auto node = ptr.lock()->node_ptr_from_this(); - // set to check interference for merged node - get_coalesce_data(node)->check_ = true; - get_coalesce_data(var)->check_ = false; - } - } else { - var->temp_data() = loop_phi_coalesce_data_t(); - } - // If the var is defined not inside any loop, a copy will - // be made when coalescing, and orignal var is preserved - bool from_loop = !curr_loop_phi_.empty(); - get_coalesce_data(var)->from_loop_ = from_loop; - // process loop phi define - if (init.defined() && init.isa()) { - auto phi = init.static_as(); - if (phi->is_loop_phi_) { - assert(phi->values_.size() == 2); - auto &phi_val_0 = phi->values_[0]; - auto &phi_val_1 = phi->values_[1]; - if (get_coalesce_data(phi_val_0) && get_coalesce_data(phi_val_1)) { - // critical loop var defined before current phi var - // exist interference (potentially swap problem) - // may need parallel copy implementation to resolve - get_coalesce_data(var)->interfered_ = true; - get_coalesce_data(phi_val_0)->interfered_ = true; - get_coalesce_data(phi_val_1)->interfered_ = true; - } else if (phi_val_0.isa()) { - // TODO(longsheng): deal with uninitialized vars - get_or_create_coalesce_data(var)->interfered_ = true; - get_or_create_coalesce_data(phi_val_0)->interfered_ = true; - get_or_create_coalesce_data(phi_val_1)->interfered_ = true; - } else { - // assume phi_val_0 is the incoming var - // assume phi_val_1 is the critical var - // save incoming phi check state - COMPILE_ASSERT(get_coalesce_data(phi_val_0), - "incoming phi var should be defined already."); - auto &curr_state = curr_check_state_.back(); - bool check = get_coalesce_data(phi_val_0)->check_; - curr_state.emplace_back(std::make_pair(phi_val_0, check)); - // set to check interference for incoming phi vars - // and critical phi vars - bool v0_check = get_coalesce_data(phi_val_0)->from_loop_; - get_coalesce_data(phi_val_0)->check_ = v0_check; - phi_val_1->temp_data() = loop_phi_coalesce_data_t(var); - get_coalesce_data(phi_val_1)->check_ = true; - } - // record loop phi in current loop - curr_loop_phi_.back().emplace_back(std::make_pair(var, phi)); - } else { - // if phi merged loop phi vars inherent the interferece state - merge_interfered_data(var, phi); - if (phi->values_.size() == 2) { - auto v0_data = get_coalesce_data(phi->values_[0]); - auto v1_data = get_coalesce_data(phi->values_[1]); - bool v0_interfered = v0_data ? v0_data->interfered_ : true; - bool v1_interfered = v1_data ? v1_data->interfered_ : true; - bool interfered = v0_interfered || v1_interfered; - bool v0_check = v0_data && v0_data->from_loop_; - if (v0_data && !interfered) { - v0_data->check_ = v0_check; - } - if (v1_data && !interfered) { - v1_data->check_ = true; - } - } - } - } - } - - func_c dispatch(func_c v) override { - for (auto &p : v->params_) { - process_define(p); - } - return ssa_viewer_t::dispatch(std::move(v)); - } - - stmt_c dispatch(stmt_c s) override { - if (!s->get_temp_data().isa()) { - s->temp_data() = dessa_analysis_data_t(current_); - } else { - // if the vardef is used before define, update the parent - auto &dessa_data = s->temp_data().get(); - assert(dessa_data.parent_ == nullptr); - dessa_data.parent_ = current_; - } - auto old = current_; - current_ = s.get(); - auto ret = ssa_viewer_t::dispatch(std::move(s)); - current_ = old; - return ret; - } - - void view(for_loop_c v) override { - curr_loop_phi_.emplace_back(std::vector>()); - curr_check_state_.emplace_back(std::vector>()); - ssa_viewer_t::view(v); - // merge current loop phi var interference - for (auto &kv : curr_loop_phi_.back()) { - merge_interfered_data(kv.first, kv.second); - } - curr_loop_phi_.pop_back(); - // recover incoming phi check state - for (auto &kv : curr_check_state_.back()) { - get_coalesce_data(kv.first)->check_ = kv.second; - } - curr_check_state_.pop_back(); - } - - void view(define_c v) override { - if (v->init_.defined()) { - dispatch(v->init_); - } - process_define(v->var_, v->init_); - defines_.emplace_back(std::move(v)); - } - - void view(var_c v) override { - auto owner = v->ssa_data_->get_owner(); - if (owner.defined()) { - if (auto dessa_data = - owner->temp_data().get_or_null()) { - dessa_data->uses_.emplace_back(current_); - } else { - // if it is a use-before-define, this usually means that this is - // in a phi node in for-loop. We insert a placeholder for the - // var-def - owner->temp_data() = dessa_analysis_data_t(nullptr); - owner->temp_data().get().uses_.emplace_back( - current_); - } - } - if (get_coalesce_data(v) && get_coalesce_data(v)->check_) { - // check for loop phi var interference - get_coalesce_data(v)->interfered_ = true; - } - } -}; - -static var_node *get_var_if_is_define(const stmt_c &s) { - if (s.isa()) { - auto def = s.static_as(); - if (def->var_.isa() && def->var_->ssa_data_) { - // assert(def->var_->ssa_data_); - return def->var_.static_as().get(); - } - } - return nullptr; -} - -struct ssa_mutator_t : public ir_visitor_t { - using ir_visitor_t::dispatch; - std::vector inserted_after; - stmt_c dispatch(stmt_c v) override { - auto &dessa_data = v->get_temp_data().get(); - if (v.isa()) { - assert(dessa_data.inserted_after_.empty()); - } else { - assert(dessa_data.inserted_within_.empty()); - } - if (dessa_data.should_remove_) { - return stmt_c(); - } - return ir_visitor_t::dispatch(std::move(v)); - } - - expr_c visit(var_c v) override { - auto coalesced = get_coalesced_from_var(v); - auto preserved = get_perservd_from_var(v); - if (coalesced.defined() && !preserved) { - return coalesced; - } - return v; - } - - stmt_c visit(define_c v) override { - if (v->var_.isa() && - v->var_.static_as()->dtype_ == datatypes::void_t) { - assert(v->init_.defined()); - return builder::make_evaluate_unattached( - ir_visitor_t::dispatch(v->init_)); - } - return ir_visitor_t::visit(std::move(v)); - } - - stmt_c visit(evaluate_c v) override { - if (v->value_.isa() && - v->value_.static_as()->dtype_ == datatypes::void_t) { - return stmt_c(); - } - return ir_visitor_t::visit(v); - } - - stmt_c visit(stmts_c v) override { - std::vector ret_vec; - bool changed = false; - // if the current stmts has "inserted_within_" - auto &dessa_data = v->get_temp_data().get(); - ret_vec.insert(ret_vec.end(), dessa_data.inserted_within_.begin(), - dessa_data.inserted_within_.end()); - for (auto &s : v->seq_) { - if (auto var_p = get_var_if_is_define(s)) { - if (var_p->ssa_data_->is_garbage()) { - changed = true; - continue; - } - } - auto sz_before = ret_vec.size(); - auto ret = dispatch(s); - - // if an stmt is inserted, the IR is changed - changed |= sz_before != ret_vec.size(); - changed |= !ret.ptr_same(s); - // we allow the return value to be empty, which means we need to - // remove the stmt from parent - if (!ret.defined()) { - changed = true; - } else { - ret_vec.emplace_back(std::move(ret)); - } // insert "inserted_after_" - auto &s_dessa_data = s->get_temp_data().get(); - if (!s_dessa_data.inserted_after_.empty()) { - std::transform( - s_dessa_data.inserted_after_.begin(), - s_dessa_data.inserted_after_.end(), std::back_inserter(ret_vec), - [this](stmt in) { - return ir_visitor_t::dispatch(std::move(in)).remove_const(); - }); - changed = true; - } - } - if (!changed) { - return v; - } else { - return copy_attr(*v, builder::make_stmts_unattached(ret_vec)); - } - } -}; - -static const stmt_base_t *get_parent(const stmt_base_t *cur) { - return cur->get_temp_data().get().parent_; -} - -static bool is_parent_of(const stmt_base_t *parent, const stmt_base_t *other) { - const stmt_base_t *cur = other; - while (cur) { - if (cur == parent) { - return true; - } - cur = get_parent(cur); - } - return false; -} - -static const stmt_base_t *find_parent_stmts(const stmt_base_t *other) { - const stmt_base_t *cur = other; - while (cur) { - if (cur->node_type_ == sc_stmt_type::stmts) { - return cur; - } - cur = get_parent(cur); - } - assert(0 && "bad dessa_analysis_data_t"); - return nullptr; -} - -static const stmt_base_t *find_parent_for(const stmt_base_t *other) { - const stmt_base_t *cur = other; - while (cur) { - if (cur->node_type_ == sc_stmt_type::for_loop) { - return cur; - } - cur = get_parent(cur); - } - return nullptr; -} - -static const stmt_base_t *find_shared_parent(const stmt_base_t *lhs, - const stmt_base_t *other) { - auto cur = lhs; - while (cur) { - if (is_parent_of(cur, other)) { - return cur; - } - cur = get_parent(cur); - } - assert(0 && "bad dessa_analysis_data_t"); - return nullptr; -} - -static const stmt_base_t *find_shared_parent_for_uses(const stmt_base_t *cur) { - const dessa_analysis_data_t &data = - cur->get_temp_data().get(); - for (auto use : data.uses_) { - cur = find_shared_parent(cur, use); - } - return find_parent_stmts(cur); -} - -// If the loop phi is coalesced, use the coalesced var to replace the usage of -// all merged loop phi vars and romove the define. -// For non-loop phis, if the 2 merged vars are both coalesced to same var, -// use the coalesced var to replace the phi node -static void coalesce_var_for_phi(const expr &coalesced, const expr &var_for_phi, - const stmt_base_t *cur) { - dessa_analysis_data_t &data = cur->temp_data().get(); - auto def = cur->node_ptr_from_this().checked_as(); - data.should_remove_ = true; // remove the phi node - - auto coalesced_loop_var = get_coalesced_from_var(var_for_phi); - if (!coalesced_loop_var.defined()) { - // this is a non loop phi merge, use coalesced var as init to define var - auto newnode = def->remake(); - newnode.static_as()->init_ = coalesced; - data.inserted_after_.insert(data.inserted_after_.begin(), newnode); - } else if (!coalesced_loop_var.ptr_same(coalesced)) { - // this is a non loop phi merge, but src and dst are both coalesced, - // use coalesced var as init to assign to other coalesced var - data.inserted_after_.insert( - data.inserted_after_.begin(), - builder::make_assign_unattached(coalesced_loop_var, coalesced)); - } -} - -// in SSA form, a var may be defined in a child scope (e.g. within if-else), but -// is used in parent scope. This function inserts a var def in parent scope and -// replaces the var def in child scope with an assignment to make the IR valid. -// returns true if the var def needs to be moved -static bool promote_scope_for_non_phi_var(const define_c &def, - const expr &value) { - dessa_analysis_data_t &curdata = - def->temp_data().get(); - auto coalesced = get_coalesced_from_var(def->var_); - if (coalesced.defined()) { - // The var is coalesced - curdata.should_remove_ = true; - curdata.inserted_after_.emplace_back( - builder::make_assign_unattached(coalesced, value)); - return false; - } - auto shared_parent = find_shared_parent_for_uses(def.get()); - if (shared_parent != get_parent(def.get())) { - curdata.should_remove_ = true; - if (value->dtype_ == datatypes::void_t) { - if (!value.isa()) { - curdata.inserted_after_.emplace_back( - builder::make_evaluate_unattached(value)); - } - } else { - shared_parent->temp_data() - .get() - .inserted_within_.emplace_back( - builder::make_var_tensor_def_unattached(def->var_, - def->linkage_)); - curdata.inserted_after_.insert( - curdata.inserted_after_.begin(), - builder::make_assign_unattached(def->var_, value)); - } - return true; - } - return false; -} - -// insert copying to phi vars after an input of the phi node is computed. If a -// phi input is defined in IR function parameters, put the copy in the top-level -// stmts -static void insert_value_copy_for_phi(const ssa_phi &phi, - const expr &var_for_phi, - const stmt_base_t *cur, - const stmt &toplevel) { - dessa_analysis_data_t &data = cur->temp_data().get(); - const stmt_base_t *shared_parent = cur; - const expr &the_target_var = - data.shadow_phi_var_.defined() ? data.shadow_phi_var_ : var_for_phi; - // if the phi has an input from parameter - expr init_val_for_param; - for (auto &val : phi->values_) { - auto valowner = val->ssa_data_->get_owner(); - if (!valowner.defined()) { - COMPILE_ASSERT(!init_val_for_param.defined(), - "PHI node can only accept only one parameter input"); - init_val_for_param = val; - valowner = toplevel; - shared_parent = toplevel.get(); - } else { - auto &tempdata = valowner->temp_data().get(); - shared_parent = find_shared_parent(shared_parent, valowner.get()); - tempdata.inserted_after_.emplace_back( - builder::make_assign_unattached(the_target_var, val)); - } - } - shared_parent = - find_shared_parent(shared_parent, find_shared_parent_for_uses(cur)); - shared_parent = find_parent_stmts(shared_parent); - auto &parent_data = shared_parent->temp_data().get(); - parent_data.inserted_within_.emplace_back( - builder::make_var_tensor_def_unattached(var_for_phi, linkage::local, - init_val_for_param)); - if (data.shadow_phi_var_.defined()) { - parent_data.inserted_within_.emplace_back( - builder::make_var_tensor_def_unattached( - data.shadow_phi_var_, linkage::local, init_val_for_param)); - } - data.should_remove_ = true; // remove the phi node -} - -static void process_phi(const ssa_phi &phi, const stmt_base_t *cur, - const stmt &toplevel) { - dessa_analysis_data_t &data = cur->temp_data().get(); - auto def = cur->node_ptr_from_this().checked_as(); - auto thevar = def->var_.checked_as(); - if (phi->values_.size() > 1) { - // first check if this phi depends on a value in for-loop - const stmt_base_t *cur_for = find_parent_for(cur); - bool is_loop_phi = phi->is_loop_phi_; - - if (is_loop_phi) { - COMPILE_ASSERT(cur_for, "Cannot find parent for-loop for loop phi"); - if (!merge_interfered_data(thevar, phi)) { - // No interference between loop_phi vars - merge_coalesced_var(thevar, phi, toplevel); - } else { - // the phi is a loop-phi - data.shadow_phi_var_ = - builder::make_var(thevar->dtype_, thevar->name_ + "_shadow"); - auto cur_for_body = - static_cast(cur_for)->body_; - // copy the value in shadow to old var at the begining of the - // for-loop - cur_for_body->temp_data() - .get() - .inserted_within_.emplace_back( - builder::make_assign_unattached(thevar, data.shadow_phi_var_)); - } - } - expr coalesced = get_coalesced_from_phi(phi); - if (coalesced.defined()) { - coalesce_var_for_phi(coalesced, thevar, cur); - } else { - insert_value_copy_for_phi(phi, thevar, cur, toplevel); - } - } else { - // not-phi - if (!promote_scope_for_non_phi_var(def, phi->values_.front())) { - // if the current scope is OK for the node - data.should_remove_ = true; - auto newnode = def->remake(); - newnode.static_as()->init_ = phi->values_.front(); - data.inserted_after_.emplace_back(std::move(newnode)); - } - } -} - -func_c dessa_transform_t::operator()(func_c f) { - ssa_analysis_viewer_t analyzer; - analyzer.dispatch(f); - for (auto &def : analyzer.defines_) { - if (def->var_.isa() && def->init_.defined()) { - if (def->init_.isa()) { - auto phi = def->init_.static_as(); - process_phi(phi, def.get(), f->body_); - } else { - assert(def->init_.defined()); - promote_scope_for_non_phi_var(def, def->init_); - } - } - } - ssa_mutator_t mutator; - return mutator.dispatch(f); -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/transform/dessa_transform.hpp b/legacy/core/src/compiler/ir/transform/dessa_transform.hpp deleted file mode 100644 index a67a06dec..000000000 --- a/legacy/core/src/compiler/ir/transform/dessa_transform.hpp +++ /dev/null @@ -1,55 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_DESSA_TRANSFORM_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_DESSA_TRANSFORM_HPP - -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -/** - * Converts the IR from SSA (Single-Static-Assignment) form to normal IR for - * codegen. It mainly does two things: - * 1. Move the var define nodes to an appropriate scope which covers all uses. - * Reason: in SSA form, we allow a variable to live across the scopes. e.g. a - * var can be defined in "then" block of if-else and be used after if-else ends. - * We need to move the def of the var to a parent scope to make the IR valid. We - * also need to take the PHI node's inputs into consideration when selecting - * where to put the definition - * 2. Resolve PHI nodes. There are two kinds of PHI nodes, which are 1) PHI in - * for-loops which depends on the value of a var of the previous iteration 2) - * PHI unrelated with loops. If all/none of the PHI node inputs depend on the - * values defined in the current for-loop, the PHI is of kind 1), otherwise it - * is of kind 2). For kind 1), we need a "shadow" copy for the var in the loop - * to solve the problems like "lost copy" or "swap problem". For kind 2), we - * just need to add an assign node for each of the inputs of PHI - * */ -class dessa_transform_t : public function_pass_t { -public: - func_c operator()(func_c f) override; - SC_DECL_PASS_INFO_FUNC(); -}; - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/transform/dyn_tsr_transform.cpp b/legacy/core/src/compiler/ir/transform/dyn_tsr_transform.cpp deleted file mode 100644 index 3fe0d1a31..000000000 --- a/legacy/core/src/compiler/ir/transform/dyn_tsr_transform.cpp +++ /dev/null @@ -1,476 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include -#include -#include -#include -#include - -#include "dyn_tsr_transform.hpp" -#include -#include -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -SC_DECL_PASS_INFO(dyn_tensor_transformer, SC_PASS_DEPENDS_ON(), - SC_PASS_REQUIRE_STATE(), SC_PASS_REQUIRE_NOT_STATE(), - SC_PASS_SET_STATE(), SC_PASS_UNSET_STATE()); - -class tensor_transform_impl_t : public ir_visitor_t { -public: - using ir_visitor_t::dispatch; - using ir_visitor_t::visit; - - // dyn var has been defined in function; - std::unordered_set var_defined_in_func_; - // old tsr => new dyn tsr defined in func param, used for call node inside - // function. Vector is used for different scope. - std::vector> tsr_replace_map_; - // for second pass callee func decl replacement - std::unordered_map func_decl_replace_map_; - // def stmts to insert before call node - std::vector def_stmts_; - // def internal dynamic var stmts to insert before call node - std::vector internal_dyn_var_stmts_; - expr create_dyn_tsr_from_tensor(const expr &in, - std::vector &def_stmts) { - COMPILE_ASSERT(in.isa() || in.isa(), - "input should be a tensor or tensor ptr node.") - auto placeholder = in->attr().get_or_else("temp.dyn_placeholder", expr()); - std::string name; - expr tsr; - if (in.isa()) { - name = in.checked_as()->name_; - tsr = in; - } else { - tsr = in; - while (tsr.isa()) { - tsr = tsr.checked_as()->base_->ptr_; - } - name = tsr.checked_as()->name_ + "_tptr"; - } - auto dyn_tsr = builder::make_tensor(std::string("dyn_") + name, - {sizeof(runtime::dynamic_tensor_t)}, - datatypes::u8); - def_stmts.push_back(builder::make_var_tensor_def_unattached(dyn_tsr)); - - expr shape_tsr, dtype, dyn_mask, ndims; - if (placeholder.defined() && - placeholder->attr().has_key("temp.dyn_shape_of_placeholder")) { - shape_tsr = - placeholder->attr().get("temp.dyn_shape_of_placeholder"); - ndims = builder::make_read_struct(placeholder, dyn_tsr_struct_t::name, - dyn_tsr_struct_t::fields::ndims); - dtype = builder::make_read_struct(placeholder, dyn_tsr_struct_t::name, - dyn_tsr_struct_t::fields::dtype); - dyn_mask = builder::make_read_struct(placeholder, dyn_tsr_struct_t::name, - dyn_tsr_struct_t::fields::dyn_mask); - } else { - // should be in (tptr/tsr) - auto plain_dims = - in->attr_->get>(attr_keys::plain_dims); - ndims = builder::make_constant({plain_dims.size()}, datatypes::s32); - shape_tsr = builder::make_tensor(std::string("dyn_shape_") + name, - {plain_dims.size()}, datatypes::index); - shape_tsr->attr().set(attr_keys::no_tensor2var, true); - shape_tsr->attr().set(attr_keys::no_dead_write, true); - int64_t etype = tsr->dtype_.is_etype_pointer() - ? tsr->dtype_.get_pointer_element().as_etype_int() - : tsr->dtype_.as_etype_int(); - dtype = builder::make_constant({etype}, datatypes::u32); - dyn_mask = - builder::make_var(datatypes::u8, std::string("dyn_mask_") + name); - uint64_t dyn_mask_int = 0; - def_stmts.push_back(builder::make_var_tensor_def_unattached(shape_tsr)); - for (size_t i = 0; i < plain_dims.size(); i++) { - if (plain_dims[i].isa() || - var_defined_in_func_.find(plain_dims[i]) != - var_defined_in_func_.end()) { - def_stmts.push_back(builder::make_assign_unattached( - builder::make_indexing(shape_tsr, i), plain_dims[i])); - } - dyn_mask_int |= (uint64_t(!plain_dims[i].isa()) << i); - } - def_stmts.push_back(builder::make_var_tensor_def_unattached( - dyn_mask, linkage::local, - builder::make_constant({dyn_mask_int}, datatypes::u8))); - } - def_stmts.push_back(builder::make_evaluate_unattached( - builder::make_write_struct(dyn_tsr, in.remove_const(), - dyn_tsr_struct_t::name, - dyn_tsr_struct_t::fields::data_ptr))); - def_stmts.push_back(builder::make_evaluate_unattached( - builder::make_write_struct(dyn_tsr, shape_tsr, dyn_tsr_struct_t::name, - dyn_tsr_struct_t::fields::dim_ptr))); - def_stmts.push_back(builder::make_evaluate_unattached( - builder::make_write_struct(dyn_tsr, ndims, dyn_tsr_struct_t::name, - dyn_tsr_struct_t::fields::ndims))); - def_stmts.push_back(builder::make_evaluate_unattached( - builder::make_write_struct(dyn_tsr, dtype, dyn_tsr_struct_t::name, - dyn_tsr_struct_t::fields::dtype))); - def_stmts.push_back(builder::make_evaluate_unattached( - builder::make_write_struct(dyn_tsr, dyn_mask, dyn_tsr_struct_t::name, - dyn_tsr_struct_t::fields::dyn_mask))); - return dyn_tsr; - } - - void add_new_dynamic_var(const expr &new_var) { - if (var_defined_in_func_.find(new_var) == var_defined_in_func_.end()) { - if (new_var->attr_ && - new_var->attr_->has_key(attr_keys::cal_expression) && - !new_var->equals( - new_var->attr_->get(attr_keys::cal_expression))) { - internal_dyn_var_stmts_.emplace_back( - builder::make_var_tensor_def_unattached( - new_var, linkage::local, - new_var->attr_->get(attr_keys::cal_expression))); - var_defined_in_func_.insert(new_var); - }; - } - } - - expr_c visit(intrin_call_c v) override { - switch (v->type_) { - case intrin_type::read_struct: - case intrin_type::write_struct: { - if (v->intrin_attrs_->get(intrin_attr::struct_name) == - dyn_tsr_struct_t::name) { - std::vector new_args; - bool changed = dispatch_expr_vector(v->args_, new_args); - // only consider first arg. - auto &arg = v->args_[0]; - auto it = tsr_replace_map_.back().find(arg); - if (it != tsr_replace_map_.back().end()) { - new_args[0] = it->second; - changed = true; - } - if (changed) { - return copy_attr(*v, make_expr(v->type_, new_args, - *v->intrin_attrs_)); - } - } - return v; - break; - } - default: - return v; - } - } - expr_c visit(call_c v) override { - if (v->func_->attr().get_or_else(attr_keys::forbid_trans, false)) { - return v; - } - std::vector new_args(v->args_.begin(), v->args_.end()); - bool changed = false; - func_t the_func = std::dynamic_pointer_cast(v->func_); - func_t proto_func = v->get_prototype(); - assert(v->args_.size() == proto_func->params_.size()); - assert(def_stmts_.empty()); - // call kernel - bool trans_all_params = - (the_func && the_func->attr_ && - the_func->attr_->get_or_else(attr_keys::always_trans, false)) || - (proto_func->attr_ && - proto_func->attr_->get_or_else(attr_keys::always_trans, false)); - for (size_t i = 0; i < v->args_.size(); i++) { - auto &arg = v->args_[i]; - auto it = tsr_replace_map_.back().find(arg); - if (it != tsr_replace_map_.back().end()) { - new_args[i] = it->second; - changed = true; - } else if (arg.isa()) { - auto tsr_arg = arg.static_as(); - bool should_trans = false; - for (auto &d : tsr_arg->dims_) { - if (d.isa()) { - should_trans = true; - break; - } - } - should_trans = - should_trans || trans_all_params || - tsr_arg->attr().get_or_else(attr_keys::always_trans, false); - if (should_trans) { - auto dyn_tsr = create_dyn_tsr_from_tensor(tsr_arg, def_stmts_); - tsr_replace_map_.back().insert(std::make_pair(arg, dyn_tsr)); - new_args[i] = dyn_tsr; - changed = true; - } - } else if (arg.isa()) { - auto tsr_arg = arg.static_as(); - bool should_trans = false; - for (auto &d : tsr_arg->shape_) { - if (d.isa()) { - should_trans = true; - break; - } - } - should_trans = - should_trans || trans_all_params || - tsr_arg->attr().get_or_else(attr_keys::always_trans, false); - if (should_trans) { - auto dyn_tsr = create_dyn_tsr_from_tensor(tsr_arg, def_stmts_); - tsr_replace_map_.back().insert(std::make_pair(arg, dyn_tsr)); - new_args[i] = dyn_tsr; - changed = true; - } - } - } - if (changed) { - if (the_func) { - return builder::remake_call(the_func, new_args, v); - } else { - return copy_attr(*v, make_expr(v->func_, new_args)); - } - } - return v; - } - stmt_c visit(stmts_c v) override { - tsr_replace_map_.emplace_back(tsr_replace_map_.back()); - bool changed = false; - std::vector seq; - seq.reserve(v->seq_.size()); - for (size_t i = 0; i < v->seq_.size(); i++) { - def_stmts_.clear(); - auto &st = v->seq_[i]; - auto n = dispatch(st); - if (!def_stmts_.empty()) { - changed = true; - seq.insert(seq.end(), def_stmts_.begin(), def_stmts_.end()); - def_stmts_.clear(); - } - if (!n.ptr_same(st)) { - changed = true; - } - seq.emplace_back(std::move(n)); - } - tsr_replace_map_.pop_back(); - if (changed) { - return copy_attr(*v, builder::make_stmts_unattached(seq)); - } - return v; - } - - // eliminate duplicate dynamic var definition. - stmt_c visit(define_c v) override { - if (var_defined_in_func_.find(v->var_) != var_defined_in_func_.end()) { - return builder::make_stmts_unattached({}); - } - auto ret = ir_visitor_t::visit(v).checked_as(); - if (ret->var_.isa()) { - auto plain_dims = any_map_t::fetch_or_else( - ret->var_->attr_.get(), attr_keys::plain_dims, std::vector()); - for (size_t i = 0; i < plain_dims.size(); i++) { - auto &d = plain_dims[i]; - if (!d.isa()) { - assert(d.isa()); - add_new_dynamic_var(d); - } - } - } else if (ret->var_.isa()) { - var_defined_in_func_.insert(ret->var_); - } - return ret; - } - - stmt_c visit(assign_c v) override { - auto d = v->value_; - if (d.isa()) { - add_new_dynamic_var(d); - } - return v; - } - - func_c dispatch(func_c v) override { - if (std::const_pointer_cast(v)->attr().get_or_else( - attr_keys::forbid_trans, false)) { - return v; - } - tsr_replace_map_.clear(); - tsr_replace_map_.emplace_back(); - bool all_const = true; - std::vector new_params; - // base and shape define stmts insert to front of body. - assert(def_stmts_.empty()); - internal_dyn_var_stmts_.clear(); - var_defined_in_func_.clear(); - new_params.reserve(v->params_.size()); - // kernel func should transform all tensors. - bool trans_all_params = - v->attr_ && v->attr_->get_or_else(attr_keys::always_trans, false); - for (auto &p : v->params_) { - if (p.isa()) { - auto old_tsr = p.static_as(); - std::vector> shape_vars; - bool cur_const = true; - auto plain_dims = old_tsr->attr().get_or_else>( - attr_keys::plain_dims, std::vector()); - for (size_t i = 0; i < plain_dims.size(); i++) { - auto &d = plain_dims[i]; - if (!d.isa()) { - assert(d.isa()); - if (var_defined_in_func_.find(d) == var_defined_in_func_.end()) { - shape_vars.emplace_back(i, d); - var_defined_in_func_.insert(d); - } - cur_const = false; - } - } - if (trans_all_params || - p->attr().get_or_else(attr_keys::always_trans, false)) { - cur_const = false; - } - if (!cur_const) { - expr dyn_tsr; - auto it = tsr_replace_map_.back().find(old_tsr); - if (it != tsr_replace_map_.back().end()) { - dyn_tsr = it->second; - } else { - dyn_tsr = builder::make_tensor(std::string("dyn_") + old_tsr->name_, - {sizeof(runtime::dynamic_tensor_t)}, - datatypes::u8); - tsr_replace_map_.back().insert(std::make_pair(old_tsr, dyn_tsr)); - } - new_params.push_back(dyn_tsr); - - if (!shape_vars.empty()) { - // shape tensor define - auto dyn_shape_tsr = - builder::make_tensor(std::string("dyn_shape_") + old_tsr->name_, - {plain_dims.size()}, datatypes::index); - dyn_shape_tsr->attr().set(attr_keys::no_tensor2var, true); - auto def = builder::make_var_tensor_def_unattached( - dyn_shape_tsr, linkage::local, - builder::make_read_struct(dyn_tsr, dyn_tsr_struct_t::name, - dyn_tsr_struct_t::fields::dim_ptr)); - def_stmts_.push_back(def); - for (auto &var_pair : shape_vars) { - def_stmts_.push_back(builder::make_var_tensor_def_unattached( - var_pair.second, linkage::local, - builder::make_indexing(dyn_shape_tsr, var_pair.first))); - } - } - // base tensor define, may be tensor ptr - if (old_tsr.isa()) { - old_tsr->attr().set(attr_keys::no_dead_write, true); - old_tsr->attr().set(attr_keys::no_tensor2var, true); - auto def = builder::make_var_tensor_def_unattached( - old_tsr, linkage::local, - builder::make_read_struct(dyn_tsr, dyn_tsr_struct_t::name, - dyn_tsr_struct_t::fields::data_ptr)); - def_stmts_.push_back(def); - } - } else { - new_params.emplace_back(old_tsr); - } - all_const &= cur_const; - } else { - new_params.push_back(p); - } - } - std::vector def_stmts_backup = std::move(def_stmts_); - def_stmts_.clear(); - auto body = dispatch(v->body_); - bool changed = !all_const || !body.ptr_same(v->body_); - var_defined_in_func_.clear(); - if (changed) { - if (!all_const) { - if (body.isa()) { - auto seq = body.static_as()->seq_; - seq.insert(seq.begin(), def_stmts_backup.begin(), - def_stmts_backup.end()); - if (!internal_dyn_var_stmts_.empty()) { - seq.insert(seq.begin() + def_stmts_backup.size(), - internal_dyn_var_stmts_.begin(), - internal_dyn_var_stmts_.end()); - internal_dyn_var_stmts_.clear(); - } - body = copy_attr(*body, make_stmt(std::move(seq))); - } else { - def_stmts_backup.push_back(body.remove_const()); - body = make_stmt(std::move(def_stmts_backup)); - } - } - auto new_func = - copy_attr(*v, builder::make_func(v->name_, new_params, - body.remove_const(), v->ret_type_)); - // save map from symbol to decl. - func_decl_replace_map_.insert(std::make_pair(v->name_, new_func->decl_)); - return std::move(new_func); - } - return v; - } -}; - -class func_decl_replace_impl_t : public ir_visitor_t { -public: - using ir_visitor_t::dispatch; - using ir_visitor_t::visit; - // inherit from tensor_transform pass - std::unordered_map &func_decl_replace_map_; - - expr_c visit(call_c v) override { - func_t the_func = std::dynamic_pointer_cast(v->func_); - if (the_func) { - auto it = func_decl_replace_map_.find(the_func->name_); - if (it != func_decl_replace_map_.end()) { - return copy_attr( - *v, builder::make_call( - std::const_pointer_cast(it->second), v->args_)); - } - } else { - func_t proto_func = v->get_prototype(); - auto it = func_decl_replace_map_.find(proto_func->name_); - if (it != func_decl_replace_map_.end()) { - v->func_->attr().set("prototype", it->second); - return copy_attr(*v, make_expr(v->func_, v->args_)); - } - } - return v; - } - func_decl_replace_impl_t(std::unordered_map &replace_map) - : func_decl_replace_map_(replace_map) {} -}; - -func_c dyn_tensor_transformer_t::operator()(func_c f) { - tensor_transform_impl_t trans_pass; - f = trans_pass.dispatch(f); - func_decl_replace_impl_t replace_pass(trans_pass.func_decl_replace_map_); - return replace_pass.dispatch(f); -} - -const_ir_module_ptr -dyn_tensor_transformer_t::operator()(const_ir_module_ptr f) { - tensor_transform_impl_t trans_pass; - f = dispatch_module_on_visitor(&trans_pass, f); - func_decl_replace_impl_t replace_pass(trans_pass.func_decl_replace_map_); - f = dispatch_module_on_visitor(&replace_pass, f); - return f; -} -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/transform/dyn_tsr_transform.hpp b/legacy/core/src/compiler/ir/transform/dyn_tsr_transform.hpp deleted file mode 100644 index 920c66e50..000000000 --- a/legacy/core/src/compiler/ir/transform/dyn_tsr_transform.hpp +++ /dev/null @@ -1,49 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_DYN_TSR_TRANSFORM_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_DYN_TSR_TRANSFORM_HPP -#include "../module_pass.hpp" -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace attr_keys { -constexpr const char *plain_dims = "pass.plain_dims"; -constexpr const char *always_trans = "pass.always_trans"; -constexpr const char *forbid_trans = "pass.forbid_trans"; -// calculation expression for internal vars which could be generated by known -// vars. -constexpr const char *cal_expression = "pass.cal_expression"; - -} // namespace attr_keys -/** - * Do dynamic tensor transform, tensor_node=>dynamic_tensor(void pointer in IR, - * actually runtime struct). Extract and define vars from tensor when enter the - * function. This pass only acts on tensors who has dynamic shape(var_node) - * */ -class dyn_tensor_transformer_t : public module_pass_t { -public: - func_c operator()(func_c f); - const_ir_module_ptr operator()(const_ir_module_ptr f) override; - SC_DECL_PASS_INFO_FUNC(); -}; -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/transform/dynamic_parallel_transform.cpp b/legacy/core/src/compiler/ir/transform/dynamic_parallel_transform.cpp deleted file mode 100644 index ccd4ef593..000000000 --- a/legacy/core/src/compiler/ir/transform/dynamic_parallel_transform.cpp +++ /dev/null @@ -1,897 +0,0 @@ -/* - * Copyright (C) 2025 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions - * and limitations under the License. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "dynamic_parallel_transform.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -SC_DECL_PASS_INFO(dynamic_parallel_transform, SC_PASS_DEPENDS_ON(), - SC_PASS_REQUIRE_STATE(), SC_PASS_REQUIRE_NOT_STATE(), - SC_PASS_SET_STATE(), SC_PASS_UNSET_STATE()); - -func_t get_dyn_threadpool_shared_buffer_func() { - static func_t f = builder::_decl_func("sc_dyn_threadpool_shared_buffer", - datatypes::u8.get_pointerof(), - {_arg_("size", datatypes::index)}); - return f; -} - -func_t get_dyn_threadpool_loop_end_func() { - static func_t f = builder::_decl_func( - "sc_dyn_threadpool_loop_end", datatypes::index, - {_arg_("scope", datatypes::index), _arg_("size", datatypes::index)}); - return f; -} - -func_t get_dyn_threadpool_submit_func() { - static func_t f = builder::_decl_func( - "sc_dyn_threadpool_create_work_items", datatypes::index, - {_arg_("pfunc", datatypes::pointer), _arg_("iter", datatypes::pointer), - _arg_("iter_num", datatypes::index), _arg_("loop_len", datatypes::index), - _arg_("num_blocks", datatypes::index), - _arg_("tid_hint", datatypes::index), - _arg_("buffer_num", datatypes::index), - _arg_("buffers", datatypes::pointer), _arg_("flags", datatypes::index)}); - return f; -} - -func_t get_dyn_threadpool_run_func() { - static func_t f = - builder::_decl_func("sc_dyn_threadpool_run", datatypes::void_t, {}); - return f; -} - -func_t get_dyn_threadpool_destroy_func() { - static func_t f = builder::_decl_func("sc_dyn_threadpool_sched_destroy", - datatypes::void_t, {}); - return f; -} - -func_t get_dyn_threadpool_init_func() { - static func_t f = - builder::_decl_func("sc_dyn_threadpool_sched_init", datatypes::void_t, - {_arg_("stream", datatypes::pointer), - _arg_("module_data", datatypes::pointer), - _arg_("args", datatypes::generic.get_pointerof()), - _arg_("num_roots", datatypes::index), - _arg_("queue_size", datatypes::index), - _arg_("num_threads", datatypes::index)}); - return f; -} - -// what a parallel-for scope defines/captures(from parent parallel-fors) -struct parallel_for_scope_t { - std::vector iters_; - std::unordered_set defined_; - std::unordered_set captured_; - std::vector ordered_captured_; - std::weak_ptr inlined_to_; - for_loop_c loop_; - uint64_t nested_level_; - // if this scope is the start of an original parallel for - bool is_start_; - // if this scope is the end of an original parallel for - bool is_end_ = true; - bool in_if_scope_ = false; - bool in_for_scope_ = false; - uint64_t tid_step_; - - parallel_for_scope_t(const for_loop_c &loop, uint64_t nested_level, - bool is_start, uint64_t tid_step, - const std::vector *parent_iters, - const expr_c &cur_iter) - : loop_{loop}, nested_level_{nested_level}, is_start_{is_start}, - tid_step_{tid_step} { - if (parent_iters) { - iters_ = *parent_iters; - if (cur_iter.defined()) { - iters_.emplace_back(cur_iter); - } - defined_ = std::unordered_set(iters_.begin(), iters_.end()); - } - } -}; - -struct dyn_parallel_analysis_result_t { - // the collection of all captured var/tensor of nested parallel fors - std::unordered_set captured_; - std::vector ordered_captured_; - - std::vector> pfor_chain_{ - std::make_shared(for_loop_c(), 0, true, 0, nullptr, - expr_c())}; -}; - -/** - * this sub-pass traverses the nested parallel-for bodies to find 0) - * parallel-for loop scopes 1) captured var/tensors in main_thread 2) captured - * tensors in parent parallel-for scopes - * */ -class dyn_parallel_analysis_impl_t : public ir_viewer_t { -public: - using ir_viewer_t::dispatch; - using ir_viewer_t::view; - const std::vector> &main_thread_defined_; - const std::unordered_set &globals_; - - dyn_parallel_analysis_result_t &result_; - tir_pos_tracer pass_error_tracer_; - uint64_t num_threads_; - - void assert_value_is(const expr &v, int64_t val) const { - COMPILE_ASSERT_POS( - v.isa() && get_const_as_int(v.static_as()) == val, - "Expecting " << val << " for the boundary of the loop, got " << v); - } - - dyn_parallel_analysis_impl_t( - uint64_t num_threads, - const std::vector> &main_thread_defined, - const std::unordered_set &globals, - dyn_parallel_analysis_result_t &result) - : main_thread_defined_(main_thread_defined), globals_(globals), - result_(result), num_threads_{num_threads} {} - - expr_c dispatch(expr_c v) override { - TIR_ERROR_TRACE(v); - ir_viewer_t::dispatch(v); - return v; - } - - stmt_c dispatch(stmt_c v) override { - TIR_ERROR_TRACE(v); - ir_viewer_t::dispatch(v); - return v; - } - - bool try_capture_main_thread(const expr_c &v) { - if (result_.captured_.count(v)) { - return true; - } - for (auto &d : main_thread_defined_) { - if (d.count(v)) { - result_.captured_.insert(v); - result_.ordered_captured_.emplace_back(v); - return true; - } - } - return false; - } - - bool try_capture_in_parallel(const expr_c &v) { - int64_t found = -1; - for (int64_t i = result_.pfor_chain_.size() - 1; i >= 0; i--) { - auto &cur_scope = result_.pfor_chain_[i]; - if (cur_scope->captured_.count(v)) { - found = i; - break; - } - if (cur_scope->defined_.count(v)) { - found = i; - break; - } - } - if (found == -1) { - return false; - } - for (uint64_t i = found + 1; i < result_.pfor_chain_.size(); i++) { - auto &cur_scope = result_.pfor_chain_[i]; - cur_scope->captured_.insert(v); - cur_scope->ordered_captured_.emplace_back(v); - } - return true; - } - - void view(define_c v) override { - result_.pfor_chain_.back()->defined_.insert(v->var_); - ir_viewer_t::view(std::move(v)); - } - - void view(tensor_c v) override { - if (!try_capture_in_parallel(v)) { - auto defined_in_main = try_capture_main_thread(v); - if (!defined_in_main) { - COMPILE_ASSERT_POS(globals_.count(v), "Use of undefined tensor " << v); - } - } - ir_viewer_t::view(std::move(v)); - } - - void view(var_c v) override { - // we only allow using variables in the current parallel-for scope, or - // in main-thread captures, or in module globals - if (!result_.pfor_chain_.back()->defined_.count(v)) { - auto defined_in_main = try_capture_main_thread(v); - if (!defined_in_main) { - COMPILE_ASSERT_POS(globals_.count(v), - "Use of undefined variable " << v); - } - } - // ir_viewer_t::view(std::move(v)); - } - - void view(if_else_c v) override { - auto &chain_data = result_.pfor_chain_.back()->in_if_scope_; - auto old_chain_data = chain_data; - chain_data = true; - ir_viewer_t::view(std::move(v)); - chain_data = old_chain_data; - } - - void view(for_loop_c v) override { - auto &chain_data = result_.pfor_chain_.back()->in_for_scope_; - auto old_chain_data = chain_data; - if (v->kind_ == for_type::NORMAL) { - chain_data = true; - result_.pfor_chain_.back()->defined_.insert(v->var_); - ir_viewer_t::view(v); - result_.pfor_chain_.back()->defined_.erase(v->var_); - } else if (v->kind_ == for_type::PARALLEL) { - COMPILE_ASSERT_POS(!chain_data && - !result_.pfor_chain_.back()->in_if_scope_, - "Nested parallel cannot be in if or normal for"); - assert_value_is(v->iter_begin_, 0); - assert_value_is(v->step_, 1); - ir_viewer_t::dispatch(v->iter_begin_); - ir_viewer_t::dispatch(v->iter_end_); - ir_viewer_t::dispatch(v->step_); - - auto current_scope = result_.pfor_chain_.back(); - // the current scope has another parallel-for, so it is not the end - current_scope->is_end_ = false; - auto old_num_threads = num_threads_; - uint64_t thread_id_step = 1; - if (v->num_threads_ > 0) { - thread_id_step = old_num_threads / v->num_threads_; - num_threads_ = thread_id_step; - } - // dispatch body in a new scope - auto ptr = std::make_shared( - v, current_scope->nested_level_ + 1, true, thread_id_step, - ¤t_scope->iters_, v->var_); - COMPILE_ASSERT(ptr->nested_level_ <= 6, - "At most 6 nested parallel loops is supported"); - result_.pfor_chain_.emplace_back(ptr); - // dispatch into the loop body - ir_viewer_t::dispatch(v->body_); - num_threads_ = old_num_threads; - // push a new scope for the remainder of the current pfor. is_start - // should be false - auto new_scope = std::make_shared( - current_scope->loop_, current_scope->nested_level_, false, - current_scope->tid_step_, ¤t_scope->iters_, expr_c()); - result_.pfor_chain_.emplace_back(new_scope); - } else { - throw std::runtime_error("Bad loop type"); - } - chain_data = old_chain_data; - } -}; - -// transform the nested parallel for scope bodies into functions -static int transform_scopes_to_funcs( - const std::string &name, const stmts_c &body, - const dyn_parallel_analysis_result_t &result, int idx_in_result, - const std::unordered_set &globals, std::vector &out_funcs) { - auto current_scope = result.pfor_chain_.at(idx_in_result); - COMPILE_ASSERT(body.get() == current_scope->loop_->body_.get(), - "Bad parallel for scope state"); - stmts new_body; - std::unordered_map *replace_map = nullptr; - std::unique_ptr cpyer; - auto prepare_new_var_tensor = [&replace_map, &globals, &result, - ¤t_scope]() { - // make global vars unchanged - for (auto &k : globals) { - (*replace_map)[k] = k.remove_const(); - } - std::vector new_args; - // remake iterators - for (auto &k : current_scope->iters_) { - new_args.emplace_back(k->remake()); - (*replace_map)[k] = new_args.back(); - } - // remake thread captured - for (auto &k : current_scope->ordered_captured_) { - new_args.emplace_back(k->remake()); - (*replace_map)[k] = new_args.back(); - } - // remake main-thread captured - for (auto &k : result.ordered_captured_) { - new_args.emplace_back(k->remake()); - (*replace_map)[k] = new_args.back(); - } - return new_args; - }; - auto prepare_new_out_func = [&replace_map, &cpyer, &idx_in_result, &name, - &new_body, &prepare_new_var_tensor, - &out_funcs]() { - new_body = builder::make_stmts_unattached({}).static_as(); - new_body->temp_data() = std::unordered_map(); - replace_map = - &(new_body->temp_data().get>()); - cpyer = utils::make_unique(*replace_map, true); - std::vector new_args = prepare_new_var_tensor(); - auto ret = builder::make_func(name + "_0_closure_N" + - std::to_string(idx_in_result), - new_args, new_body, datatypes::void_t); - ret->attr()[function_attrs::private_] = true; - ret->attr()[function_attrs::no_parallel] = true; - ret->decl_->attr()[function_attrs::private_] = true; - ret->decl_->attr()[function_attrs::no_parallel] = true; - out_funcs.emplace_back(ret); - return ret; - }; - auto current_new_func = prepare_new_out_func(); - for (auto &s : body->seq_) { - COMPILE_ASSERT(!s.isa(), - "Expecting nested stmts to be flattened, got " << body); - auto captured_def = - s.cast() - .flat_map([](const define &v) { return v->var_.cast(); }) - .filter([&](const tensor &v) { - return (uint64_t)idx_in_result + 1 < result.pfor_chain_.size() && - result.pfor_chain_[idx_in_result + 1]->captured_.count(v); - }); - if (captured_def.has_value()) { - // if "s" is a define node and the tensor is captured by the next - // scope, need special allocation for this tensor - auto def = s.static_as(); - COMPILE_ASSERT(!def->init_.defined(), - "Captured tensors should not have init pointer" << def); - auto newdef = (*cpyer)(s).static_as(); - auto old_tsr = captured_def.get(); - newdef->init_ = builder::make_call( - get_dyn_threadpool_shared_buffer_func(), - {do_cast_and_fold((*cpyer)(old_tsr->dims_.at(0)) * - utils::get_sizeof_type(old_tsr->elem_dtype_))}); - new_body->seq_.emplace_back(newdef); - continue; - } - - auto loop = s.cast() - .filter([](const for_loop_c &l) { - return l->kind_ == for_type::PARALLEL; - }) - .get_or_else(for_loop_c()); - if (!loop.defined()) { - // if it is not a parallel-for - new_body->seq_.emplace_back((*cpyer)(s).remove_const()); - continue; - } - // "s" is a parallel for - idx_in_result = - transform_scopes_to_funcs(name, loop->body_.as(), result, - idx_in_result + 1, globals, out_funcs) + - 1; - // switch the local states to the new scope - current_scope = result.pfor_chain_.at(idx_in_result); - COMPILE_ASSERT(body.get() == current_scope->loop_->body_.get(), - "Bad parallel for scope state"); - current_new_func = prepare_new_out_func(); - } - return idx_in_result; -} - -static void finalize_body_funcs(const dyn_parallel_analysis_result_t &result, - std::vector &body_funcs, - std::vector &out_funcs) { - assert(body_funcs.size() == result.pfor_chain_.size() - 1); - // generate place holders for wrappers - out_funcs.reserve(body_funcs.size()); - for (size_t i = 0; i < body_funcs.size(); i++) { - auto &scope = result.pfor_chain_.at(i + 1); - uint64_t depth = scope->nested_level_; - uint64_t num_buffers = scope->ordered_captured_.size(); - uint64_t num_args = result.ordered_captured_.size(); - auto func = builder::make_func( - body_funcs[i]->name_ + "_wrapper", - {builder::make_tensor("itr", {depth}, datatypes::index), - builder::make_tensor("buffers", {num_buffers}, datatypes::generic), - builder::make_tensor("args", {num_args}, datatypes::generic)}, - builder::make_stmts_unattached({}), datatypes::void_t); - func->attr()[function_attrs::private_] = true; - func->decl_->attr()[function_attrs::private_] = true; - out_funcs.emplace_back(func); - } - auto find_parent_scope = - [&body_funcs, &result](const for_loop_c &v, size_t idx_in_body_funcs, - size_t &out_idx_in_body_funcs) { - uint64_t diff = 0; - for (int64_t i = (int64_t)idx_in_body_funcs; i >= 0; i--) { - // skip dummy droped for loop bodies - if (!body_funcs[i]) { - continue; - } - auto &scope = result.pfor_chain_.at(i + 1); - if (scope->is_start_ && v.ptr_same(scope->loop_)) { - out_idx_in_body_funcs = i; - return diff; - } - if (utils::is_uninitialized_weakptr(scope->inlined_to_)) { - // if the scope is inlined to another, don't need to - // add to the diff because it is not visible by the - // runtime - diff++; - } - } - throw std::runtime_error("Bad dyn_parallel_analysis_result"); - }; - for (size_t i = 0; i < body_funcs.size(); i++) { - auto &cur_func = body_funcs[i]; - if (!cur_func) { - out_funcs[i] = nullptr; - continue; - } - auto &cur_func_body = cur_func->body_.static_as()->seq_; - std::vector *cur_body = &cur_func->body_.static_as()->seq_; - auto &cur_scope = result.pfor_chain_.at(i + 1); - // generate code for 1) count down barrier 2) submit the next loop - // workload - - /* - 1) if the current scope is at the end of a parallel-for, count down - barrier like: - u64 cur_scope_1 = sc_dyn_threadpool_loop_end(nullptr, 0); - if(cur_scope_1) { - sc_dyn_threadpool_create_work_items(...); - } - 2) an optimization, if there are consecutive multiple empty scopes that - ends a parallel-for after the current scope, fold them in a nested "if" - TIR: - pfor(t) { - pfor(i) { - pfor(j) { - pfor(k) { - code(); - } // <<<< loop_end here - } // <<<< an empty scope and loop_end here - } // <<<< an empty scope and loop_end here - code2() // non empty scope - } - The tail of loop k can be transformed to: - // count down the barrier for loop k - u64 cur_scope_1 = sc_dyn_threadpool_loop_end(0, 0); - // if the current thread is the last one on loop k - if(cur_scope_1) { - // go up "1" parent scope and count down the barrier for loop j - u64 cur_scope_2 = sc_dyn_threadpool_loop_end(cur_scope_1, 1); - // if the current thread is the last one on loop j - if (cur_scope_2) { - // go up "1" parent scope and count down the barrier for loop i - u64 cur_scope_3 = sc_dyn_threadpool_loop_end(cur_scope_2, 1); - // if the current thread is the last one on loop j - if(cur_scope_3) { - // submit workload for "code2" - sc_dyn_threadpool_create_work_items(...); - } - } - } - */ - auto cur_tail_scope = cur_scope; - int scope_count = 0; - expr current_scope_handle = UINT64_C(0); - uint64_t last_parent_scope_idx = i; - auto cur_tail_scope_idx = i + 1; - bool has_barrier = false; - while (cur_tail_scope->is_end_) { - if (cur_tail_scope != cur_scope) { - // if the scope's body is not empty, we cannot optimize it. - // break. - auto &cur_scope_func = body_funcs.at(cur_tail_scope_idx - 1); - if (!cur_scope_func->body_.static_as()->seq_.empty()) { - break; - } - // remove the empty scopy body - cur_scope_func->temp_data_ = nullptr; - cur_scope_func = nullptr; - } - has_barrier = true; - auto scope_ret = builder::make_var( - datatypes::index, "__scope_handle_" + std::to_string(scope_count)); - scope_count++; - uint64_t new_scope = 0; - auto scope_diff = find_parent_scope(cur_tail_scope->loop_, - last_parent_scope_idx, new_scope); - last_parent_scope_idx = new_scope; - cur_body->emplace_back(builder::make_var_tensor_def_unattached( - scope_ret, linkage::local, - get_dyn_threadpool_loop_end_func()(current_scope_handle, - scope_diff))); - auto if_body = builder::make_stmts_unattached({}).static_as(); - cur_body->emplace_back(builder::make_if_else_unattached( - scope_ret != UINT64_C(0), if_body, stmt_c())); - cur_body = &if_body->seq_; - // now check the next scope - cur_tail_scope_idx++; - if (cur_tail_scope_idx >= result.pfor_chain_.size()) { - break; - } - cur_tail_scope = result.pfor_chain_[cur_tail_scope_idx]; - current_scope_handle = scope_ret; - } - // now submit the next scope's body - // first find the next body to submit - size_t next_func_idx = 0; - for (size_t j = i + 1; j < body_funcs.size(); j++) { - if (body_funcs[j]) { - next_func_idx = j; - break; - } - } - if (!next_func_idx) { - if (has_barrier) { - // if the current scope is the last parallel-for body - // remove the barrier code (if any) - // delele the var definition - cur_func_body.pop_back(); - // delele the "if" - cur_func_body.pop_back(); - } - body_funcs[i]->temp_data_ = nullptr; - // no function to submit, skip - continue; - } - // add the job submission code in cur_body - const auto &next_scope_info = result.pfor_chain_.at(next_func_idx + 1); - expr loop_len; - uint64_t num_iter = 0; - uint64_t num_threads = 1; - uint64_t tid_step = next_scope_info->tid_step_; - auto &replace = body_funcs[i] - ->body_->temp_data() - .get>(); - if (next_scope_info->is_start_) { - num_iter = next_scope_info->nested_level_ - 1; - num_threads = - std::max((uint64_t)next_scope_info->loop_->num_threads_, UINT64_C(1)); - // if the next scope is the begining of a loop - ir_copier_t cpy{replace, true}; - loop_len = cpy(next_scope_info->loop_->iter_end_).remove_const(); - } else { - // if the next scope returns to a parent loop - num_iter = next_scope_info->nested_level_; - loop_len = UINT64_C(1); - auto &nextfunc = body_funcs.at(next_func_idx); - if (nextfunc->body_.static_as()->seq_.empty()) { - /* an optimization: if the next scope is empty, directly call - the body: - pfor() { - pfor() { - // current scope - } - // <<< next scope here. It only submits the next pfor - pfor() { - - } - } - */ - std::vector args; - for (auto &itr : next_scope_info->iters_) { - args.emplace_back(*utils::find_map_value(replace, itr).get()); - } - for (auto &itr : next_scope_info->ordered_captured_) { - args.emplace_back(*utils::find_map_value(replace, itr).get()); - } - for (auto &itr : result.ordered_captured_) { - args.emplace_back(*utils::find_map_value(replace, itr).get()); - } - auto thecall = builder::make_call(nextfunc->decl_, args); - thecall->attr()["inline_level"] = 2; - cur_body->emplace_back(builder::make_evaluate_unattached(thecall)); - assert(utils::is_uninitialized_weakptr(next_scope_info->inlined_to_)); - next_scope_info->inlined_to_ = cur_scope; - continue; - } - } - uint64_t num_shared_buffers = next_scope_info->ordered_captured_.size(); - expr buffers; - if (num_shared_buffers) { - // an optimization, if the current scope has the same share buffers - // of the next scope, just pass nullptr - std::vector *cur_buffers = &cur_scope->ordered_captured_; - if (!utils::is_uninitialized_weakptr(cur_scope->inlined_to_)) { - auto scope = cur_scope->inlined_to_.lock(); - assert(scope); - cur_buffers = &scope->ordered_captured_; - } - bool fast_path = false; - auto &next_buffers = next_scope_info->ordered_captured_; - // check that every buffer in next buffer is in the current buffers - if (cur_buffers->size() >= next_buffers.size()) { - fast_path = true; - for (size_t bid = 0; bid < next_buffers.size(); bid++) { - if (!next_buffers[bid].ptr_same((*cur_buffers)[bid])) { - fast_path = false; - break; - } - } - } - if (fast_path) { - buffers = get_ir_null(); - } else { - buffers = builder::make_tensor("_shared_buf", {num_shared_buffers}, - datatypes::generic); - cur_body->emplace_back( - builder::make_var_tensor_def_unattached(buffers)); - for (size_t idx = 0; idx < next_buffers.size(); idx++) { - auto &buf = next_buffers[idx]; - auto itr = replace.find(buf); - COMPILE_ASSERT(itr != replace.end(), - "Bad capture state, cannot find the captured " - "buffer"); - cur_body->emplace_back(builder::make_assign_unattached( - buffers[idx], - builder::make_cast(datatypes::generic, itr->second))); - } - buffers = builder::make_cast(datatypes::pointer, buffers); - } - } else { - buffers = get_ir_null(); - } - // clear temp data - body_funcs[i]->temp_data_ = nullptr; - cur_body->emplace_back( - builder::make_evaluate_unattached(get_dyn_threadpool_submit_func()( - builder::make_func_addr(out_funcs[next_func_idx]->decl_), - get_ir_null(), num_iter, loop_len, num_threads, - /*fix-me: (yijie) outer_loop_hash*/ UINT64_C(0), num_shared_buffers, - buffers, - /*flags*/ tid_step | runtime::dynamic_threadpool::work_item_flags:: - bind_last_level))); - } -} - -// finally, fill in the body of the placeholders for wrappers -static void generate_wrappers(const dyn_parallel_analysis_result_t &result, - std::vector &body_funcs, - std::vector &wrapper_funcs) { - for (size_t i = 0; i < wrapper_funcs.size(); i++) { - if (!wrapper_funcs[i]) { - continue; - } - auto &wrapper = wrapper_funcs[i]; - auto &bodyfunc = body_funcs[i]; - auto ¤t_scope = result.pfor_chain_.at(i + 1); - // args of wrapper: - // uint64_t *itr, generic_val* shared_buffers, generic_val *args - auto &body = wrapper->body_.static_as()->seq_; - auto &itr = wrapper->params_[0]; - auto &shared_buffers = wrapper->params_[1]; - auto &args = wrapper->params_[2]; - std::vector funcargs; - for (uint64_t j = 0; j < current_scope->iters_.size(); j++) { - funcargs.emplace_back(itr[j]); - } - for (uint64_t j = 0; j < current_scope->ordered_captured_.size(); j++) { - auto dtype = current_scope->ordered_captured_[j]->dtype_; - funcargs.emplace_back(builder::make_cast(dtype, shared_buffers[j])); - } - for (uint64_t j = 0; j < result.ordered_captured_.size(); j++) { - auto dtype = result.ordered_captured_[j]->dtype_; - funcargs.emplace_back(builder::make_cast(dtype, args[j])); - } - body.emplace_back(builder::make_evaluate_unattached( - builder::make_call(bodyfunc->decl_, funcargs))); - } -} - -class dyn_parallel_transform_impl_t : public ir_visitor_t { -public: - using ir_visitor_t::dispatch; - using ir_visitor_t::visit; - bool always_transform_; - std::vector> defined_; - ir_module_ptr mod_; - std::unordered_set globals_; - func_c cur_func_; - int capture_count_ = 0; - int num_threads_; - - dyn_parallel_transform_impl_t(bool always_transform, const ir_module_ptr &mod, - int num_threads) - : always_transform_(always_transform), mod_(mod), - num_threads_(num_threads) { - for (auto &v : mod->get_module_vars()) { - globals_.insert(v->var_); - } - } - - stmt transform_loop(for_loop_c v) { - v = normalize_parallel_for_loop(v); - dyn_parallel_analysis_result_t result; - dyn_parallel_analysis_impl_t ana{(uint64_t)num_threads_, defined_, globals_, - result}; - ana.pass_error_tracer_.cur_func_ = cur_func_.get(); - ana.view(v); - // remove the dummy tail scope - result.pfor_chain_.pop_back(); - std::vector body_funcs, wrapper_funcs; - transform_scopes_to_funcs(cur_func_->name_, v->body_.as(), result, 1, - globals_, body_funcs); - finalize_body_funcs(result, body_funcs, wrapper_funcs); - generate_wrappers(result, body_funcs, wrapper_funcs); - // the parallel-for bodies are ready. Now try to submit the root jobs - auto retstmts = builder::make_stmts_unattached({}).static_as(); - auto captures = builder::make_tensor( - "__captures" + std::to_string(capture_count_), - {(uint64_t)result.ordered_captured_.size()}, datatypes::generic); - capture_count_++; - retstmts->seq_.emplace_back( - builder::make_var_tensor_def_unattached(captures)); - for (uint64_t i = 0; i < result.ordered_captured_.size(); i++) { - auto &cap = result.ordered_captured_[i]; - retstmts->seq_.emplace_back(builder::make_assign_unattached( - captures[i], builder::make_cast(datatypes::generic, cap))); - } - using namespace runtime::dynamic_threadpool; - auto null_for_stream = get_ir_null(); - null_for_stream->attr()["auto_fill_stream"] = true; - auto null_for_module_data = get_ir_null(); - null_for_module_data->attr()["auto_fill_module_data"] = true; - retstmts->seq_.emplace_back( - builder::make_evaluate_unattached(get_dyn_threadpool_init_func()( - null_for_stream, null_for_module_data, captures, UINT64_C(1), - /*queue size*/ UINT64_C(256), uint64_t(num_threads_)))); - uint64_t num_threads = v->num_threads_; - if (num_threads == 0) { - // normal parallel for - num_threads = num_threads_; - } - retstmts->seq_.emplace_back( - builder::make_evaluate_unattached(get_dyn_threadpool_submit_func()( - builder::make_func_addr(wrapper_funcs.front()->decl_), - get_ir_null(), UINT64_C(0), v->iter_end_, num_threads, - /*fix-me: (yijie) outer_loop_hash*/ UINT64_C(0), - /*num_shared_buffers*/ UINT64_C(0), get_ir_null(), - /*flags*/ - uint64_t(work_item_flags::is_root | - work_item_flags::bind_last_level | - result.pfor_chain_.at(1)->tid_step_)))); - retstmts->seq_.emplace_back( - builder::make_evaluate_unattached(get_dyn_threadpool_run_func()())); - retstmts->seq_.emplace_back( - builder::make_evaluate_unattached(get_dyn_threadpool_destroy_func()())); - for (auto &f : body_funcs) { - if (f) { - mod_->add_func({f}); - } - } - for (auto &f : wrapper_funcs) { - if (f) { - mod_->add_func({f}); - } - } - return retstmts; - } - - func_c dispatch(func_c f) override { - cur_func_ = f; - capture_count_ = 0; - defined_.emplace_back( - std::unordered_set(f->params_.begin(), f->params_.end())); - auto ret = ir_visitor_t::dispatch(f); - defined_.pop_back(); - return ret; - } - - expr_c dispatch(expr_c v) override { return v; } - - stmt_c visit(define_c v) override { - defined_.back().insert(v->var_); - return v; - } - - stmt_c visit(for_loop_c v) override { - defined_.back().insert(v->var_); - auto ret = ir_visitor_t::visit(v); - defined_.back().erase(v->var_); - return ret; - } - - stmt_c visit(stmts_c v) override { - std::vector ret; - defined_.emplace_back(); - bool changed = false; - for (auto &s : v->seq_) { - auto loop = s.cast() - .filter([](const for_loop_c &l) { - return l->kind_ == for_type::PARALLEL; - }) - .get_or_else(for_loop_c()); - if (loop.defined()) { - bool should_transform = - always_transform_ || - any_map_t::fetch_or_else(loop->attr_.get(), - attr_keys::dynamic_parallel, false); - if (should_transform) { - changed = true; - ret.emplace_back(transform_loop(loop)); - } else { - // don't need to dispatch on statically dispatched - // parallel-for - ret.emplace_back(dispatch(s)); - } - } else { - auto news = dispatch(s); - changed |= !news.ptr_same(s); - ret.emplace_back(news); - } - } - defined_.pop_back(); - if (!changed) { - return v; - } - return copy_attr(*v, builder::make_stmts_unattached(ret)); - } -}; - -const_ir_module_ptr -dynamic_parallel_transform_t::operator()(const_ir_module_ptr f) { - auto ret = std::make_shared(*f); - auto threads = runtime_config_t::get().get_num_threads(); - dyn_parallel_transform_impl_t impl{always_transform_, ret, threads}; - auto &contents = ret->get_contents(); - size_t sz = contents.size(); - for (size_t i = 0; i < sz; i++) { - if (threads == 1) { - contents[i] = std::const_pointer_cast( - remove_parallel_on_func(contents[i])); - } else { - contents[i] = - std::const_pointer_cast(impl.dispatch(contents[i])); - } - } - ret->attr_[ir_module_t::attr_key_t::MANAGED_THREAD_POOL] = - thread_pool_mode_t::DYNAMIC; - return ret; -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/transform/dynamic_parallel_transform.hpp b/legacy/core/src/compiler/ir/transform/dynamic_parallel_transform.hpp deleted file mode 100644 index 3f0de4cff..000000000 --- a/legacy/core/src/compiler/ir/transform/dynamic_parallel_transform.hpp +++ /dev/null @@ -1,97 +0,0 @@ -/******************************************************************************* - * Copyright 2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_DYNAMIC_PARALLEL_TRANSFORM_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_DYNAMIC_PARALLEL_TRANSFORM_HPP - -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -/** - * Split nested parallel-for loops into "scopes" and runtime library calls - * A "scope" is a basic block for parallel-for body, without any other nested - * parallel loops. - * The parallel body: - * pfor() { - * A() - * pfor() { - * C() - * } - * B() - * } - * .. will have 3 scopes containing A(), B(), C() respectively. - * This pass also handles dynamic buffer management, if a buffer is shared by - * multiple scopes. A more complicated example: - * - * buffer B - * pfor(i) { - * use(i) - * buffer A - * pfor(j) { - * use(i,j) - * pfor(k) { - * use(A,B) - * use(i,j,k) - * } - * } - * } - * - * ... will be transformed to - * buffer B - * submit(func1) - * func1 = (i,B) { - * use(i) - * buffer A = shared_malloc(); - * submit(func2, A, B) - * } - * func2 = (i,j,A,B) { - * use(i,j) - * submit(func3, A, B) - * } - * func3 = (i,j,k,A,B) { - * use(A,B) - * use(i,j,k) - * } - */ -class dynamic_parallel_transform_t : public module_pass_t { -public: - // transform all parallel-for loops and disregard whether the parallel-for - // is in dynamic mode - bool always_transform_; - dynamic_parallel_transform_t(bool always_transform) - : always_transform_{always_transform} {} - const_ir_module_ptr operator()(const_ir_module_ptr f) override; - SC_DECL_PASS_INFO_FUNC(); -}; - -func_t get_dyn_threadpool_shared_buffer_func(); -func_t get_dyn_threadpool_loop_end_func(); -func_t get_dyn_threadpool_submit_func(); -func_t get_dyn_threadpool_run_func(); -func_t get_dyn_threadpool_destroy_func(); -func_t get_dyn_threadpool_init_func(); - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/transform/func_inline.cpp b/legacy/core/src/compiler/ir/transform/func_inline.cpp deleted file mode 100644 index 9c59937c7..000000000 --- a/legacy/core/src/compiler/ir/transform/func_inline.cpp +++ /dev/null @@ -1,378 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#include "tensor_shrink.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -SC_DECL_PASS_INFO(func_inliner, SC_PASS_DEPENDS_ON(validator, trace_inserter), - SC_PASS_REQUIRE_STATE(), SC_PASS_REQUIRE_NOT_STATE(), - SC_PASS_SET_STATE(FUNC_INLINED), SC_PASS_UNSET_STATE()); - -// if v is a stmts_node_t, return v -// else, return a stmts_node_t containing v -static stmts_c promote_stmt_to_stmts(stmt_c v) { - if (v.isa()) { - return v.static_as(); - } - return builder::make_stmts_unattached({std::move(v)}).static_as(); -} - -// The function body copier -class func_body_copier_t : public ir_copier_impl_t { -public: - using ir_copier_impl_t::copy; - using ir_copier_impl_t::dispatch; - using ir_copier_impl_t::view; - bool already_returned_ = false; - expr ret_var_; - - func_body_copier_t(std::unordered_map &replace_map, - expr ret_var) - : ir_copier_impl_t(replace_map), ret_var_(std::move(ret_var)) {} - - expr copy(const expr_c &v) override { - dispatch(v); - auto ret = std::move(returned_expr_); - bool attr_copied = false; - // if v is var/tensor and it is in replace_map_, don't copy attr. - // Otherwise, it changes the original var/tensor's attr - if (!utils::is_one_of(v->node_type_, sc_expr_type::var, - sc_expr_type::tensor) || - replace_map_.count(v) == 0) { - copy_attr(*v, expr(ret)); - attr_copied = true; - } - if (ret.isa() || ret.isa()) { - if (ret->attr_ && - ret->attr_->has_key(tensor_shrinker_attrs::should_shrink)) { - COMPILE_ASSERT(attr_copied, "The shrink info should be copied: " << v); - auto &shrink_info = ret->attr_->get( - tensor_shrinker_attrs::should_shrink); - std::vector new_info; - new_info.reserve(shrink_info.base_.size()); - for (auto &old : shrink_info.base_) { - new_info.emplace_back(copy(old)); - } - shrink_info.base_ = std::move(new_info); - - new_info.reserve(shrink_info.shape_.size()); - for (auto &old : shrink_info.shape_) { - new_info.emplace_back(copy(old)); - } - shrink_info.shape_ = std::move(new_info); - } - } - return ret; - } - - void view(returns_c v) override { - already_returned_ = true; - if (v->value_.defined()) { - COMPILE_ASSERT(ret_var_.defined(), - "The function to inline returns a value, but ret_var_ is " - "not set"); - expr newval = copy(v->value_); - returned_stmt_ = make_stmt(ret_var_, std::move(newval)); - } else { - returned_stmt_ = make_stmt(std::vector()); - } - } - - stmt_c dispatch(stmt_c v) override { - COMPILE_ASSERT(!already_returned_, - "return_node should be the last statement in the IR, got " - << v); - return ir_copier_impl_t::dispatch(std::move(v)); - } -}; - -// the count of the inlined function calls -static std::atomic count(0); - -// TODO(xxx): Recursively inline the target function (with max recursion depth) -class func_inliner_impl_t : public ir_visitor_t { -public: - struct insertion_point_t { - std::vector &base; - size_t index; - }; - - // recursion inline depth - int recursions = 0; - // the current insertion point within a stmts_node_t - insertion_point_t *ins_point = nullptr; - const_ir_module_ptr modu_ = nullptr; - bool need_index_flatten_; - func_inliner_impl_t(bool need_index_flatten, - const const_ir_module_ptr &modu = nullptr) - : modu_(modu), need_index_flatten_(need_index_flatten) {} - - bool is_func_marked_inline(node_base *callee) { - if (auto f = dynamic_cast(callee)) { - if (modu_) { - if (auto real_f = modu_->get_func(f->name_)) { - return any_map_t::fetch_or_else(real_f->attr_.get(), "inline_level", - -1) == 2; - } - } - } - return false; - } - expr_c visit(call_c v) override { - bool is_parallel_call = bool(!v->para_attr_.empty()); - auto callee = v->func_; - auto ret = ir_visitor_t::visit(std::move(v)); - if (!is_parallel_call && - (any_map_t::fetch_or_else(ret->attr_.get(), "inline_level", -1) == 2 || - is_func_marked_inline(callee.get()))) { - if (modu_) { - return inline_at(ret.checked_as(), modu_); - } else { - return inline_at(ret.checked_as()); - } - } - return ret; - } - - stmt_c visit(if_else_c v) override { - auto cond = dispatch(v->condition_); - auto thencase = dispatch(promote_stmt_to_stmts(v->then_case_)); - stmt_c elsecase; - if (v->else_case_.defined()) - elsecase = dispatch(promote_stmt_to_stmts(v->else_case_)); - bool changed = !cond.ptr_same(v->condition_) || - !elsecase.ptr_same(v->else_case_) || - !thencase.ptr_same(v->then_case_); - if (changed) { - return copy_attr( - *v, builder::make_if_else_unattached(cond, thencase, elsecase)); - } - return v; - } - - stmt_c visit(for_loop_c v) override { - auto var = dispatch(v->var_); - auto begin = dispatch(v->iter_begin_); - auto end = dispatch(v->iter_end_); - auto step = dispatch(v->step_); - auto body = dispatch(promote_stmt_to_stmts(v->body_)); - - bool changed = !((var.ptr_same(v->var_) && begin.ptr_same(v->iter_begin_) && - end.ptr_same(v->iter_end_) && step.ptr_same(v->step_) && - body.ptr_same(v->body_))); - if (changed) { - return copy_attr(*v, builder::make_for_loop_unattached( - var, begin, end, step, body, v->incremental_, - v->kind_, v->num_threads_)); - } - return v; - } - - stmt_c visit(evaluate_c v) override { - bool is_call = v->value_.isa(); - auto newv = dispatch(v->value_); - if (!newv.ptr_same(v->value_)) { - // if we have inlined a function returning void - if (!newv.defined()) { - return make_stmt(std::vector()); - } - // if we have inlined a function call and the function call returns - // a value, do not make it a variable definition - if (is_call && (newv.isa() || newv.isa())) { - return make_stmt(std::vector()); - } - return builder::make_evaluate_unattached(newv); - } - return v; - } - - stmt_c visit(stmts_c v) override { - bool changed = false; - // a shadow array of stmt - std::vector bb; - bb.reserve(v->seq_.size()); - insertion_point_t insp{bb, 0}; - auto old_insp = ins_point; - ins_point = &insp; - for (unsigned i = 0; i < v->seq_.size(); i++) { - // always append to the tail of the BB - insp.index = bb.size(); - auto newv = dispatch(v->seq_.at(i)); - changed |= !newv.ptr_same(v->seq_.at(i)); - bb.emplace_back(std::move(newv).remove_const()); - } - ins_point = old_insp; - if (changed) { - return copy_attr(*v, make_stmt(std::move(bb))); - } - return v; - } - - expr_c inline_at(call_c site, const const_ir_module_ptr &modu = nullptr) { - auto the_func = std::dynamic_pointer_cast(site->func_); - // if the callee is a declaration, find its function body in modu. - if (!the_func) { - return site; - } - if (modu) { - if (auto real_func = modu->get_func(the_func->name_)) { - the_func = real_func; - } - } - if (!the_func->body_.defined()) { - return site; - } - recursions++; - COMPILE_ASSERT(recursions < 20, "Reached max inline recursion depth"); - - // fill in the replace map - std::unordered_map rmap; - bool has_tptr_arg = false; - for (size_t i = 0; i < site->args_.size(); i++) { - rmap.insert(std::make_pair(the_func->params_.at(i), site->args_.at(i))); - // if the arg has tensor ptr, we need to call index flatten in the - // new body to make sure nested tptr like "&&a[0][0]" is flattened - if (site->args_[i].isa()) { - has_tptr_arg = true; - } - } - if (modu) { - auto module_vars = modu->get_module_vars(); - for (size_t i = 0; i < module_vars.size(); i++) { - rmap.insert(std::make_pair(module_vars[i]->var_, module_vars[i]->var_)); - } - } - - // a "simple" function is a function with only one statement: return ... - // we can extract and inline the returned expression - stmt the_only_stmt; - if (!the_func->body_.isa()) { - the_only_stmt = the_func->body_; - } else { - auto &thestmts = the_func->body_.static_as()->seq_; - if (thestmts.size() == 1) { - the_only_stmt = thestmts.front(); - } - } - // if the function has only one stmt and it is a return, then it is - // "simple" - if (the_only_stmt.defined() && the_only_stmt.isa()) { - // if it is a "simple" function: - expr the_expr = the_only_stmt.static_as()->value_; - assert(the_expr.defined()); - // first copy the function body - func_body_copier_t impl(rmap, expr()); - expr_c newexpr = impl.copy(the_expr); - // then recursively inline the "call_node"s in the body - newexpr = dispatch(std::move(newexpr)); - recursions--; - return newexpr; - } else { - expr outvar; - std::vector::iterator itr = - ins_point->base.begin() + ins_point->index; - if (site->dtype_ != datatypes::void_t) { - // declare the return value as a var - outvar = builder::make_var(site->dtype_, - "_retval" + std::to_string(count++)); - itr = ins_point->base.insert( - itr, builder::make_var_tensor_def_unattached(outvar)) + - 1; - } - func_body_copier_t impl(rmap, outvar); - stmt_c new_body = impl.copy(promote_stmt_to_stmts(the_func->body_)); - // then recursively inline the "call_node"s in the body - new_body = dispatch(std::move(new_body)); - if (has_tptr_arg && need_index_flatten_) { - // need to flatten the nested tensorptrs - new_body = index_flattener_t()(new_body); - } - // push the new stmt in the insertion point - ins_point->base.insert(itr, std::move(new_body).remove_const()); - - recursions--; - return outvar; - } - } -}; - -expr_c func_inliner_t::inline_at(call_c c, std::vector &seq, - size_t insert_idx, - const const_ir_module_ptr &modu) { - func_inliner_impl_t impl{false}; - func_inliner_impl_t::insertion_point_t insp{seq, insert_idx}; - impl.ins_point = &insp; - return impl.inline_at(std::move(c), modu); -} - -static func_t get_callee(const stmt &s) { - return s.cast() - .flat_map([](const evaluate &v) { return v->value_.cast(); }) - .map([](const call &v) { - return std::dynamic_pointer_cast(v->func_); - }) - .get_or_else(nullptr); -} - -// check if the function is simple enough to inline. Currently, we consider a -// function which only have eval-call as a simply function. -static bool is_simple_func(const func_t &f) { - if (!f->body_.defined()) { - return false; - } - for (auto &s : f->body_.checked_as()->seq_) { - if (!get_callee(s)) { - return false; - } - } - return true; -} - -const_ir_module_ptr func_inliner_t::operator()(const_ir_module_ptr f) { - func_inliner_impl_t impl(needs_index_flatten_, f); - auto mainf = f->get_entry_func(); - for (auto &fu : f->get_contents()) { - if (fu != mainf && is_simple_func(fu)) { - fu->attr()["inline_level"] = 2; - fu->decl_->attr()["inline_level"] = 2; - } - } - - return dispatch_module_on_visitor(&impl, f); -} - -func_c func_inliner_t::operator()(func_c f) const { - func_inliner_impl_t impl{needs_index_flatten_}; - return impl.dispatch(std::move(f)); -}; - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/transform/func_inline.hpp b/legacy/core/src/compiler/ir/transform/func_inline.hpp deleted file mode 100644 index 64a0ed309..000000000 --- a/legacy/core/src/compiler/ir/transform/func_inline.hpp +++ /dev/null @@ -1,62 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_FUNC_INLINE_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_FUNC_INLINE_HPP - -#include "../module_pass.hpp" -#include "../sc_function.hpp" -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -/** - * Inlines function calls with attr["inline_level"] = 2 - * Or manually inline a call_node - * @arg needs_index_flatten is we need to run index_flatten to flatten nested - * tensor ptr on the changed part of TIR after inlining - * */ -class func_inliner_t : public module_pass_t { -public: - bool needs_index_flatten_; - func_inliner_t(bool needs_index_flatten = true) - : needs_index_flatten_(needs_index_flatten) {} - const_ir_module_ptr operator()(const_ir_module_ptr f) override; - func_c operator()(func_c f) const; - /** - * Inlines the function call and inserts the body to an existing stmt array - * - * @param c the call_node to inline - * @param seq the stmt array to insert into - * @param insert_idx the index of the insertion point within `seq` - * @return if the function returns a value, the return value of the call. - * Otherwise, null - * */ - expr_c inline_at(call_c c, std::vector &seq, size_t insert_idx, - const const_ir_module_ptr &modu = nullptr); - SC_DECL_PASS_INFO_FUNC(); -}; - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/transform/index2var.cpp b/legacy/core/src/compiler/ir/transform/index2var.cpp deleted file mode 100644 index 01251eb94..000000000 --- a/legacy/core/src/compiler/ir/transform/index2var.cpp +++ /dev/null @@ -1,984 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#include "index2var.hpp" -#include "pointer_alias_info.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -SC_MODULE(pass.index2var) - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -SC_DECL_PASS_INFO(index2var, - SC_PASS_DEPENDS_ON(constant_folder, ir_simplifier, validator, - index_flattener, - parallel_workload_dispatcher, - tensor_inplace), - SC_PASS_REQUIRE_STATE(FUNC_INLINED), - SC_PASS_REQUIRE_NOT_STATE(), SC_PASS_SET_STATE(), - SC_PASS_UNSET_STATE(IR_SIMPLIFIED)); -// it requires tensor_inplace for pointer alias info - -// the visitor to find the mutable dependencies in the indices of indexing -// nodes. e.g., for A[i+j], it will find i and j as dependencies. Note that if -// there is indexing_node/call_node in the indices, this pass will set -// `is_valid_` = false, meaning that the indices are untraceable - we are unable -// to tell if the indices are changed after some statements. -class var_dependency_finder_t : public ir_viewer_t { - utils::weakptr_hashset_t *vars_; - // the output flag to mark if the input expr is good for index2var_t - bool is_valid_ = true; - var_dependency_finder_t(utils::weakptr_hashset_t *vars) - : vars_(vars) {} - - void view(call_c v) override { - bool is_pure_no_args = - some_opt(dynamic_cast(v->func_.get())) - .filter([](func_base *f) { return f->params_.empty(); }) - .map([](func_base *f) { return f->attr_.get(); }) - .filter([](any_map_t *m) { - return m->get_or_else(function_attrs::pure, false); - }) - .has_value(); - if (is_pure_no_args) { - ir_viewer_t::view(v); - return; - } - is_valid_ = false; - SC_MODULE_INFO << "Found call node in index: " << v; - } - void view(indexing_c v) override { - is_valid_ = false; - SC_MODULE_INFO << "Found indexing node in index: " << v; - } - void view(var_c v) override { vars_->insert(v.impl); } - -public: - static bool find(utils::weakptr_hashset_t &vars, - array_ref idx) { - var_dependency_finder_t f(&vars); - for (auto &v : idx) { - f.dispatch(v); - } - return f.is_valid_; - } -}; - -struct written_tensor_analysis_result_t { - std::unordered_set written_; -}; - -struct tensor_usage_analysis_result_t { - bool used_in_broadcast_; - // the cached result of alias_info::get_alias_info - alias_info::tensor_alias_identity_t *alias_id_ = nullptr; - - tensor_usage_analysis_result_t(bool used_in_broadcast) - : used_in_broadcast_(used_in_broadcast) {} - void for_each_alias_tensor( - const std::unordered_map - &alias_map, - const std::function &func) const { - for (auto &cliq : alias_id_->alias_cliques_) { - for (auto aid : cliq->set_) { - auto other_alias_id = aid.lock(); - // if the tensor has been removed, skip - if (!other_alias_id) { - continue; - } - auto itr = alias_map.find(other_alias_id.get()); - if (itr != alias_map.end()) { - func(itr->second); - } - } - } - } -}; - -struct var_state_analysis_result_t { - utils::weakptr_hashset_t depending_on_; - int assignments_ = 0; - bool is_complex_ = false; -}; - -// the visitor to find all tensor written in all stmts. Also finds if the tensor -// is read in "broadcast" intrinsic. -class index2var_analysis_t : public ir_viewer_t { -public: - using ir_viewer_t::dispatch; - std::unordered_set written_; - std::unordered_map alias_map_; - - stmt_c dispatch(stmt_c v) override { - written_ = std::unordered_set{}; - ir_viewer_t::dispatch(v); - v->temp_data() = written_tensor_analysis_result_t{std::move(written_)}; - written_ = std::unordered_set{}; - return v; - } - - expr get_tensor_from_indexing(const expr &v) { - if (v.isa()) { - auto idx = v.static_as(); - COMPILE_ASSERT(idx->ptr_.isa(), - "The indexing should be based on a tensor. " << v); - return idx->ptr_; - } - return expr(); - } - - var_state_analysis_result_t *get_or_create_var_state(const expr &v) { - if (auto ret = v->temp_data().get_or_null()) { - return ret; - } - v->temp_data() = var_state_analysis_result_t{}; - return v->temp_data().get_or_null(); - } - - void view(define_c v) override { - if (v->var_.isa() && v->init_.defined()) { - auto var_state = get_or_create_var_state(v->var_); - var_state->assignments_++; - var_state->is_complex_ = - !var_dependency_finder_t::find(var_state->depending_on_, {v->init_}); - } - } - void view(assign_c v) override { - ir_viewer_t::view(v); - auto tsr = get_tensor_from_indexing(v->var_); - if (tsr.defined()) { - written_.insert(tsr); - } else { - auto var_state = get_or_create_var_state(v->var_); - var_state->assignments_++; - if (var_state->assignments_ == 1) { - var_state->is_complex_ = !var_dependency_finder_t::find( - var_state->depending_on_, {v->value_}); - } - } - } - - void view(tensor_c v) override { - ir_viewer_t::view(v); - auto alias = alias_info::get_alias_info(*v); - if (!alias || alias->has_no_alias()) { - return; - } - tensor_usage_analysis_result_t *result = - v->temp_data().get_or_null(); - if (!result) { - alias_map_[alias] = v; - v->temp_data() = tensor_usage_analysis_result_t{false}; - result = &v->temp_data_->get(); - } - result->alias_id_ = alias; - } - - void view(intrin_call_c v) override { - ir_viewer_t::view(v); - if (v->type_ == intrin_type::broadcast) { - auto &arg = v->args_.at(0); - auto tsr = get_tensor_from_indexing(arg); - if (tsr.defined()) { - if (auto result = tsr->temp_data() - .get_or_null()) { - result->used_in_broadcast_ = true; - } else { - tsr->temp_data() = tensor_usage_analysis_result_t{true}; - } - } - } - } - - void view(for_loop_c v) override { - ir_viewer_t::view(v); - written_ = v->body_->get_temp_data() - .get() - .written_; - } - - void view(if_else_c v) override { - ir_viewer_t::view(v); - - written_ = v->then_case_->get_temp_data() - .get() - .written_; - if (v->else_case_.defined()) { - auto &else_result = v->else_case_->get_temp_data() - .get() - .written_; - written_.insert(else_result.begin(), else_result.end()); - } - } - void view(stmts_c v) override { - for (auto &s : v->seq_) { - dispatch(s); - } - - for (auto &s : v->seq_) { - auto &result = - s->get_temp_data().get().written_; - written_.insert(result.begin(), result.end()); - } - } -}; - -enum class compare_result { - EQ, - NE, - UNKNOWN, -}; - -static bool check_index_no_overlap(ir_comparer &cmper, const expr_c &v1, - const expr_c &v2, uint16_t lanes) { - // this helper function will decompose the "v" into "lhs+const" if possible - auto get_const_add_rhs = [](const expr_c &v, expr_c &lhs) -> int64_t { - expr_c ret_lhs = v; - auto ret = - v.cast() - .map([&ret_lhs](const add &v) { - ret_lhs = v->l_; - return v->r_.as(); - }) - .filter([](const constant &v) { - return utils::is_one_of(get_type_category_nothrow(v->dtype_), - CATE_INT, CATE_UINT); - }) - .map(get_const_as_int); - if (ret.has_value()) { - lhs = std::move(ret_lhs); - return ret.get(); - } - lhs = v; - return 0; - }; - expr_c v1_lhs, v2_lhs; - auto v1_rhs_const = get_const_add_rhs(v1, v1_lhs); - auto v2_rhs_const = get_const_add_rhs(v2, v2_lhs); - if (v1_rhs_const == 0 && v2_rhs_const == 0) { - return false; - } - if (std::abs(v1_rhs_const - v2_rhs_const) < (int64_t)lanes) { - // may overlap - return false; - } - // compare if the base is the same - return cmper.compare(v1_lhs, v2_lhs); -} - -class indexing2var_impl_t : public ir_visitor_t { - struct scope_info_t; - // the "cache" for an element of a tensor - // currently, one tensor has only one "cache" - struct tensor_cache_t { - tensor_c tsr_; - std::vector idx_; - // the last write of the cached value. Null if the value is not yet - // written in the original IR - stmts last_write_; - // the var for the cache - var_c var_; - // the vector size - unsigned lanes_; - // the vector rows - unsigned rows_; - expr_c mask_; - // the tensor cache var definition may be lifted to a parent for-loop - scope_info_t *may_lift_to_ = nullptr; - utils::weakptr_hashset_t dependencies_; - tensor_cache_t(tensor_c tsr, std::vector &&idx, var_c var, - int lanes, int rows, expr_c mask = expr()) - : tsr_(std::move(tsr)), idx_(std::move(idx)), var_(std::move(var)), - lanes_(lanes), rows_(rows), mask_(std::move(mask)) {} - // returns EQ if `v` is exactly the same of the cached indexing - // returns NE if `v` does not overlap with cached indexing - // otherwise, UNKNOWN - compare_result is_match(const indexing_c &v) const { - if (!v->ptr_.ptr_same(tsr_.static_as())) { - return compare_result::UNKNOWN; - } - assert(idx_.size() == v->idx_.size()); - ir_comparer cmp(false, false, true); - if (v->dtype_.lanes_ == lanes_ && v->dtype_.rows_ == rows_) { - bool same_index = true; - if (v->idx_.size() == 1) { - if (!cmp.compare(v->idx_[0], idx_[0])) { - // the index is not the same now - if (!check_index_no_overlap(cmp, v->idx_[0], idx_[0], lanes_)) { - // if the indexing may overlap, skip - return compare_result::UNKNOWN; - } - // the index is proven not overlapping - same_index = false; - } - } else { - for (unsigned i = 0; i < v->idx_.size(); i++) { - if (!cmp.compare(v->idx_[i], idx_[i])) { - return compare_result::UNKNOWN; - } - } - } - - if (v->mask_.defined()) { - if (!mask_.defined()) { - return compare_result::UNKNOWN; - } - if (!cmp.compare(v->mask_, mask_)) { - return compare_result::UNKNOWN; - } - } else { - if (mask_.defined()) { - return compare_result::UNKNOWN; - } - } - return same_index ? compare_result::EQ : compare_result::NE; - } - return compare_result::UNKNOWN; - } - bool is_valid() const { return tsr_.defined(); } - }; - - using tensor_cache_ptr = std::shared_ptr; - - struct scope_info_t { - const for_loop_node_t *loop_; - std::vector *insert_point_; - const std::unordered_set &written_tensors_; - std::unordered_set outstanding_cache_; - // the number of times we flush cached of an tensor - std::unordered_map num_flushes_; - // the statements to be inserted after current call of visit() of a - // sub-node in stmts node - std::vector insert_after_; - bool is_cache_defined_here(const tensor_cache_ptr &v) { - return outstanding_cache_.find(v) != outstanding_cache_.end(); - } - - bool tensor_not_written_here(const expr_c &v) { - return written_tensors_.find(v) == written_tensors_.end(); - } - - void increment_num_flushes(const expr_c &v, int cnt = 1) { - num_flushes_[v] += cnt; - } - }; - - // the tensor -> tensor_cache, it stores all "cached" indexing - std::unordered_multimap cached_index_; - // the var -> tensor_cache map. The var is the variable used in indexing, - // NOT the caching var. e.g. if there is A[i,j] cached, there will be - // {i->cache of A} and {j-> cache of A} in dependency_map_. If - std::unordered_multimap dependency_map_; - // the number of vars for caching - int var_cnt_ = 0; - // the insertion point for the current stmts. We may insert var definition - // and var initialization here - std::vector *insertion_point_ = nullptr; - // the cached tensors which are created in the current scope (stmts). The - // first dimension is a stack of scopes. At the end of a scope, we need to - // evict all cache items in scope_info_.back(), then call - // scope_info_.pop() - std::deque scope_info_; - int for_depth_ = 0; - std::unordered_map &alias_map_; - std::unordered_set loop_vars_; - const for_loop_node_t *cur_for_loop_ = nullptr; - - // do write back and etc, don't remove it from the map - void invalidate_internal(tensor_cache_ptr c, // NOLINT - std::vector *writeback_point, - scope_info_t *scope) { - if (c->is_valid()) { - // if the cache is dirty, write back after the last write - if (c->last_write_.defined()) { - if (!writeback_point) { - writeback_point = &c->last_write_->seq_; - } - writeback_point->emplace_back(builder::make_assign_unattached( - builder::make_indexing(c->tsr_, c->idx_, c->lanes_, c->mask_, - c->rows_), - c->var_)); - } - // mark the cache invalid - scope->increment_num_flushes(c->tsr_); - c->tsr_ = tensor_c(); - } - } - - // flushes the cache for all indexing of the same tensor - void invalidate(tensor_cache_ptr c, // NOLINT - std::vector *writeback_point = nullptr) { // NOLINT - invalidate_if_exist(c->tsr_, writeback_point); - } - - // invalidates a tensor in the cache, returns true if is in cache - bool invalidate_if_exist(const expr_c &arg, - std::vector *writeback_point = nullptr) { - auto itrs = cached_index_.equal_range(arg); - auto ret = (itrs.first != itrs.second); - for (auto itr = itrs.first; itr != itrs.second; ++itr) { - invalidate_internal(itr->second, writeback_point, &scope_info_.back()); - } - cached_index_.erase(itrs.first, itrs.second); - return ret; - } - - void invalidate_all() { - while (!cached_index_.empty()) { - auto itr = cached_index_.begin()->first; - invalidate_if_exist(itr); - } - } - - bool is_var_dependent_only_on_loop_var(const expr_c &itr, int recur_count) { - if (recur_count > 7) { - // the dependency is too complex, or there is a loop in dependency - return false; - } - if (loop_vars_.count(itr) == 0) { - // if it depends on a non-loop var - if (auto var_state = - itr->get_temp_data().get_or_null()) { - if (var_state->assignments_ == 1 && !var_state->is_complex_) { - for (auto v : var_state->depending_on_) { - auto ptr = v.lock(); - if (ptr.get() == itr.get()) { - // self dependent - return false; - } - if (!is_var_dependent_only_on_loop_var(expr_c(ptr), - recur_count + 1)) { - return false; - } - } - } - } else { - return false; - } - } - return true; - } - - void collect_var_dependency(const std::shared_ptr &itr, - utils::weakptr_hashset_t &out) { - if (out.has(itr)) { - return; - } - out.insert(itr); - if (auto var_state = - itr->get_temp_data().get_or_null()) { - if (var_state->assignments_ == 1 && !var_state->is_complex_) { - for (auto v : var_state->depending_on_) { - auto ptr = v.lock(); - collect_var_dependency(ptr, out); - } - } - } - } - - // inserts an indexing to cache - // if `is_read`, sets the cached var to the indexing value - // if the indexing node is not cache-able, returns `v` and leaves - // `out_cache` = nullptr - expr_c make_cache(indexing_c v, const std::vector &old_idx, - bool is_read, tensor_cache_ptr &out_cache) { - SC_MODULE_INFO << "Make cache: " << v; - // the vars that the indices of `v` depends on - utils::weakptr_hashset_t vars; - // if we can trace the changes of the indices - bool is_valid = var_dependency_finder_t::find(vars, old_idx); - if (v->mask_.defined()) { - is_valid = is_valid && var_dependency_finder_t::find(vars, &(v->mask_)); - } - if (!is_valid) { - out_cache = nullptr; - return std::move(v); - } - tensor tsr = v->ptr_.as(); - assert(tsr.defined()); - if (auto ana = tsr->get_temp_data() - .get_or_null()) { - // if the tensor is used in broadcast and it is currently loaded as - // scalar - if (ana->used_in_broadcast_ && v->dtype_.lanes_ == 1) { - out_cache = nullptr; - return std::move(v); - } - } - var vcache = - builder::make_var(v->dtype_, "__cached_" + std::to_string(var_cnt_++)) - .static_as(); - assert(insertion_point_); - // declare a var, insert before the current stmt - // if read, set the var cache to the value in memory - insertion_point_->emplace_back(builder::make_var_tensor_def_unattached( - vcache, linkage::local, is_read ? v.remove_const() : expr())); - out_cache = std::make_shared( - tsr, std::vector(v->idx_.begin(), v->idx_.end()), vcache, - v->dtype_.lanes_, v->dtype_.rows_, v->mask_); - scope_info_.back().outstanding_cache_.insert(out_cache); - // remember the dependency - // and check if the index only depends on loop vars - bool only_loop_dependent = true; - for (auto itr : vars) { - auto ptr = itr.lock(); - dependency_map_.insert(std::make_pair(expr(ptr), out_cache)); - only_loop_dependent = only_loop_dependent && - is_var_dependent_only_on_loop_var(expr_c(ptr), 0); - } - if (only_loop_dependent) { - utils::weakptr_hashset_t dep_vars; - for (auto dvar : vars) { - collect_var_dependency(dvar.lock(), dep_vars); - } - // traverse the parent for-loops to find if it is possible to move - // the definition up in a later moment - scope_info_t *lifted_scope = nullptr; - for (auto itr = scope_info_.rbegin(); itr != scope_info_.rend(); ++itr) { - auto &sinfo = *itr; - if (!sinfo.loop_) { - break; - } - for_loop_node_t *nested_loop = - sinfo.loop_->body_.cast() - .filter([](const stmts &v) { return v->seq_.size() == 1UL; }) - .map([](const stmts &v) { - return v->seq_[0].as().get(); - }) - .get_or_else(nullptr); - - lifted_scope = &sinfo; - if (sinfo.loop_->kind_ != for_type::NORMAL) { - break; - } - if (dep_vars.has(sinfo.loop_->var_.impl) != 0) { - // if the index depends on the current loop, break - break; - } - if (itr != scope_info_.rbegin() && !nested_loop) { - break; - } - } - if (lifted_scope && lifted_scope != &scope_info_.back()) { - out_cache->may_lift_to_ = lifted_scope; - out_cache->dependencies_ = std::move(dep_vars); - } - } - // put into the cache - cached_index_.insert(std::make_pair(tsr, out_cache)); - return std::move(vcache); - } - - expr_c visit(cast_c v) override { - if (v->in_.isa()) { - if (invalidate_alias_group(v->in_, true)) { - SC_MODULE_INFO << "Evict due to cast: " << v; - } - scope_info_.back().increment_num_flushes(v->in_); - } - return ir_visitor_t::visit(std::move(v)); - } - - expr_c visit(call_c v) override { - auto ret = ir_visitor_t::visit(std::move(v)); - for (auto &arg : ret.as()->args_) { - if (arg.isa()) { - if (invalidate_alias_group(arg, true)) { - SC_MODULE_INFO << "Evict due to function call: " << ret; - } - scope_info_.back().increment_num_flushes(arg); - } - } - return ret; - } - - expr_c visit(tensorptr_c v) override { - // dispatch the fields inside of indexing, without calling - // index2var_t::visit on indexing. We don't need to create cache slot - // for tensorptr - auto ret_base = ir_visitor_t::visit(v->base_); - auto ret_idx = ret_base.as(); - auto tsr = ret_idx->ptr_.as(); - if (invalidate_alias_group(tsr, true)) { - SC_MODULE_INFO << "Evict due to tensorptr: " << v; - } - scope_info_.back().increment_num_flushes(tsr); - if (ret_base.ptr_same(v->base_)) { - return std::move(v); - } else { - return builder::tensor_ptr(tsr, ret_idx->idx_); - } - } - - bool invalidate_alias_group(const expr_c &tsr, bool invalidate_self) { - auto analysis_result = - tsr->get_temp_data().get_or_null(); - bool ret = false; - if (analysis_result && analysis_result->alias_id_) { - auto ths = this; - // if the tensor has alias - analysis_result->for_each_alias_tensor( - alias_map_, [&tsr, ths, &ret](const expr_c &v) { - if (!v.ptr_same(tsr)) { - ret |= ths->invalidate_if_exist(v); - } - }); - } - if (invalidate_self) { - ret |= invalidate_if_exist(tsr); - } - return ret; - } - - expr_c visit_indexing(const indexing_c &v, bool is_read, - tensor_cache_ptr &out_cache) { - auto ret = ir_visitor_t::visit(v).as(); - auto tsr = ret->ptr_.as(); - if (tsr->attr_ && - tsr->attr_->get_or_else(attr_keys::must_tensor2var, false)) { - // if the tensor is marked to be transformed to var, no need to - // optimize - return ret; - } - if (tsr->attr_ && tsr->attr_->get_or_else(attr_keys::no_index2var, false)) { - return ret; - } - if (!is_read) { - // if it is not read, need to evict all other tensors in the alias - // group. no need to invalidate tsr itself - if (invalidate_alias_group(tsr, false)) { - SC_MODULE_INFO << "Alias group invalidated for " << tsr; - } - } - auto its = cached_index_.equal_range(tsr); - for (auto itr = its.first; itr != its.second; ++itr) { - // if the tensor is cached - auto cmp_result = itr->second->is_match(ret); - if (cmp_result == compare_result::EQ) { - // the cached index is a match, return the var - out_cache = itr->second; - // If the indexing is cached, we can use it if: - // 1. it is read and we are not in for_loop (if it is in a - // for-loop, we currently do not know if there will be writes on - // the cache, which will invalidate the parent scope cache) - // 2. or the cache is created in the same scope of the access - if ((is_read && for_depth_ == 0) || - scope_info_.back().is_cache_defined_here(out_cache) || - scope_info_.back().tensor_not_written_here(tsr)) { - return itr->second->var_; - } - SC_MODULE_INFO << "Evict parent scope cache in child scope: " << ret; - // if we need to write to a cached var which is defined in - // parent scopes (not in current scope), we need to evict it - // If we don't, consider this case - // A[0] = 1 // cached here in parent scope - // if (...) { - // A[0] = 1 // no write back here - // } else { - // A[0] = 1 // write back due to the use of A[1] - // A[1] = 2 - // } - // in the above case, the else-block will evict A[0] because - // the use of A[1], but the last use of A[0] is still in the - // else block, so the write to A[0] in then-block will be - // lost - invalidate(itr->second); - break; - } else if (cmp_result == compare_result::NE) { - // do nothing. We will make cache later - } else { - SC_MODULE_INFO << "Evict old for unmatched index: " << ret; - // the tensor is cached, but with different index, evict it - invalidate(itr->second); - break; - } - } - return make_cache(std::move(ret), v->idx_, is_read, out_cache); - } - - expr_c visit(indexing_c v) override { - if (v->attr_ && v->attr_->get_or_else(attr_keys::no_index2var, false)) { - return v; - } - tensor_cache_ptr out_cache; - return visit_indexing(v, true, out_cache); - } - - stmt_c visit(stmts_c v) override { - auto old_insert = insertion_point_; - std::vector seq; - seq.reserve(v->seq_.size()); - insertion_point_ = &seq; - const for_loop_node_t *parent_loop = cur_for_loop_; - cur_for_loop_ = nullptr; - scope_info_.emplace_back(scope_info_t{ - parent_loop, - insertion_point_, - v->get_temp_data().get().written_, - {}, - {}, - {}}); - - bool changed = false; - for (auto &s : v->seq_) { - auto newstmt = dispatch(s); - changed |= !newstmt.ptr_same(s); - seq.emplace_back(std::move(newstmt)); - auto &insert_after = scope_info_.back().insert_after_; - if (!insert_after.empty()) { - changed = true; - seq.insert(seq.end(), insert_after.begin(), insert_after.end()); - insert_after.clear(); - } - } - changed |= v->seq_.size() != seq.size(); - - // sort the cache to make stable result for testing - std::vector outstanding_cache{ - scope_info_.back().outstanding_cache_.begin(), - scope_info_.back().outstanding_cache_.end()}; - std::sort(outstanding_cache.begin(), outstanding_cache.end(), - [](const tensor_cache_ptr &p1, const tensor_cache_ptr &p2) { - return p1->var_->name_ < p2->var_->name_; - }); - std::vector bases; - bases.reserve(outstanding_cache.size()); - // evict all cache items that will die at the end of this stmts - for (auto &v : outstanding_cache) { - // where to writeback the cache: the parent scope? or after the last - // write? - if (v->is_valid()) { - SC_MODULE_INFO << "Evict at the end of scope: " << v->tsr_ - << " idx=" << utils::print_vector(v->idx_) - << ", may_lift_to=" - << (v->may_lift_to_ ? v->may_lift_to_->loop_->var_ - : expr(0)); - auto check_is_tensor_defined_in_current_scope = [&v, &seq]() { - for (auto &s : seq) { - if (s.cast() - .filter([&v](const define &def) { - return def->var_.get() == v->tsr_.get(); - }) - .has_value()) { - return true; - } - } - return false; - }; - if (v->may_lift_to_) { - SC_MODULE_INFO << "Evict " << v->tsr_ << " num_flushes=" - << scope_info_.back().num_flushes_[v->tsr_] - << ", current_scope=" - << check_is_tensor_defined_in_current_scope(); - } - std::vector *writeback_point = nullptr; - scope_info_t *lift_scope = &scope_info_.back(); - // check if tensor cache can be lifted: a) check may_lift_to_ b) - // check if the cache has been flushed in the current scope c) - // if the tensor is defined in current scope, we cannot lift it - /* - Why we can't lift if the cache has been flushed? For example, - _for_(i, 0, 100, 1) { - _for_(j, 0, 100, 1) { - _for_(k, 0, 100, 1) { - A[{n, 0}] = 0; - A[{i, 0}] = 0; - } - } - } - "A[{i, 0}] = 0" cannot be lift because "A[{A, 0}] = 0" is - flushed in the k-loop, and "A[{i, 0}] = 0" may affect it - */ - if (v->may_lift_to_ && scope_info_.back().num_flushes_[v->tsr_] == 0 && - !check_is_tensor_defined_in_current_scope()) { - utils::weakptr_hashset_t &dep_vars = v->dependencies_; - // if the tensor cache var can be lifted, move the - // definition - for (auto itr = seq.begin(); itr != seq.end();) { - // if the assign node assigns to dependency var - bool assign_dep = - (*itr) - .cast() - .map([](const assign &def) { - return def->var_.as().get(); - }) - .filter([&dep_vars](var_node *var) { - return dep_vars.has(var->shared_from_this()); - }) - .get_or_else(nullptr); - // if the var definition is for the cache var or - // dependency var - // clang-format off - bool def_var = (*itr).cast() - .map([](const define &def) { - return def->var_.as().get(); - }) - .filter([&v, &dep_vars]( - var_node *var) { - return var == v->var_.get() - || dep_vars.has( - var->shared_from_this()); - }) - .get_or_else(nullptr); - // clang-format on - if (assign_dep || def_var) { - v->may_lift_to_->insert_point_->emplace_back(std::move(*itr)); - itr = seq.erase(itr); - continue; - } - - ++itr; - } - writeback_point = &v->may_lift_to_->insert_after_; - lift_scope = v->may_lift_to_; - } - bases.emplace_back(v->tsr_); - // each cache may have different writeback_point even if the - // base tensor is the same, so we first call invalidate_internal - // and then call invalidate to remove it from the map - invalidate_internal(v, writeback_point, lift_scope); - } - } - for (auto &v : bases) { - invalidate_if_exist(v); - } - if (scope_info_.size() > 1) { - auto itrprev = scope_info_.rbegin(); - auto &prev = *++itrprev; - for (auto &kv : scope_info_.back().num_flushes_) { - prev.increment_num_flushes(kv.first, kv.second); - } - } - scope_info_.pop_back(); - insertion_point_ = old_insert; - - if (changed) { - return builder::make_stmts_unattached(seq); - } - return std::move(v); - } - stmt_c visit(for_loop_c v) override { - if (v->kind_ != for_type::NORMAL) { - // before entering a parallel-for loop, invalidate all cache - // because it is not profitable to cache the value across the thread - // boundary, and it will have problems in dynamic parallel transform - invalidate_all(); - } - for_depth_++; - cur_for_loop_ = v.get(); - loop_vars_.insert(v->var_); - auto ret = ir_visitor_t::visit(v); - if (v->kind_ != for_type::NORMAL) { - invalidate_all(); - } - loop_vars_.erase(v->var_); - for_depth_--; - return ret; - } - stmt_c visit(assign_c v) override { - if (v->var_.isa()) { - auto rhs = dispatch(v->value_); - tensor_cache_ptr out_cache; - auto lhs = - visit_indexing(v->var_.static_as(), false, out_cache); - // cache creation may fail due to there is a call/indexing in - // the indices - if (out_cache) { - auto mask = v->var_.static_as()->mask_; - if (mask.defined()) { - if (mask.isa()) { - rhs = get_expr_as_int(mask) > 0 - ? rhs - : builder::make_constant({UINT64_C(0)}, rhs->dtype_); - } else { - rhs = builder::make_select( - mask, rhs, builder::make_constant({UINT64_C(0)}, rhs->dtype_)); - } - } - // if successfully created a cache for the indexing - auto ret = builder::make_stmts_unattached( - {builder::make_assign_unattached(lhs, rhs)}); - out_cache->last_write_ = ret.static_as(); - return ret; - } else if (!rhs.ptr_same(v->value_) || !lhs.ptr_same(v->var_)) { - return builder::make_assign_unattached(lhs, rhs); - } else { - return std::move(v); - } - } else { - assert(v->var_.isa()); - // if a var is changed, all indexing nodes depends on this var - // should be evicted - auto its = dependency_map_.equal_range(v->var_); - if (its.first != its.second) { - for (auto it = its.first; it != its.second; ++it) { - if (it->second->is_valid()) { - SC_MODULE_INFO << "Evict due to change of index = " << v->var_ - << ", tensor = " << it->second->tsr_; - invalidate(it->second); - } - } - dependency_map_.erase(v->var_); - } - return ir_visitor_t::visit(std::move(v)); - } - } - -public: - using ir_visitor_t::dispatch; - indexing2var_impl_t(std::unordered_map &alias_map) - : alias_map_(alias_map) {} -}; - -func_c index2var_t::operator()(func_c f) { - if (f->attr_ && f->attr_->get_or_else(function_attrs::low_level, false)) { - return f; - } - index2var_analysis_t pass; - pass.dispatch(f); - indexing2var_impl_t impl{pass.alias_map_}; - return impl.dispatch(f); -} - -stmt_c index2var_t::operator()(const stmts_c &f) { - index2var_analysis_t pass; - pass.dispatch(f); - indexing2var_impl_t impl{pass.alias_map_}; - return impl.dispatch(f); -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/transform/index2var.hpp b/legacy/core/src/compiler/ir/transform/index2var.hpp deleted file mode 100644 index 59c19af98..000000000 --- a/legacy/core/src/compiler/ir/transform/index2var.hpp +++ /dev/null @@ -1,60 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_INDEX2VAR_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_INDEX2VAR_HPP - -#include "../function_pass.hpp" - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace attr_keys { -constexpr const char *no_index2var = "pass.no_index2var"; -} -/** - * Replace indexing on tensors with vars. "Cache" the values from/to tensors to - * local variables. At the end of the scope (stmts node), the cached vars will - * be written back to the tensors - * @note This pass should run after function inlining pass - * Currently this pass is very conservative. A tensor has only one "cache slot". - * It will "flush" the var back to tensor if - * 1. In the current/parent scopes, there are more than one different index on - * the same tensor. e.g. if there is A[i] and then A[j] in use, we don't know if - * A[i] and A[j] points to the same address, so we cannot cache both of them, - * and need to "flush" the var for A[i] back to A[i]. However, if A[i] is the - * only outstanding use of tensor A, we can use A[i] mutiple times and cache it - * in a var. If `i` is changed between to uses of A[i], we also need to flush it - * 2. There is a tensorptr on the tensor, the "cached" value of the tensor be - * "flushed" - * 3. The tensor is passed to a function call - * 4. The lifetime of the cached var is over at the end of the scope - * 5. There is a write to a tensor which is cached in parent scope - * */ -class index2var_t : public function_pass_t { -public: - func_c operator()(func_c f) override; - stmt_c operator()(const stmts_c &f); - SC_DECL_PASS_INFO_FUNC(); -}; - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/transform/index_flatten.cpp b/legacy/core/src/compiler/ir/transform/index_flatten.cpp deleted file mode 100644 index 23fbe00ed..000000000 --- a/legacy/core/src/compiler/ir/transform/index_flatten.cpp +++ /dev/null @@ -1,329 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#include "index_flatten.hpp" -#include "../builder.hpp" -#include "../util_module_passes.hpp" -#include "../visitor.hpp" -#include "./constant_fold.hpp" -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -SC_DECL_PASS_INFO(index_flattener, - SC_PASS_DEPENDS_ON(dyn_tensor_transformer, - interface_generalizer, tensor_shrinker), - SC_PASS_REQUIRE_STATE(), SC_PASS_REQUIRE_NOT_STATE(), - SC_PASS_SET_STATE(), SC_PASS_UNSET_STATE(CONST_FOLDED)); - -static bool process_indexing(ir_visitor_t *ths, - const std::vector &old_dims, - const std::vector &old_strides, - const std::vector &idx, - std::vector &newidx, bool is_2d) { - bool changed = ths->dispatch_expr_vector(idx, newidx); - assert(!idx.empty()); - if (idx.size() == 1) { - return changed; - } else { - COMPILE_ASSERT(old_strides.size() == old_dims.size(), - "Dims and strides shall have same length."); - if (newidx.size() == 2 && is_2d) { - newidx = std::vector{ - builder::make_constant({UINT64_C(0)}, datatypes::s32), newidx[0], - newidx[1]}; - return true; - } - size_t start_idx = newidx.size() - 1; - if (is_2d) { - start_idx -= 2; - assert(newidx.size() > 2); - } - expr_c flattened = - builder::make_mul(newidx[start_idx], old_strides[start_idx]); - for (int64_t i = (int64_t)start_idx - 1; i >= 0; i--) { - flattened = builder::make_add( - builder::make_mul(newidx[i], old_strides[i]), flattened); - } - if (is_2d) { - auto idx0 = newidx[newidx.size() - 1]; - auto idx1 = newidx[newidx.size() - 2]; - newidx = std::vector{std::move(flattened), std::move(idx0), - std::move(idx1)}; - } else { - newidx = std::vector{std::move(flattened)}; - } - return true; - } -} - -static bool process_indexing(ir_visitor_t *ths, tensor_c &tsr, - const std::vector &idx, - std::vector &newidx, bool is_2d) { - tensor_c old = tsr; - tsr = ths->dispatch(tsr).checked_as(); - bool changed = !tsr.ptr_same(old); - COMPILE_ASSERT(old->dims_.size() == idx.size(), - "Unmatched dimensions of indexing: tsr= " - << tsr << utils::print_vector(old->dims_) - << ", indexing on" << utils::print_vector(idx)); - changed |= - process_indexing(ths, old->dims_, old->strides_, idx, newidx, is_2d); - return changed; -} - -sc_dims get_expr_to_dims(const std::vector &dims); - -static const std::vector *get_base_shape(const expr_c &ex) { - if (ex.isa()) { - return &ex.static_as()->dims_; - } - COMPILE_ASSERT(ex.isa(), "Expecting tensorptr, got: " << ex); - auto tptr = ex.static_as(); - if (tptr->is_slice_) { - return get_base_shape(tptr->base_->ptr_); - } else { - return &tptr->shape_; - } -} - -static std::vector get_dense_stride(const std::vector &shape) { - std::vector result(shape.size(), 1); - for (int i = shape.size() - 2; i >= 0; --i) { - result[i] = result[i + 1] * shape[i + 1]; - } - return result; -} - -static std::vector get_base_stride(const expr_c &ex) { - if (ex.isa()) { - return ex.static_as()->strides_; - } - COMPILE_ASSERT(ex.isa(), "Expecting tensorptr, got: " << ex); - auto tptr = ex.static_as(); - if (tptr->is_slice_) { - COMPILE_ASSERT(tptr->base_.isa(), - "tptr's base should be indexing, but got: " << tptr->base_); - return get_base_stride(tptr->base_->ptr_); - } else { - // when is_slice_ == false we create a dense tensor based on new shape - return get_dense_stride(tptr->shape_); - } -} - -static bool is_const_expr_vector(const std::vector &vec) { - return std::all_of(vec.begin(), vec.end(), [](const expr &x) { - return do_cast_and_fold(x).isa(); - }); -} - -/* process indexing on tptr(slice=false) with base = tptr(slice=true) - A = tensor[H,W,C] - B = tensorptr(A,[0,1,1,0], [H-1,W-1,C], is_slice=true) - C = tensorptr(B, [0,0,0,0], [S,C], is_slice=false) // S = (H-1) * (W-1) - indexing C[s,c] - let flatten_idx = s * C + c; - flatten(C[s,c]) = flatten(B) + (flatten_idx / C / (W-1) % (H-1) * W * C + - flatten_idx / C % (W-1) * C + flatten_idx % C) - - if shape are const, we could know C = C, S = (H-1)*(W-1) and the computation - could be simplied to - flatten(C[s,c]) = flatten(C) + (s / (W-1) % (H-1) * W * - C + s % (W-1) * C + c) -*/ -static void process_slice_reshape_indexing( - ir_visitor_t *ths, const std::vector &old_dims, - const std::vector &slice_shape, - const std::vector &parent_stride, const std::vector &idx, - std::vector &newidx) { - ths->dispatch_expr_vector(idx, newidx); - assert(!idx.empty()); - bool is_shape_all_const = is_const_expr_vector(slice_shape); - is_shape_all_const &= is_const_expr_vector(parent_stride); - is_shape_all_const &= is_const_expr_vector(old_dims); - if (is_shape_all_const) { - // if shape is all const, the computation could be simplified according - // to shape - expr flattened = 0; - auto stride = get_dense_stride(old_dims); - auto slice_const_shape = get_expr_to_dims(slice_shape); - auto real_shape = get_expr_to_dims(old_dims); - auto slice_dense_stride = get_expr_to_dims(get_dense_stride(slice_shape)); - - auto acc_shape = 1; - int acc_idx = real_shape.size() - 1; - expr flatten_remain = 0UL; - for (auto i = (int)slice_const_shape.size() - 1; i >= 0; i--) { - while (acc_shape % slice_const_shape[i] != 0 && acc_idx >= 0) { - flatten_remain = idx[acc_idx] * stride[acc_idx] + flatten_remain; - acc_shape *= real_shape[acc_idx]; - acc_idx--; - } - if (acc_shape == slice_const_shape[i]) { - flattened = flatten_remain / expr((uint64_t)slice_dense_stride[i]) * - parent_stride[i] + - flattened; - flatten_remain = 0UL; - acc_shape = 1; - } else { - flattened = flatten_remain / expr((uint64_t)slice_dense_stride[i]) % - (uint64_t)slice_const_shape[i] * parent_stride[i] + - flattened; - acc_shape /= slice_const_shape[i]; - } - } - newidx = std::vector{std::move(flattened)}; - } else { - std::vector flatten_idx; - process_indexing(ths, old_dims, get_dense_stride(old_dims), idx, - flatten_idx, false); - auto flatten_remain = flatten_idx.back(); - expr flattened = 0UL; - for (auto i = (int)slice_shape.size() - 1; i >= 0; i--) { - flattened = - (flatten_remain % slice_shape[i]) * parent_stride[i] + flattened; - flatten_remain = flatten_remain / slice_shape[i]; - } - newidx = std::vector{std::move(flattened)}; - } -} - -class index_flatten_t : public ir_consistent_visitor_t { -public: - using ir_consistent_visitor_t::dispatch; - using ir_consistent_visitor_t::visit; - - expr_c visit(indexing_c v) override { - std::vector newidx; - bool changed; - bool is_2d = v->dtype_.rows_ != 0; - COMPILE_ASSERT(!is_2d || v->idx_.size() >= 2UL, - "2D load/store should have at least 2D index"); - if (v->ptr_.isa()) { - auto ptr = v->ptr_.static_as(); - auto old_ptr = ptr; - changed = process_indexing(this, ptr, v->idx_, newidx, is_2d); - if (is_2d) { - throw std::runtime_error("2D memory is not supported"); - } else { - if (changed) { - return copy_attr(*v, - builder::make_indexing(ptr, newidx, v->dtype_.lanes_, - v->mask_, v->dtype_.rows_)); - } else { - return std::move(v); - } - } - } else { - // indexing on a tensor_ptr - tensorptr_c oldptr = v->ptr_.checked_as(); - auto ptr = dispatch(v->ptr_).checked_as(); - if (!oldptr->is_slice_ && oldptr->base_->ptr_.isa() && - oldptr->base_->ptr_.static_as()->is_slice_) { - COMPILE_ASSERT(!is_2d, - "2D indexing not yet supported on reshaped tensorptr"); - auto old_parent_ptr = oldptr->base_->ptr_.static_as(); - auto base_shape = oldptr->shape_; - auto slice_shape = old_parent_ptr->shape_; - auto parent_stride = get_base_stride(old_parent_ptr); - process_slice_reshape_indexing(this, base_shape, slice_shape, - parent_stride, v->idx_, newidx); - } else { - const std::vector *shape = get_base_shape(oldptr); - const std::vector stride = get_base_stride(oldptr); - // 1D input might not have shape info - assert(oldptr->base_->idx_.size() == 1 || !shape->empty()); - // flatten the indices using the reshaped dimensions - COMPILE_ASSERT(shape->size() == v->idx_.size(), - "Unmatched dimensions of indexing: " - << v << ", shape:" << utils::print_vector(*shape)); - process_indexing(this, *shape, stride, v->idx_, newidx, is_2d); - if (is_2d) { - throw std::runtime_error("2D memory is not supported"); - } - } - // flatten the indices using tensor_ptr's dimensions - // then, add the flattened 1D index with the flattened 1D offset - // of the base tensor_ptr - assert(ptr->base_->idx_.size() == 1); - return copy_attr(*v, - builder::make_indexing( - ptr->base_->ptr_, - builder::make_add(ptr->base_->idx_[0], newidx[0]), - v->dtype_.lanes_, v->mask_, v->dtype_.rows_)); - } - } - - expr_c visit(tensor_c v) override { - COMPILE_ASSERT(v->dims_.size() == v->strides_.size(), - "Tensor dims and strides shall have same length."); - if (v->dims_.size() == 1 && v->strides_[0].isa() && - get_expr_as_int(v->strides_[0]) == 1) { - // if already flattened, return - return v; - } else { - expr range = 1; - for (size_t i = 0; i < v->dims_.size(); ++i) { - range = builder::make_add( - builder::make_mul(v->dims_[i] - 1, v->strides_[i]), range); - } - // here stride is {1} since all strided info are compressed into - // dims - auto ret = - copy_attr(*v, builder::make_stensor( - v->name_, {range}, std::vector{UINT64_C(1)}, - v->elem_dtype_, v->address_space_, v->init_value_)); - return ret; - } - } - - expr_c visit(tensorptr_c v) override { - auto base = dispatch(v->base_); - // we should remove the shape info - if (base.ptr_same(v->base_) && v->shape_.empty()) { - return v; - } - return copy_attr(*v, make_expr( - base.remove_const().checked_as(), - std::vector{}, false)); - } -}; - -func_c index_flattener_t::operator()(func_c f) { - index_flatten_t pass; - return pass.dispatch(std::move(f)); -} - -stmt_c index_flattener_t::operator()(stmt_c f) { - index_flatten_t pass; - return pass.dispatch(std::move(f)); -} - -const_ir_module_ptr index_flattener_t::operator()(const_ir_module_ptr f) { - index_flatten_t pass; - return dispatch_module_on_visitor(&pass, f); -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/transform/index_flatten.hpp b/legacy/core/src/compiler/ir/transform/index_flatten.hpp deleted file mode 100644 index 2a15ab86c..000000000 --- a/legacy/core/src/compiler/ir/transform/index_flatten.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_INDEX_FLATTEN_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_INDEX_FLATTEN_HPP - -#include "../module_pass.hpp" -#include "../sc_function.hpp" - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -/** - * Transforms multi-dimemsion tensors into 1-D tensors. - * e.g. tensor A: float[100,200] => tensor A: float[100*200] - * Also transforms the corresponding indexing nodes: - * e.g. A[10,20] => A[10*200+20] - * */ -class index_flattener_t : public module_pass_t { -public: - const_ir_module_ptr operator()(const_ir_module_ptr f) override; - func_c operator()(func_c f); - stmt_c operator()(stmt_c f); - SC_DECL_PASS_INFO_FUNC(); -}; - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/transform/insert_trace.cpp b/legacy/core/src/compiler/ir/transform/insert_trace.cpp deleted file mode 100644 index 39f457a86..000000000 --- a/legacy/core/src/compiler/ir/transform/insert_trace.cpp +++ /dev/null @@ -1,114 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#include "insert_trace.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -SC_DECL_PASS_INFO(trace_inserter, SC_PASS_DEPENDS_ON(validator), - SC_PASS_REQUIRE_STATE(), SC_PASS_REQUIRE_NOT_STATE(), - SC_PASS_SET_STATE(), SC_PASS_UNSET_STATE()); -class trace_inserter_impl_t : public ir_visitor_t { -public: - using ir_visitor_t::dispatch; - int func_id; - func_c dispatch(func_c v) override { - if (v->attr_ && (v->attr_->get_or_else(function_attrs::skip_trace, false) || - v->attr_->get_or_else(function_attrs::low_level, false))) { - return v; - } - func_id = register_traced_func(v->name_); - auto oldbody = v->body_; - assert(oldbody.isa()); - const auto &seq = oldbody.static_as()->seq_; - bool is_return_last = !seq.empty() && seq.back().isa(); - std::vector newseq; - newseq.emplace_back( - builder::make_evaluate_unattached(builtin::make_trace(func_id, 0, 0))); - for (const auto &s : seq) { - newseq.emplace_back(dispatch(s).remove_const()); - } - if (!is_return_last) { - newseq.emplace_back(builder::make_evaluate_unattached( - builtin::make_trace(func_id, 1, 0))); - } - return copy_attr( - *v, builder::make_func(v->name_, v->params_, - make_stmt(std::move(newseq)), - v->ret_type_)); - } - - stmt_c visit(evaluate_c v) override { - if (runtime_config_t::get().trace_mode_ >= - runtime_config_t::trace_mode_t::KERNEL && - v->value_.isa()) { - auto intrin = v->value_.static_as(); - if (intrin->type_ == intrin_type::brgemm) { - return builder::make_stmts_unattached({ - builder::make_evaluate_unattached( - builtin::make_trace_kernel(0, 0, 0)), - v, - builder::make_evaluate_unattached(builtin::make_trace_kernel( - 0, 1, intrin->args_.at(brgemm_args::NUM))), - }); - } else if (intrin->type_ == intrin_type::list_brgemm) { - return builder::make_stmts_unattached({ - builder::make_evaluate_unattached( - builtin::make_trace_kernel(1, 0, 0)), - v, - builder::make_evaluate_unattached(builtin::make_trace_kernel( - 1, 1, - intrin->args_.at(brgemm_args::NUM) * - intrin->args_.at(brgemm_args::LEN))), - }); - } - } - return v; - } - - stmt_c visit(returns_c v) override { - return builder::make_stmts_unattached( - {builder::make_evaluate_unattached(builtin::make_trace(func_id, 1, 0)), - v}); - } -}; - -const_ir_module_ptr trace_inserter_t::operator()(const_ir_module_ptr m) { - auto ret = std::make_shared(*m); - auto &funcs = ret->get_contents(); - for (unsigned i = 0; i < funcs.size(); i++) { - funcs[i] = std::const_pointer_cast( - trace_inserter_impl_t().dispatch(funcs[i])); - } - return ret; -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/transform/insert_trace.hpp b/legacy/core/src/compiler/ir/transform/insert_trace.hpp deleted file mode 100644 index 41ed4d356..000000000 --- a/legacy/core/src/compiler/ir/transform/insert_trace.hpp +++ /dev/null @@ -1,41 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_INSERT_TRACE_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_INSERT_TRACE_HPP - -#include "../module_pass.hpp" - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -/** - * Inserts calls to sc_make_trace when entering and exit from functions. - * This pass will register the function id => function name mapping when a - * module is compiled. - * */ -class trace_inserter_t : public module_pass_t { -public: - const_ir_module_ptr operator()(const_ir_module_ptr m) override; - SC_DECL_PASS_INFO_FUNC(); -}; -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/transform/interface_generalize.cpp b/legacy/core/src/compiler/ir/transform/interface_generalize.cpp deleted file mode 100644 index b9ac860ea..000000000 --- a/legacy/core/src/compiler/ir/transform/interface_generalize.cpp +++ /dev/null @@ -1,96 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#include - -#include "../builder.hpp" -#include "../easy_build.hpp" -#include "interface_generalize.hpp" -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -SC_DECL_PASS_INFO(interface_generalizer, - SC_PASS_DEPENDS_ON(dyn_tensor_transformer), - SC_PASS_REQUIRE_STATE(), SC_PASS_REQUIRE_NOT_STATE(), - SC_PASS_SET_STATE(), SC_PASS_UNSET_STATE()); - -const_ir_module_ptr -interface_generalizer_t::operator()(const_ir_module_ptr in) { - auto ret = in->copy(); - auto &funcs = ret->get_contents(); - auto len = funcs.size(); - builder::ir_builder_t builder; - for (unsigned i = 0; i < len; i++) { - auto f = funcs[i]; - if (f->body_.defined() && - (!f->attr_ || - !f->attr_->get_or_else(function_attrs::private_, false))) { - std::string wrapper_name = f->name_ + "_0wrapper"; - assert(!ret->get_func(wrapper_name)); - _function_(datatypes::void_t, wrapper_func, - _arg_("args", datatypes::generic, {(int)f->params_.size()})) { - _bind_(args); - std::vector fargs; - fargs.reserve(f->params_.size()); - for (uint64_t idx = 0; idx < f->params_.size(); idx++) { - auto ¶m = f->params_[idx]; - assert(param->dtype_.lanes_ == 1); - if (param->dtype_ == datatypes::generic) { - fargs.emplace_back(args[idx]); - } else { - fargs.emplace_back(builder::make_cast(param->dtype_, args[idx])); - } - } - builder.push_evaluate(builder::make_call(f->decl_, fargs)); - } - wrapper_func->name_ = wrapper_name; - wrapper_func->decl_->name_ = wrapper_name; - if (f->attr_) { - if (f->attr_->get_or_else(function_attrs::is_main, false)) { - wrapper_func->attr()[function_attrs::is_main] = true; - } - if (auto comments = - f->attr_->get_or_null>("comments")) { - std::vector new_comments = { - comments->at(0), "@param args The array of arguments. It should " - "contain the following:"}; - for (size_t i = 1; i < comments->size(); i++) { - auto &comment = (*comments)[i]; - new_comments.emplace_back(" " + comment); - if (!comment.empty() && comment[0] == '@') { - new_comments.back()[2] = '-'; - } - } - wrapper_func->attr()["comments"] = std::move(new_comments); - } - } - ret->add_func({wrapper_func}); - } - } - return ret; -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/transform/interface_generalize.hpp b/legacy/core/src/compiler/ir/transform/interface_generalize.hpp deleted file mode 100644 index d4c59cf67..000000000 --- a/legacy/core/src/compiler/ir/transform/interface_generalize.hpp +++ /dev/null @@ -1,42 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_INTERFACE_GENERALIZE_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_INTERFACE_GENERALIZE_HPP - -#include "../module_pass.hpp" - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -/** - * Generates a wrapper for each function, which accepts generic values as - * parameters - * */ -class interface_generalizer_t : public module_pass_t { -public: - const_ir_module_ptr operator()(const_ir_module_ptr m) override; - SC_DECL_PASS_INFO_FUNC(); -}; - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/transform/loop_function_motion.cpp b/legacy/core/src/compiler/ir/transform/loop_function_motion.cpp deleted file mode 100644 index 3eeaa01f6..000000000 --- a/legacy/core/src/compiler/ir/transform/loop_function_motion.cpp +++ /dev/null @@ -1,85 +0,0 @@ -/******************************************************************************* - * Copyright 2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include "loop_function_motion.hpp" -#include "../builder.hpp" -#include "../util_module_passes.hpp" -#include "../viewer.hpp" -#include "../visitor.hpp" -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -SC_DECL_PASS_INFO(simple_loop_function_motion, - SC_PASS_DEPENDS_ON(loop_merger, parallel_workload_dispatcher, - ir_simplifier, closurizer_cpu), - SC_PASS_REQUIRE_STATE(), SC_PASS_REQUIRE_NOT_STATE(), - SC_PASS_SET_STATE(), SC_PASS_UNSET_STATE(IR_SIMPLIFIED)); - -class pure_func_hoister_t : public ir_visitor_t { -public: - using ir_visitor_t::dispatch; - using ir_visitor_t::visit; - int index_ = 0; - // Currently all pure funcs will be promoted to outermost loop - int loop_depth_ = 0; - // Each pure func ret value will assign to a var - std::vector call_var_defs_; - expr_c visit(call_c v) override { - // Currently only promote pure funcs with no arg - if (is_pure_func_call(v) && v->args_.empty() && loop_depth_ > 0) { - assert(v->dtype_ != datatypes::void_t); - func_t callee = v->get_prototype(); - auto var_name = - "call_var_" + std::to_string(index_++) + "_" + callee->name_; - auto call_var = builder::make_var(v->dtype_, var_name); - call_var_defs_.emplace_back( - builder::make_var_tensor_def_unattached(call_var, linkage::local, v)); - return call_var; - } - return v; - } - stmt_c visit(for_loop_c v) override { - loop_depth_++; - auto vv = ir_visitor_t::visit(std::move(v)); - loop_depth_--; - if (loop_depth_ == 0 && !call_var_defs_.empty()) { - call_var_defs_.emplace_back(vv); - auto ret = builder::make_stmts_unattached(call_var_defs_); - call_var_defs_.clear(); - return ret; - } - return vv; - } -}; - -func_c simple_loop_function_motion_t::operator()(func_c v) { - pure_func_hoister_t pure_func_hoister; - - return pure_func_hoister.dispatch(std::move(v)); -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/transform/loop_function_motion.hpp b/legacy/core/src/compiler/ir/transform/loop_function_motion.hpp deleted file mode 100644 index 7524fa569..000000000 --- a/legacy/core/src/compiler/ir/transform/loop_function_motion.hpp +++ /dev/null @@ -1,40 +0,0 @@ -/******************************************************************************* - * Copyright 2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_LOOP_FUNCTION_MOTION_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_LOOP_FUNCTION_MOTION_HPP - -#include "../function_pass.hpp" - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -/** - * Simple non-SSA LICM for pure function version, hoist pure function call - * inside loop. If the function has args related to loop vars, do not hoist. - * */ -class simple_loop_function_motion_t : public function_pass_t { -public: - func_c operator()(func_c f) override; - SC_DECL_PASS_INFO_FUNC(); -}; -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/transform/loop_invariant_code_motion.cpp b/legacy/core/src/compiler/ir/transform/loop_invariant_code_motion.cpp deleted file mode 100644 index 8c9b30229..000000000 --- a/legacy/core/src/compiler/ir/transform/loop_invariant_code_motion.cpp +++ /dev/null @@ -1,963 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#include "loop_invariant_code_motion.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -SC_DECL_PASS_INFO(loop_invariant_code_motion, SC_PASS_DEPENDS_ON(ssa_transform), - SC_PASS_REQUIRE_STATE(SSA_STAGE), SC_PASS_REQUIRE_NOT_STATE(), - SC_PASS_SET_STATE(), SC_PASS_UNSET_STATE()); - -struct licm_analysis_data_t { - const stmt_base_t *parent_; - // if the var is global or volatile - bool volatile_ = false; - // The loop vars stmt depending on - std::unordered_set dep_vars_; - std::unordered_set dep_tensors_; - // scope - size_t if_for_depth_ = 0; - - licm_analysis_data_t(const stmt_base_t *parent) : parent_(parent) {} -}; - -static bool unordered_intersects(const std::unordered_set &a, - const std::unordered_set &b) { - for (const auto &elem : a) { - if (b.end() != std::find_if(b.begin(), b.end(), [&](const expr_c &v) { - return v.ptr_same(elem); - })) { - return true; - } - } - return false; -} - -// Promotion depends on call, call nodes must be hoisted before this pass. -// Currently tensorptr can not be hoisted. -static bool expr_can_hoist(const expr_c &s) { - return passlet::non_volatile_expr(s.get()) || s.isa() || - s.isa(); -} - -static bool stmt_can_hoist(const stmt_c &s) { - return s.isa() || s.isa() || s.isa() || - s.isa(); -} - -struct tensor_analysis_data_t { - // The base tensor vars depends on - std::unordered_set base_tensors_; -}; -#define BASE_TENSORS(V) \ - (V)->temp_data().get().base_tensors_ - -// Analyze call and tensor volatility in loops for licm promotion assessment -struct loop_analysis_viewer_t : public ssa_viewer_t { - using ssa_viewer_t::dispatch; - using ssa_viewer_t::view; - const stmt_base_t *current_ = nullptr; - std::vector cur_loop_vars_; - std::vector> cur_loop_ptrs_; - std::unordered_map call_volatile_map_; - std::unordered_map> base_tensor_map_; - std::unordered_map> tensor_volatile_map_; - std::unordered_map alias_map_; - - expr_c dispatch(expr_c s) override { - if (!s->get_temp_data().isa()) { - s->temp_data() = tensor_analysis_data_t(); - } - return ssa_viewer_t::dispatch(std::move(s)); - } - - stmt_c dispatch(stmt_c s) override { - if (!s->get_temp_data().isa()) { - s->temp_data() = tensor_analysis_data_t(); - } - auto old = current_; - current_ = s.get(); - auto ret = ssa_viewer_t::dispatch(std::move(s)); - current_ = old; - return ret; - } - - void view(for_loop_c v) override { - cur_loop_vars_.emplace_back(v->var_); - cur_loop_ptrs_.emplace_back(std::unordered_set()); - call_volatile_map_[v->var_] = false; - tensor_volatile_map_[v->var_] = std::unordered_set(); - ssa_viewer_t::view(v); - // If inner loop call volatile, mark outer nested loops as call volatile - if (call_volatile_map_[v->var_]) { - for (const auto &loop_var : cur_loop_vars_) { - call_volatile_map_[loop_var] = true; - } - } - // If current loop contain volatile tensor, mark all alias as volatile - auto &cur_tensor_volatile_map = tensor_volatile_map_[v->var_]; - for (const auto &tsr : cur_tensor_volatile_map) { - auto alias = alias_info::get_alias_info(*tsr); - if (!alias || alias->has_no_alias()) { - continue; - } - for (auto &cliq : alias->alias_cliques_) { - for (auto aid : cliq->set_) { - auto other_alias_id = aid.lock(); - // if the tensor has been removed, skip - if (!other_alias_id) { - continue; - } - auto itr = alias_map_.find(other_alias_id.get()); - if (itr != alias_map_.end()) { - cur_tensor_volatile_map.insert(itr->second); - } - } - } - } - // for each volatile base tensor, filter loop volatile pointers - // mark all volatile pointers - std::unordered_set volatile_ptrs; - const auto &cur_loop_ptrs = cur_loop_ptrs_.back(); - for (const auto &tsr : cur_tensor_volatile_map) { - auto iter = base_tensor_map_.find(tsr); - if (iter != base_tensor_map_.end()) { - auto &alias = iter->second; - // only pointer used inside loop will be considered - // avoiding marking other loop's volatile pointers - for (const auto &a : alias) { - if (cur_loop_ptrs.find(a) != cur_loop_ptrs.end()) { - volatile_ptrs.insert(a); - } - } - } - } - cur_tensor_volatile_map.insert(volatile_ptrs.begin(), volatile_ptrs.end()); - // end of loop scope - cur_loop_vars_.pop_back(); - cur_loop_ptrs_.pop_back(); - } - - void view(var_c v) override { - // Prepare for volatile pointer filter - if (v->dtype_.is_pointer() && !cur_loop_ptrs_.empty()) { - for (auto &loop_ptrs : cur_loop_ptrs_) { - loop_ptrs.insert(v); - } - } - } - - void view(tensor_c v) override { - // Prepare for tensor alias analysis - auto alias = alias_info::get_alias_info(*v); - if (alias) { - alias_map_[alias] = v; - } - // Prepare for base tensor analysis - BASE_TENSORS(v).insert(v); - } - - void view(cast_c v) override { - ssa_viewer_t::view(v); - // Prepare for base tensor analysis - auto &st_base = BASE_TENSORS(current_); - auto &vi_base = BASE_TENSORS(v->in_); - st_base.insert(vi_base.begin(), vi_base.end()); - } - - void view(binary_c v) override { - ssa_viewer_t::view(v); - // Prepare for base tensor analysis - auto &st_base = BASE_TENSORS(current_); - auto &vl_base = BASE_TENSORS(v->l_); - auto &vr_base = BASE_TENSORS(v->r_); - st_base.insert(vl_base.begin(), vl_base.end()); - st_base.insert(vr_base.begin(), vr_base.end()); - } - - void view(define_c v) override { - ssa_viewer_t::view(v); - if (v->var_.isa()) { - auto &st_base = BASE_TENSORS(current_); - auto &va_base = BASE_TENSORS(v->var_); - va_base.insert(st_base.begin(), st_base.end()); - if (v->var_->dtype_.is_pointer()) { - for (const auto &tsr : va_base) { - base_tensor_map_[tsr].insert(v->var_); - } - } - } else if (v->var_.isa()) { - for (const auto &loop_var : cur_loop_vars_) { - auto &volatile_map = tensor_volatile_map_[loop_var]; - volatile_map.insert(v->var_); - } - } - } - - void view(call_c v) override { - ssa_viewer_t::view(v); - // If loop contain call, mark this loop as call volatile - if (!cur_loop_vars_.empty()) { - call_volatile_map_[cur_loop_vars_.back()] = true; - } - // If loop contain volatile tensor, mark this loop and - // outer nested loops with this volatile tensor - // In call(&A[i]), A considered volatile - if (!cur_loop_vars_.empty()) { - for (const auto &arg : v->args_) { - auto &base = BASE_TENSORS(arg); - for (const auto &loop_var : cur_loop_vars_) { - auto &volatile_map = tensor_volatile_map_[loop_var]; - volatile_map.insert(base.begin(), base.end()); - } - } - } - } - - void view(assign_c v) override { - ssa_viewer_t::view(v); - // If loop contain volatile tensor, mark this loop and - // outer nested loops with this volatile tensor - // In A[i] = x, A considered volatile - if (v->var_.isa() && !cur_loop_vars_.empty()) { - auto &base = BASE_TENSORS(v->var_.static_as()->ptr_); - for (const auto &loop_var : cur_loop_vars_) { - auto &volatile_map = tensor_volatile_map_[loop_var]; - volatile_map.insert(base.begin(), base.end()); - } - } - } -}; - -static std::vector * -find_stmt_in_map(std::unordered_map> &m, - const stmt_c &st) { - for (auto &it : m) { - if (std::find_if(it.second.begin(), it.second.end(), - [&st](const stmt_c &in) { return in.ptr_same(st); }) != - it.second.end()) { - return &it.second; - } - } - return nullptr; -} - -template static stmt get_owner(T &data) { - assert(data.defined()); - return data->ssa_data_->get_owner(); -} - -static bool get_licm_data_volatile(const stmt &owner) { - assert(owner.defined()); - return owner->temp_data().get().volatile_; -} - -static void set_licm_data_volatile(const stmt &owner, bool vlt) { - owner->temp_data().get().volatile_ = vlt; -} - -static size_t get_owner_depth(const stmt &owner) { - assert(owner.defined()); - return owner->temp_data().get().if_for_depth_; -} - -struct var_exist_analysis_t : public ssa_viewer_t { - using ir_viewer_t::dispatch; - using ir_viewer_t::view; - bool has_loop_var = false; - var_c cur_loop_var_need_find; - - void view(var_c v) override { - if (cur_loop_var_need_find.defined()) { - if (v.ptr_same(cur_loop_var_need_find)) { - has_loop_var = true; - } - } - } - - void view(ssa_phi_c v) override { - if (v->is_loop_phi_) { - has_loop_var = true; - } - if (has_loop_var) { - return; - } - ssa_viewer_t::view(v); - } - - void view(stmts_c v) override { - for (auto &s : v->seq_) { - dispatch(s); - if (has_loop_var) { - break; - } - } - } -}; - -struct var_property_set_t : public ssa_viewer_t { - using ir_viewer_t::dispatch; - using ir_viewer_t::view; - var_c cur_property_var_; - bool find_property_var_ = false; - std::unordered_set &visited_set_; - std::vector &modified_vars_; - - var_property_set_t(std::unordered_set &set_vars, - std::vector &changed_vars) - : visited_set_(set_vars), modified_vars_(changed_vars) {} - - void view(ssa_phi_c v) override { // ssa phi already have proper attribute - } - - void view(stmts_c v) override { - for (auto &s : v->seq_) { - dispatch(s); - } - } - - void view(var_c v) override { - if (cur_property_var_.defined()) { - if (v.ptr_same(cur_property_var_)) { - find_property_var_ = true; - } - } - } - void process_vars_property(expr &var, expr &value, - licm_analysis_data_t *v_licm) { - if (var.defined() && visited_set_.count(var)) { - return; - } - if (value.defined()) { - find_property_var_ = false; - dispatch(value); - if (find_property_var_) { - auto property_var_owner = get_owner(cur_property_var_); - if (property_var_owner.defined()) { - auto property_var_prop = property_var_owner->temp_data() - .get_or_null(); - licm_analysis_data_t *cur_licm = v_licm; - if (!visited_set_.count(var) && cur_licm != nullptr) { - cur_licm->dep_vars_.insert(property_var_prop->dep_vars_.begin(), - property_var_prop->dep_vars_.end()); - cur_licm->volatile_ = property_var_prop->volatile_; - modified_vars_.emplace_back(var); - visited_set_.insert(var); - } - } - } - find_property_var_ = false; - } - } - - void view(define_c v) override { - process_vars_property(v.remove_const()->var_, v.remove_const()->init_, - v->temp_data().get_or_null()); - } -}; - -static void -set_loop_var_stmt_dep_var(std::vector &cur_loop_vars_, - std::vector &phi_values, stmt &owner, - std::unordered_set &modified_vars) { - for (int j = cur_loop_vars_.size() - 1; j >= 0; j--) { - auto &x = cur_loop_vars_[j]; - var_exist_analysis_t loop_var_analysis; - loop_var_analysis.cur_loop_var_need_find = x.dyn_as(); - loop_var_analysis.dispatch(owner); - if (loop_var_analysis.has_loop_var) { - auto x_owner = x->ssa_data_->get_owner(); - if (x_owner.defined()) { - auto loop_depth = get_owner_depth(x_owner); - for (size_t i = 0; i < phi_values.size(); i++) { - auto cur_phi_owner = phi_values[i]->ssa_data_->get_owner(); - if (cur_phi_owner.defined()) { - auto &st_data = - cur_phi_owner->temp_data_->get(); - // if already define in outer scope or same - // scope as for loop, don't need to move - if (loop_depth < get_owner_depth(cur_phi_owner)) { - st_data.dep_vars_.insert(x); - auto parent_stmt = cur_phi_owner->temp_data() - .get() - .parent_->node_ptr_from_this() - .remove_const(); - if (parent_stmt.defined()) { - modified_vars.insert(phi_values[0]); - } - } - } - } - } - } - } -} - -struct non_loop_phi_volatile_set_t : public ssa_viewer_t { - using ir_viewer_t::dispatch; - using ir_viewer_t::view; - std::vector cur_loop_vars_; - std::vector phi_values_; - // used to avoid visit repeatly - std::unordered_set visited_set_; - // modified_vars - std::unordered_set &changed_vars_; - - non_loop_phi_volatile_set_t(const std::vector &loop_vars_, - const std::vector &values_phi, - std::unordered_set &modified_vars) - : cur_loop_vars_(loop_vars_), phi_values_(values_phi), - changed_vars_(modified_vars) {} - - void view(var_c v) override { - if (visited_set_.count(v)) { - return; - } - auto owner = v->ssa_data_->get_owner(); - auto is_loopvar = - std::find_if(cur_loop_vars_.begin(), cur_loop_vars_.end(), - [&v](expr_c &x) { return x.dyn_as().ptr_same(v); }); - // non loop var need to find whether relate with loop var - if (is_loopvar == cur_loop_vars_.end() && owner.defined()) { - set_loop_var_stmt_dep_var(cur_loop_vars_, phi_values_, owner, - changed_vars_); - if (visited_set_.count(v) == 0 && !v->ssa_data_->is_garbage()) { - visited_set_.insert(v); - dispatch(owner); - } - } - } - - void view(ssa_phi_c v) override { ssa_viewer_t::view(v); } - - void view(stmts_c v) override { - for (auto &s : v->seq_) { - dispatch(s); - } - } -}; - -static void process_with_non_loop_phi(std::vector &phi_values, - ssa_phi_c &phi, - std::vector &cur_loop_vars_) { - std::unordered_set modified_vars_set; - std::vector modified_vars; - for (size_t i = 1; i < phi_values.size(); i++) { - auto owner = phi_values[i]->ssa_data_->get_owner(); - if (owner.defined()) { - // first parent is {} scope, second parent is if scope - // if{} - assert(owner->temp_data_->get().parent_); - auto cur_owner_scope = owner->temp_data_->get() - .parent_->temp_data() - .get() - .parent_->node_ptr_from_this() - .remove_const(); - // All the current phi value need to be same volatile to avoid - // error. - if (get_licm_data_volatile(cur_owner_scope)) { - auto cur_owner = phi_values[0]->ssa_data_->get_owner(); - if (cur_owner.defined()) { - auto cur_depth = get_owner_depth(cur_owner); - // be in same scope - if (cur_depth + 1 == get_owner_depth(owner) && cur_owner.defined()) { - set_licm_data_volatile(cur_owner, true); - auto parent_stmt = cur_owner->temp_data() - .get() - .parent_->node_ptr_from_this() - .remove_const(); - // propagation - if (parent_stmt.defined()) { - modified_vars_set.insert(phi_values[0]); - } - } - } - } - // Find vars that may be affected by loop var - non_loop_phi_volatile_set_t vlt_set(cur_loop_vars_, phi_values, - modified_vars_set); - vlt_set.dispatch(cur_owner_scope); - // need to find more accurate owner that have loop var - set_loop_var_stmt_dep_var(cur_loop_vars_, phi_values, cur_owner_scope, - modified_vars_set); - } - } - // propagation - modified_vars.reserve(modified_vars_set.size()); - for (auto &x : modified_vars_set) { - modified_vars.emplace_back(x); - } - while (!modified_vars.empty()) { - auto x = modified_vars.back(); - modified_vars.pop_back(); - auto cur_owner = x->ssa_data_->get_owner(); - if (cur_owner.defined()) { - auto parent_stmt = cur_owner->temp_data() - .get() - .parent_->node_ptr_from_this() - .remove_const(); - if (parent_stmt.defined()) { - var_property_set_t property_set(modified_vars_set, modified_vars); - property_set.cur_property_var_ = x.dyn_as(); - property_set.dispatch(parent_stmt); - } - } - } -} - -struct licm_analysis_viewer_t : public ssa_viewer_t { - using ssa_viewer_t::dispatch; - using ssa_viewer_t::view; - const stmt_base_t *current_ = nullptr; - size_t if_for_depth_count_ = 0; - // - std::vector cur_if_scopes_; - std::vector cur_loop_vars_; - // currently we treat if_scope as a special stmt to judge - // how loop invariants interact with it - std::unordered_map if_scope_loop_map_; - std::unordered_map> if_scope_var_map_; - // - licm_analysis_viewer_t() { - // Create a dummy if_scope - cur_if_scopes_.emplace_back( - builder::make_if_else_unattached(false, {}, {})); - } - stmt_c dispatch(stmt_c s) override { - if (!s->get_temp_data().isa()) { - s->temp_data() = licm_analysis_data_t(current_); - } else { - // if the vardef is used before define, update the parent - auto &licm_data = s->temp_data().get(); - assert(licm_data.parent_ == nullptr); - licm_data.parent_ = current_; - } - if (!stmt_can_hoist(s)) { - s->temp_data().get().volatile_ = true; - return s; - } - auto old = current_; - current_ = s.get(); - auto ret = ssa_viewer_t::dispatch(std::move(s)); - current_ = old; - return ret; - } - expr_c dispatch(expr_c s) override { - if (!expr_can_hoist(s) && current_ != nullptr && !cur_loop_vars_.empty()) { - current_->temp_data().get().volatile_ = true; - return s; - } - if (!s->get_temp_data().isa()) { - s->temp_data() = licm_analysis_data_t(current_); - } - return ssa_viewer_t::dispatch(std::move(s)); - } - void view(for_loop_c v) override { - if_scope_loop_map_[v->var_] = cur_if_scopes_.back(); - cur_loop_vars_.emplace_back(v->var_); - v->temp_data().get().if_for_depth_ = - if_for_depth_count_; - ++if_for_depth_count_; - ssa_viewer_t::view(v); - cur_loop_vars_.pop_back(); - auto &v_data = v->temp_data().get(); - auto &body_data = v->body_->temp_data().get(); - v_data.volatile_ |= body_data.volatile_; - v_data.dep_vars_.insert(body_data.dep_vars_.begin(), - body_data.dep_vars_.end()); - v_data.dep_tensors_.insert(body_data.dep_tensors_.begin(), - body_data.dep_tensors_.end()); - --if_for_depth_count_; - } - void view(stmts_c v) override { - ssa_viewer_t::view(v); - auto &v_data = v->temp_data().get(); - for (auto &st : v->seq_) { - // if st is garbage - if (!st->get_temp_data().isa()) { - continue; - } - auto &st_data = st->temp_data().get(); - v_data.dep_vars_.insert(st_data.dep_vars_.begin(), - st_data.dep_vars_.end()); - v_data.dep_tensors_.insert(st_data.dep_tensors_.begin(), - st_data.dep_tensors_.end()); - if (st_data.volatile_) { - v_data.volatile_ = true; - break; - } - } - } - void view(define_c v) override { - auto &st_data = v->temp_data().get(); - st_data.if_for_depth_ = if_for_depth_count_; - dispatch(v->var_); - if (v->init_.defined()) { - dispatch(v->init_); - } - // synchronize volatile attribute between define node and its var node; - if (v->var_.isa()) { - auto &var_data = v->var_->temp_data().get(); - var_data = st_data; - if_scope_var_map_[cur_if_scopes_.back()].insert(v->var_); - } else if (v->var_.isa() && !cur_loop_vars_.empty()) { - auto &var_data = v->var_->temp_data().get(); - var_data.dep_vars_.insert(cur_loop_vars_.back()); - st_data.volatile_ = true; - } - } - void view(if_else_c v) override { - ++if_for_depth_count_; - cur_if_scopes_.emplace_back(v); - ssa_viewer_t::view(v); - auto &st_data = v->temp_data().get(); - auto &then_data = v->then_case_->temp_data().get(); - st_data.volatile_ |= then_data.volatile_; - st_data.dep_vars_.insert(then_data.dep_vars_.begin(), - then_data.dep_vars_.end()); - st_data.dep_tensors_.insert(then_data.dep_tensors_.begin(), - then_data.dep_tensors_.end()); - if (v->else_case_.defined()) { - auto &else_data = v->else_case_->temp_data().get(); - st_data.volatile_ |= else_data.volatile_; - st_data.dep_vars_.insert(else_data.dep_vars_.begin(), - else_data.dep_vars_.end()); - st_data.dep_tensors_.insert(else_data.dep_tensors_.begin(), - else_data.dep_tensors_.end()); - } - // Pass down all dependencies to vars defined in this if_else_c - auto &vars_in_if_scope_ = if_scope_var_map_[v]; - for (auto &var : vars_in_if_scope_) { - auto &var_data = var->temp_data().get(); - var_data.volatile_ |= st_data.volatile_; - var_data.dep_vars_.insert(st_data.dep_vars_.begin(), - st_data.dep_vars_.end()); - var_data.dep_tensors_.insert(st_data.dep_tensors_.begin(), - st_data.dep_tensors_.end()); - } - vars_in_if_scope_.clear(); - // End of this if_scopes - cur_if_scopes_.pop_back(); - --if_for_depth_count_; - } - - void view(var_c v) override { - auto owner = v->ssa_data_->get_owner(); - auto &is_param = v->ssa_data_->is_param_; - if (current_ == nullptr) { - assert(is_param); - return; - } - auto &st_data = current_->temp_data().get(); - auto &var_data = v->temp_data().get(); - if (var_data.volatile_) { - st_data.volatile_ = true; - return; - } - st_data.dep_vars_.insert(var_data.dep_vars_.begin(), - var_data.dep_vars_.end()); - st_data.dep_tensors_.insert(var_data.dep_tensors_.begin(), - var_data.dep_tensors_.end()); - // param can be considered as invariant. - // if a var is used before defined, treat it as volatile. - if (owner.defined() && - owner->temp_data().get_or_null()) { - if (owner.isa()) { - st_data.dep_vars_.insert(v); - } - } else if (!is_param) { - var_data.volatile_ = true; - st_data.volatile_ = true; - } - } - void view(tensor_c v) override { - auto &is_param = v->ssa_data_->is_param_; - if (current_ == nullptr) { - assert(is_param); - return; - } - auto &st_data = current_->temp_data().get(); - auto &var_data = v->temp_data().get(); - st_data.dep_vars_.insert(var_data.dep_vars_.begin(), - var_data.dep_vars_.end()); - if (var_data.volatile_) { - st_data.volatile_ = true; - return; - } - } - void view(indexing_c v) override { - ssa_viewer_t::view(v); - auto &st_data = current_->temp_data().get(); - st_data.dep_tensors_.insert(v->ptr_); - } - void view(ssa_phi_c v) override { - ssa_viewer_t::view(v); - if (v->is_loop_phi_) { - for (auto &val : v->values_) { - auto owner = val->ssa_data_->get_owner(); - if (owner.defined() && - owner->temp_data().get_or_null()) { - owner->temp_data().get().volatile_ = true; - } - } - } else if (!cur_loop_vars_.empty() && v->values_.size() > 1) { - process_with_non_loop_phi(v.remove_const()->values_, v, cur_loop_vars_); - } - } -}; - -struct licm_hoist_prepare_viewer_t : public ssa_viewer_t { - using ssa_viewer_t::dispatch; - using ssa_viewer_t::view; - const stmt_base_t *current_ = nullptr; - // - std::vector cur_if_scopes_; - std::vector cur_loop_vars_; - // currently we treat if_scope as a special stmt to judge - // how loop invariants interact with it - std::unordered_map if_scope_loop_map_; - // output map: loop var => vector of invariants, the invariants will - // be inserted just before their correspond loop vars. - std::unordered_map> loop_invariant_map_; - std::unordered_set stmt_to_remove_set_; - // input map: call and tensor info - std::unordered_map &call_volatile_map_; - std::unordered_map> &tensor_volatile_map_; - // - licm_hoist_prepare_viewer_t( - std::unordered_map &call_volatile_map, - std::unordered_map> - &tensor_volatile_map) - : call_volatile_map_(call_volatile_map), - tensor_volatile_map_(tensor_volatile_map) { - // Create a dummy if_scope - cur_if_scopes_.emplace_back( - builder::make_if_else_unattached(false, {}, {})); - } - void register_loop_invariant_stmt() { - assert(current_ != nullptr); - // if stmts (need follow parent if or for), return - if (current_->node_type_ == sc_stmt_type::stmts) { - return; - } - // if the stmt is not in loop, return. - if (cur_loop_vars_.empty()) { - return; - } - auto &st_data = current_->temp_data().get(); - if (st_data.volatile_) { - return; - } - // Find the loop to promote, from inner-most to out-most - auto it = cur_loop_vars_.rbegin(); - for (; it != cur_loop_vars_.rend(); it++) { - // If any dep_tensors_ is defined or to be stored in the loop - // cannot promote - bool volatile_by_tensor = - unordered_intersects(st_data.dep_tensors_, tensor_volatile_map_[*it]); - // If any dep_vars_ is the loop var, cannot promote - bool volatile_by_var = // - st_data.dep_vars_.end() != - std::find_if(st_data.dep_vars_.begin(), st_data.dep_vars_.end(), - [&](const expr_c &v) { return v.ptr_same(*it); }); - // If loop contains call node, cannot promote - bool volatile_by_call = call_volatile_map_[*it]; - // If loop outside current if scope, cannot promote - bool outside_if_scope = - !if_scope_loop_map_[*it].ptr_same(cur_if_scopes_.back()); - // If depends not volatile, continue find the loop to promote - if (volatile_by_tensor || volatile_by_var || volatile_by_call || - outside_if_scope) { - break; - } - } - if (it != cur_loop_vars_.rbegin()) { - stmt_c s = current_->node_ptr_from_this(); - loop_invariant_map_[*(it - 1)].emplace_back(s); - stmt_to_remove_set_.insert(s); - return; - } else { - // current_ is volatile in its loop, cannot promote - return; - } - } - stmt_c dispatch(stmt_c s) override { - if (!stmt_can_hoist(s)) { - return s; - } - auto old = current_; - current_ = s.get(); - auto ret = ssa_viewer_t::dispatch(std::move(s)); - register_loop_invariant_stmt(); - current_ = old; - return ret; - } - expr_c dispatch(expr_c s) override { - if (!expr_can_hoist(s) && current_ != nullptr && !cur_loop_vars_.empty()) { - return s; - } - return ssa_viewer_t::dispatch(std::move(s)); - } - void view(for_loop_c v) override { - if_scope_loop_map_[v->var_] = cur_if_scopes_.back(); - cur_loop_vars_.emplace_back(v->var_); - ssa_viewer_t::view(v); - cur_loop_vars_.pop_back(); - } - void view(if_else_c v) override { - cur_if_scopes_.emplace_back(v); - ssa_viewer_t::view(v); - // End of this if_scopes - cur_if_scopes_.pop_back(); - } -}; - -// Second filter map and set to remove stmt by volatile == true -static void filter_stmt_by_volatile( - std::unordered_map> &hoist_map, - std::unordered_set &stmt_to_remove) { - for (auto &kv : hoist_map) { - for (auto it = kv.second.begin(); it != kv.second.end();) { - if ((*it)->temp_data().get().volatile_) { - it = kv.second.erase(it); - } else { - it++; - } - } - } - for (auto it = stmt_to_remove.begin(); it != stmt_to_remove.end();) { - if ((*it)->temp_data().get().volatile_) { - it = stmt_to_remove.erase(it); - } else { - it++; - } - } -} -struct licm_hoister_t : public ssa_visitor_t { - using ssa_visitor_t::dispatch; - using ssa_visitor_t::visit; - std::unordered_map> &hoist_map_; - std::unordered_set &stmt_to_remove_; - // if_else and for_loop may changed after dispatch - std::unordered_map stmt_replace_map_; - licm_hoister_t(std::unordered_map> &hoist_map, - std::unordered_set &stmt_to_remove) - : hoist_map_(hoist_map), stmt_to_remove_(stmt_to_remove) {} - // we are not interested in exprs. Simply return and don't dispatch down - expr_c dispatch(expr_c v) override { return v; } - - stmt_c visit(stmts_c v) override { - std::vector ret_vec; - bool changed = false; - - for (auto &s : v->seq_) { - // if s is garbage - if (!s->get_temp_data().isa()) { - changed = true; - continue; - } - auto sz_before = ret_vec.size(); - auto ret = dispatch(s); - - // if an stmt is inserted, the IR is changed - changed |= sz_before != ret_vec.size(); - changed |= !ret.ptr_same(s); - if (stmt_to_remove_.find(s) != stmt_to_remove_.end()) { - assert(s->temp_data().get().volatile_ == false); - changed = true; - continue; - } - if (ret.isa()) { - auto it = hoist_map_.find(ret.static_as()->var_); - if (it != hoist_map_.end()) { - std::vector *cur_hoist_stmts = &(it->second); - // ret_vec insert cur_hoist_stmts with replacement - for (auto &s : *cur_hoist_stmts) { - auto rep = stmt_replace_map_.find(s); - if (rep != stmt_replace_map_.end()) { - ret_vec.push_back(rep->second); - } else { - ret_vec.push_back(s); - } - } - changed = true; - } - } - ret_vec.emplace_back(ret); - } - if (!changed) { - return v; - } else { - return copy_attr(*v, builder::make_stmts_unattached(ret_vec)); - } - } - - stmt_c visit(for_loop_c v) override { - auto vv = ssa_visitor_t::visit(v); - if (!vv.ptr_same(v)) { - stmt_replace_map_[v] = vv; - } - return vv; - } - - stmt_c visit(if_else_c v) override { - auto vv = ssa_visitor_t::visit(v); - if (!vv.ptr_same(v)) { - stmt_replace_map_[v] = vv; - } - return vv; - } -}; - -func_c loop_invariant_code_motion_t::operator()(func_c f) { - // Analyze loop first - loop_analysis_viewer_t loop_analyzer; - loop_analyzer.dispatch(f); - // Mark loop-invariant code - licm_analysis_viewer_t analyzer; - analyzer.dispatch(f); - // use the mark information to prepare invariant map - licm_hoist_prepare_viewer_t hoist_prepared( - loop_analyzer.call_volatile_map_, loop_analyzer.tensor_volatile_map_); - hoist_prepared.dispatch(f); - // Move code - filter_stmt_by_volatile(hoist_prepared.loop_invariant_map_, - hoist_prepared.stmt_to_remove_set_); - licm_hoister_t hoister(hoist_prepared.loop_invariant_map_, - hoist_prepared.stmt_to_remove_set_); - return hoister.top_level_dispatch(f); -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/transform/loop_invariant_code_motion.hpp b/legacy/core/src/compiler/ir/transform/loop_invariant_code_motion.hpp deleted file mode 100644 index d2ba0faee..000000000 --- a/legacy/core/src/compiler/ir/transform/loop_invariant_code_motion.hpp +++ /dev/null @@ -1,48 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_LOOP_INVARIANT_CODE_MOTION_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_LOOP_INVARIANT_CODE_MOTION_HPP - -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -/** - * Do loop invariant code motion(LICM) in SSA (Single-Static-Assignment) - * form. It includes two steps: - * 1. Analysis and mark loop-independent variables. Notice that we do LIP only - * for local var (mainly process indexing of tensor), not for global var/tensor, - * local tensor/indexing/tensorptr node or call node. Currently we treat an - * if-else node as a whole, if an if-else node can not be hoisted, neither can - * its inner vars. - * 2. Hoist as many definitions of var out of loops as possible. The definitions - * should be hoisted out of its inner loop as far as possible. - * */ -class loop_invariant_code_motion_t : public function_pass_t { -public: - func_c operator()(func_c f) override; - SC_DECL_PASS_INFO_FUNC(); -}; - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/transform/loop_merge.cpp b/legacy/core/src/compiler/ir/transform/loop_merge.cpp deleted file mode 100644 index 57d957ee5..000000000 --- a/legacy/core/src/compiler/ir/transform/loop_merge.cpp +++ /dev/null @@ -1,179 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#include "loop_merge.hpp" -#include -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -SC_DECL_PASS_INFO(loop_merger, SC_PASS_DEPENDS_ON(constant_folder), - SC_PASS_REQUIRE_STATE(), SC_PASS_REQUIRE_NOT_STATE(), - SC_PASS_SET_STATE(), SC_PASS_UNSET_STATE()); - -// if v is not stmts or v->seq_.size() > 1, return v -// if v is a stmts node with only one stmt, return the stmt -static stmt_c extract_single_stmt(stmt_c v) { - auto vstmt = v.as(); - if (vstmt.defined() && vstmt->seq_.size() == 1) { - return vstmt->seq_.front(); - } - return v; -} - -class loop_merger_impl_t : public ir_visitor_t { - bool merge_recursive = false; - stmt_c visit(stmts_c v) override { - if (v->seq_.size() <= 1) { - // if there is only one or less statement, no chance to merge - return ir_visitor_t::visit(std::move(v)); - } - std::vector newseq; - newseq.reserve(v->seq_.size()); - bool changed = false; - for (auto itr = v->seq_.begin(); itr != v->seq_.end(); ++itr) { - auto news = dispatch(extract_single_stmt(*itr)); - if (!news.ptr_same(*itr)) { - changed = true; - } - newseq.emplace_back(news.remove_const()); - } - // the size of old seq - int seqsize = newseq.size(); - // the size of seq after merging - int newseqsize = 0; - // the util function to push an stmt to newseq. Note that newseq has - // already allocated enough size, we only need to write the value, - // instead of pushing new ones - auto repush_to_newseq = [&newseq, &newseqsize](stmt &&s) { - newseq[newseqsize] = std::move(s); - newseqsize++; - }; - // the util function to move an stmt in newseq from src position to dst - // position. We need to firstly insert it to dst position and remove it - // in src position - auto move_to_newseq = [&newseq, &newseqsize](stmt s, int src, int dst) { - if (src == dst) { - newseq[newseqsize] = std::move(s); - } else { - newseq.emplace(newseq.begin() + dst, std::move(s)); - newseq.erase(newseq.begin() + src + 1); - } - newseqsize++; - }; - std::unordered_map rmap; - ir_copier_t ir_cper(rmap, /*create_var_tensor*/ false); - - for (int i = 0; i < seqsize; i++) { - auto loop = newseq[i].as(); - // the index for the next iteration (will be added by 1) - // it is used to skip the merged for-loops - int nexti = i; - - if (loop.defined()) { - // the editable copy of the loop - for_loop writable_loop; - if (merge_recursive || - (loop->attr_ && loop->attr_->has_key(stmt_attr_key::merge_loop))) { - // use this variable to record where should declare all the - // expressions of merged loop - int expr_insert_pos = newseqsize; - for (int j = i + 1; j < seqsize; j++) { - // for each next stmt, check if it is for_loop and is - // marked stmt_attr_key::merge_loop - auto nextloop = newseq[j].as(); - if (nextloop.defined() && - (merge_recursive || - (nextloop->attr_ && - nextloop->attr_->has_key(stmt_attr_key::merge_loop)))) { - if (!writable_loop.defined()) { - writable_loop = loop->remake().static_as(); - } - int merged_loops = writable_loop->merge_all( - stmt(), ir_cper(nextloop).static_as()); - // if the next loop cannot be merged, break - if (merged_loops == 0) { - break; - } - // skip the merged loop in the for(int i...) - nexti++; - } else { - // if next stmt is var tensor def, move it up to - // expr_insert_pos - if (newseq[j].isa()) { - move_to_newseq(std::move(newseq[j]), j, expr_insert_pos++); - // skip the merged loop in the for(int i...) - nexti++; - // set changed flag - changed = true; - continue; - } else { - // if we cannot merge the next stmt, break - break; - } - } - } - } - if (writable_loop.defined()) { - // if the loop has been merged with next loops - changed = true; - bool old_merge_recursive = merge_recursive; - merge_recursive = true; - writable_loop->body_ = dispatch(writable_loop->body_).remove_const(); - merge_recursive = old_merge_recursive; - repush_to_newseq(std::move(writable_loop)); - } else { - repush_to_newseq(loop.remove_const()); - } - } else { - repush_to_newseq(std::move(newseq[i])); - } - i = nexti; - } - if (changed) { - newseq.resize(newseqsize); - return make_stmt(std::move(newseq)); - } - return v; - } -}; - -func_c loop_merger_t::operator()(func_c f) { - loop_merger_impl_t impl; - return impl.dispatch(f); -}; - -expr_c loop_merger_t::operator()(expr_c f) { - loop_merger_impl_t impl; - return impl.dispatch(std::move(f)); -}; - -stmt_c loop_merger_t::operator()(stmt_c f) { - loop_merger_impl_t impl; - return impl.dispatch(std::move(f)); -}; - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/transform/loop_merge.hpp b/legacy/core/src/compiler/ir/transform/loop_merge.hpp deleted file mode 100644 index fb512e2f7..000000000 --- a/legacy/core/src/compiler/ir/transform/loop_merge.hpp +++ /dev/null @@ -1,44 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_LOOP_MERGE_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_LOOP_MERGE_HPP - -#include "../function_pass.hpp" -#include "../sc_function.hpp" - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -/** - * Merge consecutive loops with attr[stmt_attr_key::merge_loop] defined - * */ -class loop_merger_t : public function_pass_t { -public: - func_c operator()(func_c f) override; - expr_c operator()(expr_c f); - stmt_c operator()(stmt_c f); - SC_DECL_PASS_INFO_FUNC(); -}; - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/transform/loop_split.cpp b/legacy/core/src/compiler/ir/transform/loop_split.cpp deleted file mode 100644 index cc88c5406..000000000 --- a/legacy/core/src/compiler/ir/transform/loop_split.cpp +++ /dev/null @@ -1,128 +0,0 @@ -/******************************************************************************* - * Copyright 2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#include "loop_split.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -SC_DECL_PASS_INFO(loop_splitter, SC_PASS_DEPENDS_ON(), - SC_PASS_REQUIRE_STATE(CONST_FOLDED, IR_SIMPLIFIED), - SC_PASS_REQUIRE_NOT_STATE(), SC_PASS_SET_STATE(), - SC_PASS_UNSET_STATE()); - -class loop_spiltter_impl_t : public ir_visitor_t { -public: - using ir_visitor_t::dispatch; - using ir_visitor_t::visit; - - inline int64_t divide_and_ceil(int64_t x, int64_t y) { - return (x + y - 1) / y; - } - - // not interested in any exprs - expr_c dispatch(expr_c v) override { return v; } - - stmt_c visit(for_loop_c v) override { - auto vv = ir_visitor_t::visit(std::move(v)).static_as(); - // New range for this for loop - stmt new_body; - auto for_range_new_end = [&](int64_t new_end) -> stmt_c { - const auto end = vv->iter_end_.dyn_as(); - if (end.defined() && (get_const_as_int(end) > new_end)) { - auto iter_end = - builder::make_constant({new_end}, vv->iter_end_->dtype_); - return copy_attr( - *vv, make_stmt(vv->var_, vv->iter_begin_, iter_end, - vv->step_, std::move(new_body), - vv->incremental_, vv->kind_, - vv->num_threads_)); - } - return vv; - }; - auto for_range_new_beg = [&](int64_t new_beg) -> stmt_c { - const auto begn = vv->iter_begin_.dyn_as(); - const auto step = vv->step_.dyn_as(); - if (begn.defined() && step.defined() && - (get_const_as_int(begn) < new_beg)) { - auto old_beg = get_const_as_int(begn); - auto old_stp = get_const_as_int(step); - auto n = divide_and_ceil(new_beg - old_beg, old_stp); - auto iter_beg = builder::make_constant({n * old_stp + old_beg}, - vv->iter_begin_->dtype_); - return copy_attr(*vv, make_stmt( - vv->var_, iter_beg, vv->iter_end_, vv->step_, - std::move(new_body), vv->incremental_, - vv->kind_, vv->num_threads_)); - } - return vv; - }; - // For now, the only case we experienced is one for and one if with one - // case, thus only match for loop with one if (loop var cmp) stmt - auto cond = - vv->body_.cast() - .filter([vv](const stmts &v) { - return vv->kind_ == for_type::NORMAL && v->seq_.size() == 1; - }) - .map([](const stmts &v) { return v->seq_.back().as(); }) - .filter([&new_body](const if_else &v) { - new_body = v->then_case_; - return !v->else_case_.defined(); - }) - .map([](const if_else &v) { return v->condition_.dyn_as(); }) - .filter([vv](const cmp &v) { - return v->l_.ptr_same(vv->var_) && v->r_.isa(); - }); - if (cond.has_value()) { - const auto c = cond.get(); - const auto n = get_const_as_int(c->r_.static_as()); - switch (c->node_type_) { - case sc_expr_type::cmp_lt: - return for_range_new_end(n); - case sc_expr_type::cmp_le: - return for_range_new_end(n + 1); - case sc_expr_type::cmp_gt: - return for_range_new_beg(n + 1); - case sc_expr_type::cmp_ge: - return for_range_new_beg(n); - default: - break; - } - } - // No match - return vv; - } -}; - -func_c loop_splitter_t::operator()(func_c f) { - loop_spiltter_impl_t v; - return v.dispatch(f); -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/transform/loop_split.hpp b/legacy/core/src/compiler/ir/transform/loop_split.hpp deleted file mode 100644 index 48429da44..000000000 --- a/legacy/core/src/compiler/ir/transform/loop_split.hpp +++ /dev/null @@ -1,38 +0,0 @@ -/******************************************************************************* - * Copyright 2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_LOOP_SPLIT_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_LOOP_SPLIT_HPP - -#include "../function_pass.hpp" - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -class loop_splitter_t : public function_pass_t { -public: - func_c operator()(func_c f) override; - SC_DECL_PASS_INFO_FUNC(); -}; - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/transform/loop_transform.cpp b/legacy/core/src/compiler/ir/transform/loop_transform.cpp deleted file mode 100644 index 3e23b54a6..000000000 --- a/legacy/core/src/compiler/ir/transform/loop_transform.cpp +++ /dev/null @@ -1,877 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#include - -#include "../builder.hpp" -#include "../ir_comparer.hpp" -#include "../pass/ir_copy_internal.hpp" -#include "../visitor.hpp" -#include "auto_cast.hpp" -#include "constant_fold.hpp" -#include "tensor_shrink.hpp" -#include -#include -#include -#include -#include -#include - -SC_MODULE(ir.loop_transform) -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -int64_t get_const_as_int(const constant_c &c) { - assert(!c.get()->is_vector()); - switch (get_type_category(c->dtype_)) { - case CATE_INT: - return c->value_[0].s64; - break; - case CATE_UINT: - return c->value_[0].u64; - break; - default: - COMPILE_ASSERT(0, "Bad type to get int from const: " << c); - return 0; - } -} - -int64_t get_expr_as_int(const expr_c &e) { - constant_c c = e.checked_as(); - return get_const_as_int(c); -} - -static bool is_constant_for(for_loop_node_t *loop) { - return loop->iter_begin_.isa() && loop->iter_end_.isa() && - loop->step_.isa(); -} - -static void get_constant_from_for_loop(for_loop_node_t *_loop, int64_t &min, - int64_t &max, int64_t &step, - bool check_step = true) { - COMPILE_ASSERT(is_constant_for(_loop), - "Only support constant for loops for for-loop-transforms: " - << _loop->node_ptr_from_this()); - step = get_const_as_int(_loop->step_.as()); - if (check_step) { - COMPILE_ASSERT(step == 1, "for-loop-transforms only support step=1: " - << _loop->node_ptr_from_this()); - } - min = get_const_as_int(_loop->iter_begin_.as()); - max = get_const_as_int(_loop->iter_end_.as()); - COMPILE_ASSERT( - max >= min, - "for-loop-transforms: the begin should be less than or eq to end: " - << _loop->node_ptr_from_this()); -} - -//////////////////////// -// New loop transform starts here -//////////////////////// - -class var_inplace_replacer_t : public ir_inplace_visitor_t { -public: - var_inplace_replacer_t(std::unordered_map *remap) - : remap_(remap) {} - std::unordered_map *remap_; - using ir_inplace_visitor_t::dispatch_impl; - using ir_inplace_visitor_t::visit_impl; - expr visit_impl(var v) override { - auto itr = remap_->find(v.get()); - if (itr != remap_->end()) { - changed_ = true; - return itr->second; - } - return v; - } - - // sync with loop transform - void transform_shrink_info(const expr &v) { - if (v->attr_ && v->attr_->has_key(tensor_shrinker_attrs::should_shrink)) { - auto &shrink_info = v->attr_->get( - tensor_shrinker_attrs::should_shrink); - std::vector ret; - ir_inplace_visitor_t::dispatch_expr_vector(shrink_info.base_, ret); - ir_inplace_visitor_t::dispatch_expr_vector(shrink_info.shape_, ret); - } - } - - expr visit_impl(tensor v) override { - ir_inplace_visitor_t::visit_impl(v); - transform_shrink_info(v); - return v; - } - - expr visit_impl(tensorptr v) override { - ir_inplace_visitor_t::visit_impl(v); - transform_shrink_info(v); - return v; - } -}; - -bool for_loop_node_t::isvalid() const { return var_.defined(); } - -for_loop make_const_for(var v, int64_t min, int64_t max, stmt &&body_) { - auto ret = make_stmt( - std::move(v), make_expr(min, v->dtype_), - make_expr(max, v->dtype_), - make_expr(int64_t(1), v->dtype_), std::move(body_), true, - for_type::NORMAL); - // set parent node - add_parent_node(ret->body_, ret); - return ret; -} - -// copies the IR and check if there is any static var -class ir_copier_with_unroll_check_t : public ir_copier_impl_t { - using ir_copier_impl_t::dispatch; - using ir_copier_impl_t::ir_copier_impl_t; - - void view(define_c v) override { - COMPILE_ASSERT(v->linkage_ == linkage::local, - "Only allow local variables in unroll, got: " << v); - replace_map_[v->var_] = expr(); - ir_copier_impl_t::view(std::move(v)); - } - - void view(for_loop_c v) override { - // Replace var defined by a for_loop - replace_map_[v->var_] = expr(); - ir_copier_impl_t::view(std::move(v)); - } -}; - -static int find_ths_and_then_remove(for_loop_node_t *ths, const stmts &parent) { - for (auto v = parent->seq_.begin(); v != parent->seq_.end(); v++) { - if ((*v).get() == ths) { - int pos = static_cast(v - parent->seq_.begin()); - parent->seq_.erase(v); - return pos; - } - } - COMPILE_ASSERT(false, "Cannot find the axis in the parent"); - return -1; -} - -void for_loop_node_t::unroll(uint64_t factor, const stmt &parent) { - COMPILE_ASSERT(isvalid(), "Transforming an invalid for-loop"); - if (factor == 1) { - return; - } - auto this_reserve = node_ptr_from_this(); - bool remove = false; - bool has_remainder = true; - int loop_idx = -1; - int64_t min, max, step; - bool is_const = is_constant_for(this); - if (factor == 0) { - COMPILE_ASSERT(parent.defined() && parent.isa(), - "parent is not defined or is not stmts."); - COMPILE_ASSERT(is_const, "Need const for-loop to fully unroll"); - loop_idx = find_ths_and_then_remove(this, parent.static_as()); - remove = true; - } - if (is_const) { - get_constant_from_for_loop(this, min, max, step, false); - int64_t loop_len = max - min; - if (factor == 0) { - factor = utils::divide_and_ceil(loop_len, step); - } - has_remainder = - loop_len % (factor * step) != 0 || loop_len < ((int64_t)factor * step); - } - // make new variables - var oldvar = var_.checked_as(); - var newvar = var_->remake().static_as(); - expr step_x_factor = is_const ? (step * factor) : step_ * factor; - newvar->name_ += "_u"; - - // make the new unrolled body - stmts seq = make_stmt(std::vector()); - // make the indexing variable that can be shared by all unrolled stmts - expr newidx = oldvar->remake(); - // a loop var when reserves loop, a constant node when unrolls all. - expr replace_value; - if (!remove) { - seq->seq_.emplace_back(builder::make_var_tensor_def_unattached( - newidx, linkage::local, newvar * step_x_factor + iter_begin_)); - replace_value = newidx; - } else { - replace_value = make_expr( - get_const_as_int(iter_begin_.static_as()), oldvar->dtype_); - } - // unroll by the factor - for (uint64_t i = 0; i < factor; i++) { - std::unordered_map replace_map; - ir_copier_with_unroll_check_t copier(replace_map, false); - replace_map[oldvar] = replace_value + (is_const ? i * step : i * step_); - seq->seq_.emplace_back(copier.copy(body_)); - } - - // the new loop length, constant folded and pure expr versions - if (!remove) { - uint64_t c_newloop_len = is_const ? (max - min) / (step * factor) : 0; - expr newloop_len = is_const - ? c_newloop_len - : ((iter_end_ - iter_begin_) / (step_ * factor)); - // if has remainder, should put the seq into an if-else - if (has_remainder) { - // make the remainder loop, first make the lower bound of the loop - // var - expr begins = is_const ? (c_newloop_len * (step * factor) + min) - : (newloop_len * (step_ * factor) + iter_begin_); - auto remainder = builder::make_for_loop_unattached( - oldvar, begins, iter_end_, step_, body_, incremental_, - kind_ != for_type::PARALLEL ? kind_ : for_type::NORMAL); - // put the remainder loop and the unrolled body into the if-else - stmts seq_remainder = make_stmt(std::vector()); - seq_remainder->seq_.emplace_back(std::move(remainder)); - auto if_check = builder::make_if_else_unattached( - newvar < newloop_len, std::move(seq), std::move(seq_remainder)); - seq = make_stmt(std::vector{std::move(if_check)}); - } - // update the loop vars - body_ = std::move(seq); - - iter_begin_ = 0; - if (has_remainder) { - iter_end_ = is_const ? (c_newloop_len + 1) : (newloop_len + 1); - } else { - iter_end_ = is_const ? (c_newloop_len) : (newloop_len); - } - step_ = 1; - var_ = newvar; - } else { - assert(loop_idx != -1); - auto &pseq = parent.static_as()->seq_; - pseq.insert(pseq.begin() + loop_idx, seq->seq_.begin(), seq->seq_.end()); - } -} - -static for_loop build_inner_for(for_loop_node_t *ths, int64_t min, - int64_t block, node_ptr_map *node_remap) { - // make new variables - var varptr = ths->var_.checked_as(); - // remake a new var to replace old one. Call "remake" to avoid infinite - // recursion in var_inplace_replacer after fuse(): - // cause: in fuse(), v => v / 123, we use v / 123 multiple times - // in split, we inplace change (v * 123) => (v * 321 + 1) / 123, but (v * - // 123) is used many times, we will recursively replace (v * 321 + 1) / 123 - // => ((v * 321 + 1) * 321 + 1) / 123 - // thus we need to remake v - ths->var_ = ths->var_->remake(); - std::string &varname = ths->var_.static_as()->name_; - std::string oldname = std::move(varname); - varname = oldname + "_0outer"; - - var vin = make_expr(ths->var_->dtype_, oldname + "_0inner"); - - // old iter variable is mapped to outer * block + inner - expr remapped = - ths->var_ * make_expr(block, ths->var_->dtype_) + vin; - if (min != 0) { - remapped = remapped + make_expr(min, ths->var_->dtype_); - } - std::unordered_map remap = {{varptr.get(), remapped}}; - if (node_remap) { - node_remap->insert(std::make_pair(varptr.impl, remapped.impl)); - } - var_inplace_replacer_t pass(&remap); - pass.dispatch_impl(ths->body_); - - // make inner loop - auto inner_for = make_const_for(vin, 0, block, std::move(ths->body_)); - return inner_for; -} - -for_loop for_loop_node_t::split(int64_t block, node_ptr_map *node_remap) { - COMPILE_ASSERT(isvalid(), "Transforming an invalid for-loop"); - int64_t min, max, step; - get_constant_from_for_loop(this, min, max, step); - int64_t loop_len = max - min; - COMPILE_ASSERT( - loop_len % block == 0 && loop_len >= block, - "The loop length " - << loop_len - << " should be divisible of and larger than the block size " - << block); - int64_t outer_len = loop_len / block; - - // set outer loop - iter_begin_ = make_expr(int64_t(0), var_->dtype_); - iter_end_ = make_expr(outer_len, var_->dtype_); - - auto inner_for = build_inner_for(this, min, block, node_remap); - - body_ = make_stmt(std::vector({inner_for})); - // set parent node - add_parent_node(inner_for, body_); - add_parent_node(body_, node_ptr_from_this()); - // copy loop binding axis hint - copy_binding_axis_hint(this, inner_for.get()); - return inner_for; -} - -for_loop for_loop_node_t::split_on_num_threads(int64_t num_groups, - node_ptr_map *node_remap) { - COMPILE_ASSERT(isvalid(), "Transforming an invalid for-loop"); - int64_t min, max, step; - get_constant_from_for_loop(this, min, max, step); - COMPILE_ASSERT(min == 0 && step == 1, "Only support begin is 0 and step is 1") - COMPILE_ASSERT(max == num_threads_, "Only support num_iters == num_threads") - int64_t ori_num_threads = num_threads_; - COMPILE_ASSERT(ori_num_threads % num_groups == 0, - "The num_threads " << ori_num_threads - << " should be divisible by num_groups " - << num_groups); - num_threads_ = ori_num_threads / num_groups; - iter_end_ = make_expr(uint64_t(num_threads_), var_->dtype_); - - auto inner_for = build_inner_for(this, min, num_groups, node_remap); - inner_for->num_threads_ = num_groups; - inner_for->kind_ = for_type::PARALLEL; - if (attr_ && attr_->has_key(stmt_attr_key::parallel_loop_balanced)) { - inner_for->attr()[stmt_attr_key::parallel_loop_balanced] = - attr()[stmt_attr_key::parallel_loop_balanced]; - } - body_ = make_stmt(std::vector({inner_for})); - // set parent node - add_parent_node(inner_for, body_); - add_parent_node(body_, node_ptr_from_this()); - // copy loop binding axis hint - copy_binding_axis_hint(this, inner_for.get()); - return inner_for; -} - -for_loop get_inner_for_loop(const for_loop_node_t *f) { - const for_loop_node_t *cur = f; - if (cur->body_.isa()) { - auto stmtlist = cur->body_.static_as(); - // if it is a basic block with one statement and it is a for-loop - if (stmtlist->seq_.size() == 1 && stmtlist->seq_.at(0).isa()) { - return stmtlist->seq_.at(0).static_as(); - } - } else if (cur->body_.isa()) { - return cur->body_.static_as(); - } - return for_loop(); -} - -for_loop get_last_loop_in_body(const stmt &body) { - if (body.isa()) { - auto stmtlist = body.static_as(); - if (!stmtlist->seq_.empty()) { - stmt last_stmt; - for (int64_t i = stmtlist->seq_.size() - 1; i >= 0; --i) { - last_stmt = stmtlist->seq_[i]; - if (!stmtlist->seq_[i].isa() || - !stmtlist->seq_[i].static_as()->seq_.empty()) - break; - } - if (last_stmt.isa()) { - return last_stmt.static_as(); - } else { - return get_last_loop_in_body(last_stmt); - } - } - } else if (body.isa()) { - return body.static_as(); - } - return for_loop(); -} - -for_loop for_loop_node_t::fuse(const for_loop &ax, node_ptr_map *node_remap) { - COMPILE_ASSERT(ax->isvalid(), "Transforming an invalid for-loop: ax"); - COMPILE_ASSERT(isvalid(), "Transforming an invalid for-loop: this"); - if (!get_inner_for_loop(this).ptr_same(ax)) { - SC_MODULE_INFO << "We can only fuse the next inner loop"; - return this->node_ptr_from_this().static_as(); - } - static std::atomic fuse_count(0); - expr min1, max1, step1; - expr min2, max2, step2; - auto get_expr_from_for_loop = [](for_loop_node_t *_loop, expr &min, expr &max, - expr &step) { - min = _loop->iter_begin_; - max = _loop->iter_end_; - step = _loop->step_; - }; - get_expr_from_for_loop(this, min1, max1, step1); - get_expr_from_for_loop(ax.get(), min2, max2, step2); - expr loop_len1 = max1 - min1; - expr loop_len2 = max2 - min2; - expr outer_len = loop_len1 * loop_len2; - var var1 = var_.checked_as(), var2 = ax->var_.checked_as(); - COMPILE_ASSERT(var_->dtype_ == ax->var_->dtype_, - "The fused for loop variables should have the same types, got " - << var_->dtype_ << " and " << var_->dtype_); - // make new variables - var vout = make_expr( - var_->dtype_, std::string("fused_0") + var1->name_ + "__" + var2->name_ + - "_" + std::to_string(fuse_count++)); - - std::unordered_map var_remap; - // old iter variable is mapped to vout / loop_len2 and vout % loop_len2 - expr outer = vout / loop_len2; - outer = outer + min1; - outer = do_cast_and_fold(outer); - var_remap.insert(std::make_pair(var1.get(), outer)); - if (node_remap) { - node_remap->insert(std::make_pair(var1.impl, outer.impl)); - } - expr inner = vout % loop_len2; - inner = inner + min2; - inner = do_cast_and_fold(inner).remove_const(); - - var_remap.insert(std::make_pair(var2.get(), inner)); - if (node_remap) { - node_remap->insert(std::make_pair(var2.impl, inner.impl)); - } - var_inplace_replacer_t pass(&var_remap); - auto newbody = pass.dispatch_impl(ax->body_); - - var_ = vout; - iter_begin_ = make_expr(int64_t(0), var1->dtype_); - iter_end_ = do_cast_and_fold(outer_len).remove_const(); - step_ = make_expr(int64_t(1), var1->dtype_); - - // redirect parent node - add_parent_node(newbody, node_ptr_from_this()); - - body_ = std::move(newbody); - - ax->var_ = expr(); // invalidate ax - - // fuse loop binding axis hint - fuse_binding_axis_hint(this, ax.get()); - return for_loop(shared_from_this()); -} - -class loop_replacer_t : public ir_inplace_visitor_t { -public: - using ir_inplace_visitor_t::dispatch_impl; - using ir_inplace_visitor_t::visit_impl; - for_loop_node_t *old_; - for_loop new_; - bool replaced = false; - loop_replacer_t(for_loop_node_t *old, for_loop new_) - : old_(old), new_(std::move(new_)) {} - stmt visit_impl(for_loop v) override { - if (v.get() == old_) { - replaced = true; - return new_; - } - return v; - } - - expr dispatch_impl(expr e) override { - // no need to look into expr - return e; - } -}; - -class loop_parallel_replacer_t : public ir_inplace_visitor_t { -private: - bool ignore_nested_parallel_; - -public: - loop_parallel_replacer_t(bool ignore_nested_parallel = false) - : ignore_nested_parallel_(ignore_nested_parallel) {} - using ir_inplace_visitor_t::dispatch_impl; - using ir_inplace_visitor_t::visit_impl; - stmt visit_impl(for_loop v) override { - dispatch_impl(v->body_); - if (v->kind_ == for_type::PARALLEL) { - if (v->num_threads_ > 0) { - if (!ignore_nested_parallel_) { - v->kind_ = for_type::NORMAL; - v->num_threads_ = 0; - } - } else { - v->kind_ = for_type::NORMAL; - } - } - return v; - } - expr dispatch_impl(expr e) override { return e; } -}; - -void for_loop_node_t::reorder(stmt parent, std::vector &&ax) { - COMPILE_ASSERT(!ax.empty(), "The number of axises to reorder should > 0"); - for_loop cur = for_loop(shared_from_this()); - stmt inner_body; - for (unsigned i = 0; i < ax.size(); i++) { - COMPILE_ASSERT(cur.defined(), "Bad number of axises to reorder. Got " - << ax.size() - << " to reorder, but only have " << i - << " nested for-loops"); - COMPILE_ASSERT(cur->isvalid(), "Transforming an invalid for-loop"); - inner_body = cur->body_; - COMPILE_ASSERT( - std::find_if(ax.begin(), ax.end(), - [cur](for_loop &v) { return v.ptr_same(cur); }) != - ax.end(), - "Cannot find axis " << cur->var_ << " in the given axises to reorder"); - cur = get_inner_for_loop(cur.get()); - } - - // redirect parent node - add_parent_node(inner_body, ax.back()); - - ax.back()->body_ = std::move(inner_body); - cur = ax.back(); - if (ax.size() > 1) { - for (int64_t i = ax.size() - 2; i >= 0; i--) { - ax.at(i)->body_ = - make_stmt(std::vector({std::move(cur)})); - cur = ax.at(i); - } - } - loop_replacer_t replacer(this, ax.front()); - replacer.dispatch_impl(std::move(parent)); - COMPILE_ASSERT(replacer.replaced, - "Cannot find the for-loop to replace in the parent stmt"); -} - -void flatten_stmt_and_append(const stmt &s, std::vector &out) { - if (s.isa()) { - for (auto &v : s.static_as()->seq_) { - out.push_back(v); - } - } else { - out.push_back(s); - } -} - -static void check_loop_for_merge(for_loop_node_t *ths, for_loop_node_t *ax) { - COMPILE_ASSERT(ths->isvalid() && ax->isvalid(), - "Invalid for-loop. It has been fused or merged"); - COMPILE_ASSERT(ax != ths, "The axis to merge should not be \'this\'"); -} - -static bool is_loop_range_same(for_loop_node_t *ths, for_loop_node_t *ax) { - ir_comparer ircmp(false, true, true); - ircmp.set_expr_mapping(ths->var_, ax->var_); - return ths->incremental_ == ax->incremental_ && - ths->iter_begin_->equals(ax->iter_begin_, ircmp) && - ths->iter_end_->equals(ax->iter_end_, ircmp) && - ths->step_->equals(ax->step_, ircmp); -} - -static void find_ths_and_ax_then_remove(for_loop_node_t *ths, - const stmt &parent, - for_loop_node_t *ax) { - COMPILE_ASSERT(parent.isa(), "The parent should be an stmts_node_t"); - auto s = parent.static_as(); - constexpr size_t invalid = std::numeric_limits::max(); - size_t this_in_parent = invalid, ax_in_parent = invalid; - for (size_t i = 0; i < s->seq_.size(); i++) { - auto &v = s->seq_.at(i); - if (v.get() == ths) { - this_in_parent = i; - } else if (v.get() == ax) { - ax_in_parent = i; - } - } - COMPILE_ASSERT(this_in_parent != invalid && ax_in_parent != invalid, - "Cannot find the axises in the parent"); - s->seq_.erase(s->seq_.begin() + ax_in_parent); // remove ax from parent -} - -static int *get_unroll_factor_attr(const for_loop_node_t *ths) { - if (ths->attr_) { - return ths->attr_->get_or_null(stmt_attr_key::unroll_loop); - } - return nullptr; -} - -static void do_merge(for_loop_node_t *ths, const stmt &parent, - const for_loop &ax) { - // now replace ax's variable with this->var_ - std::unordered_map remap; - remap.insert(std::make_pair(ax->var_.checked_as().get(), ths->var_)); - var_inplace_replacer_t pass(&remap); - auto axbody = pass.dispatch_impl(ax->body_); - ax->var_ = expr(); // invalidate ax - if (parent.defined()) { - find_ths_and_ax_then_remove(ths, parent, ax.get()); - } - // merge the bodies - std::vector newbody; - flatten_stmt_and_append(ths->body_, newbody); - flatten_stmt_and_append(axbody, newbody); - ths->body_ = make_stmt(std::move(newbody)); - if (auto unroll_ax = get_unroll_factor_attr(ax.get())) { - auto unroll_ths = get_unroll_factor_attr(ths); - if (!unroll_ths) { - ths->attr()[stmt_attr_key::unroll_loop] = *unroll_ax; - } else { - COMPILE_ASSERT(*unroll_ax == *unroll_ths, - "Different unroll factors when merging the loops: " - << *unroll_ths << "v.s." << *unroll_ax); - } - } -} - -for_loop for_loop_node_t::merge(const stmt &parent, const for_loop &ax) { - check_loop_for_merge(this, ax.get()); - COMPILE_ASSERT(is_loop_range_same(this, ax.get()), - "The ranges of the merged for-loops should be the same"); - do_merge(this, parent, ax); - return node_ptr_from_this().as(); -} - -for_loop for_loop_node_t::merge(stmt parent, for_loop ax, unsigned num_inner) { - for_loop_node_t *ths = this; - for_loop axis = std::move(ax); - for (unsigned i = 0; i < num_inner; i++) { - COMPILE_ASSERT(ths && axis.defined(), - "Merging " << num_inner << " inner loops, but have only " - << i << " loops in the IR"); - for_loop_node_t *next_ths = get_inner_for_loop(ths).get(); - for_loop next_ax = get_inner_for_loop(axis.get()); - ths->merge(parent, axis); - parent = ths->body_; - ths = next_ths; - axis = next_ax; - } - return node_ptr_from_this().static_as(); -} - -int for_loop_node_t::merge_all(stmt parent, for_loop ax) { - for_loop_node_t *ths = this; - for_loop axis = std::move(ax); - int num_loops = 0; - // skip axis binding check attr, usually marked in UT. - bool skip_ab_check = any_map_t::fetch_or_else( - ths->attr_.get(), stmt_attr_key::skip_axis_binding_check, false); - for (;;) { - // if there is no inner loops, break - if (!ths || !axis.defined()) { - break; - } - for_loop_node_t *next_ths = get_inner_for_loop(ths).get(); - for_loop next_ax = get_inner_for_loop(axis.get()); - - // check if the loops are valid - check_loop_for_merge(ths, axis.get()); - // if the loops are not mergable, break - if (!is_loop_range_same(ths, axis.get())) { - break; - } - // check binding axis - if (!skip_ab_check && !check_loop_binding_axis(ths, axis.get())) { - break; - } - do_merge(ths, parent, axis); - num_loops++; - parent = ths->body_; - ths = next_ths; - axis = next_ax; - } - return num_loops; -} - -void for_loop_node_t::parallel_merge(const stmt &parent, const for_loop &ax) { - COMPILE_ASSERT(isvalid(), "Invalid loop"); - COMPILE_ASSERT(ax->isvalid(), "Invalid loop"); - COMPILE_ASSERT(step_.isa() && - get_const_as_int(step_.static_as()) == 1, - "the step of this should be 1"); - COMPILE_ASSERT(ax->step_.isa() && - get_const_as_int(ax->step_.static_as()) == 1, - "the step of ax should be 1"); - find_ths_and_ax_then_remove(this, parent, ax.get()); - auto body1 = std::move(body_); - stmts body2 = - ax->body_.isa() - ? ax->body_.static_as() - : make_stmt(std::vector{std::move(ax->body_)}); - body2->seq_.insert( - body2->seq_.begin(), - builder::make_var_tensor_def_unattached( - ax->var_, linkage::local, var_ - iter_end_ + ax->iter_begin_)); - auto if_else = - builder::make_if_else_unattached(var_ < iter_end_, body1, body2); - body_ = builder::make_stmts_unattached({if_else}); - // set parent node - add_parent_node(body1, if_else); - add_parent_node(body2, if_else); - add_parent_node(if_else, body_); - add_parent_node(body_, node_ptr_from_this()); - iter_end_ = iter_end_ + ax->iter_end_ - ax->iter_begin_; - ax->var_ = expr(); -} - -void remove_parallel(stmt body, bool ignore_nested_parallel) { - loop_parallel_replacer_t replacer(ignore_nested_parallel); - replacer.dispatch_impl(std::move(body)); -} - -void remove_parallel(func_t body, bool ignore_nested_parallel) { - loop_parallel_replacer_t replacer(ignore_nested_parallel); - replacer.dispatch_impl(std::move(body)); -} - -std::vector collect_loops(const stmt &body) { - std::vector ret; - if (body.isa()) { - for (auto &smt : body.static_as()->seq_) { - if (smt.isa()) { - ret.push_back(smt.static_as()); - } - } - } else { - if (body.isa()) { - ret.push_back(body.static_as()); - } - } - return ret; -} - -std::vector collect_nested_loops(stmt body) { - std::vector ret; - auto cur = std::move(body); - bool outer_loop = true; - while (true) { - if (cur.isa()) { - auto stmts_cur = cur.static_as(); - bool continue_flag = false; - for (unsigned i = 0; i < stmts_cur->seq_.size(); i++) { - auto smt = stmts_cur->seq_[i]; - if (smt.isa()) { - ret.push_back(smt.static_as()); - cur = ret.back()->body_; - continue_flag = true; - break; - } else if (!ret.empty()) { - outer_loop = false; - break; - } - } - if (continue_flag && outer_loop) { - continue; - } - } else { - if (cur.isa()) { - ret.push_back(cur.static_as()); - cur = ret.back()->body_; - continue; - } - } - break; - } - return ret; -} - -static size_t collect_all_loops_helper(std::vector &ret, - const stmt &body) { - size_t collected = 0; - if (body.isa()) { - for (auto &smt : body.static_as()->seq_) { - collected += collect_all_loops_helper(ret, smt); - } - } else if (body.isa()) { - auto loop = body.static_as(); - ret.push_back(loop); - collected++; - collected += collect_all_loops_helper(ret, loop->body_); - } else if (body.isa()) { - auto cond = body.static_as(); - if (cond->then_case_.defined()) { - collected += collect_all_loops_helper(ret, cond->then_case_); - } - if (cond->else_case_.defined()) { - collected += collect_all_loops_helper(ret, cond->else_case_); - } - } - return collected; -} - -std::vector collect_all_loops(const stmt &body) { - std::vector ret; - collect_all_loops_helper(ret, body); - return ret; -} - -expr divide_and_ceil(const expr &, const expr &); -struct loop_normalize_visitor_t : public ir_consistent_visitor_t { - using ir_consistent_visitor_t::dispatch; - - expr_c dispatch(expr_c v) override { - // if nothing to replace, skip dispatching - if (replace_map_.empty()) { - return v; - } - return ir_consistent_visitor_t::dispatch(std::move(v)); - } - - stmt_c visit(for_loop_c v) override { - auto var = dispatch(v->var_); - auto begin = dispatch(v->iter_begin_); - auto end = dispatch(v->iter_end_); - auto step = dispatch(v->step_); - bool changed = !(var.ptr_same(v->var_) && begin.ptr_same(v->iter_begin_) && - end.ptr_same(v->iter_end_) && step.ptr_same(v->step_)); - bool is_normalized = v->kind_ != for_type::PARALLEL || - (begin.cast() - .filter([](const constant_c &v) { - return get_const_as_int(v) == 0; - }) - .has_value() && - step.cast() - .filter([](const constant_c &v) { - return get_const_as_int(v) == 1; - }) - .has_value()); - if (!is_normalized) { - auto new_beg = builder::make_constant({UINT64_C(0)}, begin->dtype_); - auto new_end = - divide_and_ceil((end - begin).remove_const(), step.remove_const()); - auto new_step = builder::make_constant({UINT64_C(1)}, step->dtype_); - auto new_itr = copy_attr( - *var, builder::make_var(var->dtype_, var.static_as()->name_)); - auto replace_itr = begin + new_itr * step; - replace_map_[var] = replace_itr; - auto body = dispatch(v->body_); - return copy_attr(*v, builder::make_for_loop_unattached( - new_itr, new_beg, new_end, new_step, body, - v->incremental_, v->kind_, v->num_threads_)); - } - - auto body = dispatch(v->body_); - changed |= !body.ptr_same(v->body_); - if (changed) { - stmt_c newv = - copy_attr(*v, builder::make_for_loop_unattached( - var, begin, end, step, body, v->incremental_, - v->kind_, v->num_threads_)); - return newv; - } - return v; - } -}; - -for_loop_c normalize_parallel_for_loop(const for_loop_c &loop) { - loop_normalize_visitor_t v; - return v.visit(loop).static_as(); -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/transform/loop_transform.hpp b/legacy/core/src/compiler/ir/transform/loop_transform.hpp deleted file mode 100644 index 19db96ce3..000000000 --- a/legacy/core/src/compiler/ir/transform/loop_transform.hpp +++ /dev/null @@ -1,86 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_LOOP_TRANSFORM_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_LOOP_TRANSFORM_HPP -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -/** - * Removes redundant loops with parallel attribute. Will reserve the outmost - * one loop with parallel. - * @param body the stmts for parallel remove - * @param ignore_nested_parallel decides whether finally remove the parallel - * loop which also owns `num_threads_` field - * */ -void remove_parallel(stmt body, bool ignore_nested_parallel = false); - -void remove_parallel(func_t body, bool ignore_nested_parallel = false); - -/** - * Collect loops inside this body. Won't recurisvely look into loop body. - * For example. - * for() # loop 1 - * for() # loop 2 - * for() # loop 3 - * for() # loop 4 - * Only loop 1 and 3 are returned. - * @param body the stmts for collection - * */ -std::vector collect_loops(const stmt &body); - -/** - * Collect nested loops inside this body. - * For example. - * for() # loop 1 - * for() # loop 2 - * for() # loop 3 - * for() # loop 4 - * - * Only loop 1 and 2 are returned because loop 3 and 4 are not nested loop. - * - * @param body the stmts for collection - * */ -std::vector collect_nested_loops(stmt body); - -/** - * Collect all loops inside this body recursively - * */ -std::vector collect_all_loops(const stmt &body); - -// get inner for_loop -for_loop get_inner_for_loop(const for_loop_node_t *f); - -// get last for_loop in body -for_loop get_last_loop_in_body(const stmt &body); - -/** - * Normalize a parallel loop and the parallel loops in the body to make the - * iter_begin=0 and step=1. If all the loops are already normalized, return the - * loop without changing. Otherwise it returns the new loop and it will not - * change the given loop. - */ -for_loop_c normalize_parallel_for_loop(const for_loop_c &loop); - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/transform/loop_unroll.cpp b/legacy/core/src/compiler/ir/transform/loop_unroll.cpp deleted file mode 100644 index 2395973da..000000000 --- a/legacy/core/src/compiler/ir/transform/loop_unroll.cpp +++ /dev/null @@ -1,109 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#include "loop_unroll.hpp" -#include -#include -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -SC_DECL_PASS_INFO(loop_unroller, SC_PASS_DEPENDS_ON(tensor_init), - SC_PASS_REQUIRE_STATE(CONST_FOLDED), - SC_PASS_REQUIRE_NOT_STATE(), SC_PASS_SET_STATE(), - SC_PASS_UNSET_STATE(IR_SIMPLIFIED)); - -class loop_unroller_impl_t : public ir_visitor_t { -public: - using ir_visitor_t::dispatch; - expr_c dispatch(expr_c v) override { return v; } - stmt_c visit(stmts_c v) override { - auto ret = ir_visitor_t::visit(v); - std::vector to_unroll; - for (auto &s : ret.static_as()->seq_) { - if (s.isa()) { - auto &attr = s.static_as()->attr_; - if (attr && attr->has_key(stmt_attr_key::unroll_loop)) { - to_unroll.emplace_back(s.static_as()); - } - } - } - if (to_unroll.empty()) { - return ret; - } - stmts writable = (ret.ptr_same(v) ? ret->remake() : ret).static_as(); - for (auto &f : to_unroll) { - f->unroll(f->attr_->get(stmt_attr_key::unroll_loop), writable); - } - return constant_folder_t()(writable); - } - - stmt_c visit(for_loop_c v) override { - // check of the loop is most inner loop - if (!is_inner_loop_.empty()) { - is_inner_loop_.back() = false; - } - is_inner_loop_.push_back(true); - auto vv = ir_visitor_t::visit(std::move(v)) - .static_as() - .remove_const(); - // fully unroll the loop if loop is most inner - // and repeat time <= 8, body size <= 64, unroll size <= 300 - auto &end = vv->iter_end_; - auto &step = vv->step_; - auto &begin = vv->iter_begin_; - auto is_most_inner = is_inner_loop_.back(); - if (vv->kind_ == for_type::NORMAL && is_most_inner && - begin.isa() && end.isa() && step.isa()) { - auto e = end.static_as()->value_[0].u64; - auto s = step.static_as()->value_[0].u64; - auto b = begin.static_as()->value_[0].u64; - auto size = vv->body_.static_as()->seq_.size(); - auto repeat = utils::divide_and_ceil(e - b, s); - if (repeat <= 8 && size <= 64 && (repeat * size) <= 300) { - vv->attr()[stmt_attr_key::unroll_loop] = 0; - } - } - // end of loop - is_inner_loop_.pop_back(); - return vv; - } - -private: - std::vector is_inner_loop_; -}; - -func_c loop_unroller_t::operator()(func_c f) { - loop_unroller_impl_t impl; - return impl.dispatch(f); -}; - -stmt_c loop_unroller_t::operator()(stmt_c f) { - loop_unroller_impl_t impl; - return impl.dispatch(std::move(f)); -}; - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/transform/loop_unroll.hpp b/legacy/core/src/compiler/ir/transform/loop_unroll.hpp deleted file mode 100644 index fa7aececc..000000000 --- a/legacy/core/src/compiler/ir/transform/loop_unroll.hpp +++ /dev/null @@ -1,43 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_LOOP_UNROLL_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_LOOP_UNROLL_HPP - -#include "../function_pass.hpp" -#include "../sc_function.hpp" - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -/** - * Unroll loops with attr[stmt_attr_key::unroll_loop]=true - * */ -class loop_unroller_t : public function_pass_t { -public: - func_c operator()(func_c f) override; - stmt_c operator()(stmt_c f); - SC_DECL_PASS_INFO_FUNC(); -}; - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/transform/module_globals_resolve.cpp b/legacy/core/src/compiler/ir/transform/module_globals_resolve.cpp deleted file mode 100644 index 1115bc5a6..000000000 --- a/legacy/core/src/compiler/ir/transform/module_globals_resolve.cpp +++ /dev/null @@ -1,536 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#include "module_globals_resolve.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -SC_MODULE(pass.module_globals_resolve); -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -SC_DECL_PASS_INFO(module_globals_resolver, - SC_PASS_DEPENDS_ON(closurizer_cpu, kernel_lowering_cpu, - dynamic_parallel_transform), - SC_PASS_REQUIRE_STATE(), SC_PASS_REQUIRE_NOT_STATE(), - SC_PASS_SET_STATE(), SC_PASS_UNSET_STATE()); - -static bool is_expr_nullptr(const expr &e) { - if (e->dtype_.is_pointer() && e.isa()) { - return e.checked_as()->value_.front().u64 == 0; - } - return false; -} - -static const char *shared_const_handle_name = "__shared_const_handle"; - -class module_globals_resolver_impl_t : public ir_visitor_t { -public: - using ir_visitor_t::dispatch; - std::unordered_map *map; - expr current_base; - expr current_rtl_ctx; - bool is_top_level = true; - uintptr_t absolute_base_ = 0; - // change the global symbol to a local variable - std::vector global_symbol_local_def_; - std::unordered_map global_symbol_replace_map_; - std::vector shared_const_base_tsr_; - - expr_c visit(call_c v) override { - func_t the_func = std::dynamic_pointer_cast(v->func_); - expr the_expr; - if (!the_func) { - the_expr = expr(std::dynamic_pointer_cast(v->func_)); - assert(the_expr.defined() && the_expr->attr_->has_key("prototype")); - the_func = the_expr->attr_->get("prototype"); - } - auto itr = map->find(the_func->name_); - if (itr == map->end()) { - // if is parallel-call function - if (v->func_ == get_parallel_call_with_env_func(true) || - v->func_ == get_parallel_call_with_env_func(false)) { - std::vector ret; - dispatch_expr_vector(v->args_, ret); - assert(ret.size() == 8UL); - ret[2] = current_rtl_ctx; - ret[3] = current_base; - return copy_attr(*v, make_expr(v->func_, ret)); - } else if (v->func_->attr_ && v->func_->attr_->get_or_else( - "is_brgemm_func_with_stream", false)) { - COMPILE_ASSERT(!v->args_.empty() && - v->args_.back()->dtype_ == datatypes::pointer, - "The last arg of brgemm function should be a " - "pointer, got " - << v); - if (is_expr_nullptr(v->args_.back())) { - std::vector newargs; - dispatch_expr_vector(v->args_, newargs); - newargs.back() = current_rtl_ctx; - return copy_attr(*v, builder::make_call(the_func, newargs)); - } - } - return ir_visitor_t::visit(v); - } - std::vector ret; - dispatch_expr_vector(v->args_, ret); - ret.insert(ret.begin(), 2, expr()); - ret[0] = current_rtl_ctx; - ret[1] = current_base; - if (the_expr.defined()) { - the_expr->attr_->set("prototype", itr->second->decl_); - return copy_attr(*v, make_expr(the_expr, ret)); - } - return copy_attr(*v, builder::make_call(itr->second->decl_, ret)); - } - - expr_c visit(constant_c v) override { - if (v->dtype_ == datatypes::pointer && v->value_[0].u64 == 0) { - if (any_map_t::fetch_or_else(v->attr_.get(), "auto_fill_stream", false)) { - return current_rtl_ctx; - } - if (any_map_t::fetch_or_else(v->attr_.get(), "auto_fill_module_data", - false)) { - return builder::make_reinterpret(current_base, datatypes::pointer); - } - } - return v; - } - expr_c visit(func_addr_c v) override { - auto itr = map->find(v->func_->name_); - if (itr == map->end()) { - return ir_visitor_t::visit(v); - } - return copy_attr(*v, builder::make_func_addr(itr->second->decl_)); - } - - expr_c get_or_create_new_var(const expr_c &old) { - auto itr = global_symbol_replace_map_.find(old); - if (itr != global_symbol_replace_map_.end()) { - return itr->second; - } - auto ret = old->remake(); - // if using absolute base, the module_global_offset is the pointer of - // the global var - if (absolute_base_ && ret.isa()) { - auto &attr = ret->attr_->get_any(attr_keys::module_global_offset); - void *ptr = reinterpret_cast(attr.get() + absolute_base_); - attr = ptr; - } - global_symbol_local_def_.emplace_back(ret); - global_symbol_replace_map_.insert(std::make_pair(old, ret)); - return ret; - } - - // for var and tensor, if they are global, change them to local definitions - expr_c visit(tensor_c v) override { - if (v->attr_ && v->attr_->has_key(attr_keys::module_global_offset)) { - return get_or_create_new_var(v); - } - return ir_visitor_t::visit(v); - } - - expr_c visit(var_c v) override { - if (v->attr_ && v->attr_->has_key(attr_keys::module_global_offset)) { - return get_or_create_new_var(v); - } - return ir_visitor_t::visit(v); - } - - stmt_c visit(define_c v) override { - if (!v->init_.defined()) { - if (auto buf = any_map_t::fetch_or_else< - std::shared_ptr>( - v->var_->attr_.get(), attr_keys::shared_const, nullptr)) { - auto idx = v->attr_->get("shared_const_handle_idx"); - auto newvar = dispatch(v->var_); - COMPILE_ASSERT(!v->init_.defined(), - "Bad shared const tensor with init"); - return copy_attr(*v, - builder::make_var_tensor_def_unattached( - newvar, linkage::local, - builder::tensor_ptr(shared_const_base_tsr_.at(idx), - {buf->offset_}))); - } - } - return ir_visitor_t::visit(v); - } - - // converts the global vars used in function to local vars defined in - // top-level scope - stmt_c visit(stmts_c v) override { - if (is_top_level) { - is_top_level = false; - auto ret = ir_visitor_t::visit(v).checked_as(); - if (global_symbol_local_def_.empty()) { - return ret; - } - std::vector seq; - for (auto &v : global_symbol_local_def_) { - expr init; - if (v.isa()) { - auto anyoffset = v->attr().get_any(attr_keys::module_global_offset); - if (auto poffset = anyoffset.get_or_null()) { - auto offset = *poffset; - if (absolute_base_) { - init = - make_expr((uint64_t)(absolute_base_ + offset), - datatypes::s8.get_pointerof()); - } else { - init = builder::tensor_ptr(current_base, {offset}); - } - } else { - init = make_expr((uint64_t)(anyoffset.get()), - datatypes::s8.get_pointerof()); - } - } - seq.emplace_back( - builder::make_var_tensor_def_unattached(v, linkage::local, init)); - } - auto old_seq_size = seq.size(); - if (shared_const_base_tsr_.empty()) { - seq.insert(seq.end(), ret->seq_.begin(), ret->seq_.end()); - } else { - // insert shared_const_base_tsr definition after definition and - // initialization of is_init - bool found_init = false; - bool shared_const_base_inserted = false; - for (auto &olds : ret->seq_) { - if (!shared_const_base_inserted) { - if (any_map_t::fetch_or_else(olds->attr_.get(), - attr_keys::is_shared_const_init_stmt, - false)) { - found_init = true; - } else { - if (found_init) { - for (auto &tsr : shared_const_base_tsr_) { - // the function name is too long for clang - // format to break - // clang-format off - seq.emplace_back( - builder::make_var_tensor_def_unattached( - tsr, - linkage::local)); - // clang-format on - } - shared_const_base_inserted = true; - } - } - } - seq.emplace_back(olds); - } - if (!shared_const_base_inserted) { - for (auto &tsr : shared_const_base_tsr_) { - seq.insert( - seq.begin() + old_seq_size, - builder::make_var_tensor_def_unattached(tsr, linkage::local)); - } - } - } - return copy_attr(*v, builder::make_stmts_unattached(seq)); - } else { - return ir_visitor_t::visit(v); - } - } -}; - -static size_t align_to_64(size_t v) { - return utils::divide_and_ceil(v, 64) * 64; -} - -static size_t get_tensor_size(const tensor &tsr, const define &def) { - COMPILE_ASSERT(tsr->dims_.size() == 1 && tsr->dims_[0].isa(), - "The global tensor should be 1D and the dims should be " - "constant, got " - << def); - auto size = get_const_as_int(tsr->dims_[0].static_as()) * - utils::get_sizeof_type(tsr->elem_dtype_); - return size; -} - -static size_t update_allocated_size(const tensor &tsr, size_t size, - size_t allocated_size) { - if (auto absptr = any_map_t::fetch_or_else( - tsr->attr_.get(), attr_keys::static_global, nullptr)) { - tsr->attr()[attr_keys::module_global_offset] = absptr; - return allocated_size; - } - if (size >= 64) { - allocated_size = align_to_64(allocated_size); - } - tsr->attr()[attr_keys::module_global_offset] = allocated_size; - allocated_size += size; - return allocated_size; -} - -/** - * shared const tensors are special local tensors. We collect all shared const - * tensors in main-entry and assign a "handle" for each of the shared const - * tensors. The handles are pointers to runtime::const_cache_proxy. To keep them - * alive, we put them in statics_table_t in the final jit_module - * */ -static std::vector process_shared_const_tensors( - const ir_module_ptr &m, - std::vector> &out_graph_tsr, - std::vector &out_base_tsr) { - auto mainf = m->get_entry_func(); - if (!mainf) { - return {}; - } - std::vector collected_base; - auto &seq = mainf->body_.checked_as()->seq_; - auto bases = m->attr_.get_or_null< - std::vector>>( - ir_module_t::attr_key_t::SHARED_CONST_BASES); - if (bases) { - out_base_tsr.reserve(bases->size()); - for (auto &base : *bases) { - out_base_tsr.emplace_back(builder::make_tensor( - "__shared_const_base_" + std::to_string(collected_base.size()), - {base->size_}, datatypes::u8)); - out_base_tsr.back()->attr()[attr_keys::shared_const_base_idx] = - static_cast(collected_base.size()); - // collect the list of the buffer handle used. If the buffer is not - // lazy, then it will be never evicted. We can directly use the - // buffer pointer instead of the handle - void *handle = - (base->is_lazy_) ? base.get() : base->get_buffer_if_not_lazy(); - collected_base.emplace_back(handle); - } - } - for (auto &s : seq) { - // if the stmt is a define node with cached_const_graph_tensor - if (auto const_graph_tsr = - s.cast() - .flat_map( - [](const define_c &v) { return v->var_.cast(); }) - .map([](const tensor &v) { - return any_map_t::fetch_or_else< - std::shared_ptr>( - v->attr_.get(), attr_keys::shared_const, nullptr); - }) - .get_or_else(nullptr)) { - out_graph_tsr.emplace_back(const_graph_tsr); - const auto &base = const_graph_tsr->buf_base_; - - void *handle = - (base->is_lazy_) ? base.get() : base->get_buffer_if_not_lazy(); - COMPILE_ASSERT(bases, "Expecting SHARED_CONST_BASES attr in the module"); - auto itr = std::find(bases->begin(), bases->end(), base); - COMPILE_ASSERT(itr != bases->end(), - "Cannot find the shared const's base buffer in " - "SHARED_CONST_BASES of the module"); - auto idx = static_cast(itr - bases->begin()); - s->attr()["shared_const_handle_idx"] = idx; - if (!out_base_tsr[idx]->attr().has_key(attr_keys::shared_const)) { - out_base_tsr[idx]->attr()[attr_keys::shared_const] = const_graph_tsr; - } - } - } - if (!collected_base.empty()) { - auto tsr = m->make_global_tensor(datatypes::index, shared_const_handle_name, - {uint64_t(collected_base.size())}); - } - return collected_base; -} - -const_ir_module_ptr -module_globals_resolver_t::operator()(const_ir_module_ptr m) { - auto ret = std::make_shared(*m); - std::vector> out_graph_tsr; - std::vector shared_const_base_tsr; - auto base_cache_tsr = - process_shared_const_tensors(ret, out_graph_tsr, shared_const_base_tsr); - // first, plan the memory layout for the static buffer to be used in an - // IR module: - // ------------ - // global vars - // ------------ - // initialized global tensors - // ------------ - // uninitialized global tensors - // ------------ - size_t allocated_size = 0; - for (auto &def : ret->get_module_vars()) { - if (def->var_.isa()) { - auto size = utils::get_sizeof_type(def->var_->dtype_); - // natural alignment of the address - allocated_size = utils::divide_and_ceil(allocated_size, size) * size; - def->var_->attr()[attr_keys::module_global_offset] = allocated_size; - allocated_size += utils::get_sizeof_type(def->var_->dtype_); - } - } - // initialized tensors - for (auto &def : ret->get_module_vars()) { - if (def->var_.isa()) { - auto tsr = def->var_.static_as(); - if (tsr->init_value_) { - auto size = get_tensor_size(tsr, def); - COMPILE_ASSERT(tsr->init_value_->size_ == size, - "The size of the global tensor (" - << tsr->init_value_->size_ - << ") does not match its " - "initializer (" - << size << "): " << def); - allocated_size = update_allocated_size(tsr, size, allocated_size); - } - } - } - size_t init_size = allocated_size; - for (auto &def : ret->get_module_vars()) { - if (def->var_.isa()) { - auto tsr = def->var_.static_as(); - if (!tsr->init_value_) { - auto size = get_tensor_size(tsr, def); - allocated_size = update_allocated_size(tsr, size, allocated_size); - } - } - } - - std::shared_ptr sym_table = - std::make_shared( - aligned_buffer_t(allocated_size, m->ctx_->engine_)); - sym_table->initialized_size_ = init_size; - sym_table->shared_tensors_ = std::move(out_graph_tsr); - for (auto &def : ret->get_module_vars()) { - auto &anyoffset = - def->var_->attr().get_any(attr_keys::module_global_offset); - if (def->var_.isa()) { - sym_table->add(def->var_.static_as()->name_, - anyoffset.get()); - } else { - auto tsr = def->var_.checked_as(); - if (auto offset = anyoffset.get_or_null()) { - sym_table->add(tsr->name_, *offset); - if (tsr->init_value_) { - memcpy(reinterpret_cast(sym_table->data_.data_) + *offset, - tsr->init_value_->data_, tsr->init_value_->size_); - } - } - } - } - // fill the shared_const_handles - if (!base_cache_tsr.empty()) { - if (auto handles = sym_table->get_or_null(shared_const_handle_name)) { - memcpy(handles, base_cache_tsr.data(), - sizeof(base_cache_tsr[0]) * base_cache_tsr.size()); - } else { - throw std::runtime_error( - "Cannot find the shared_const_handle in module data"); - } - } - uintptr_t baseptr = (uintptr_t)sym_table->data_.data_; - bool is_static_global = - m->attr_.get_or_else(ir_module_t::attr_key_t::STATIC_GLOBALS, false); - ret->attr_[ir_module_t::attr_key_t::MODULE_DATA_BUFFERS] = - std::move(sym_table); - auto &funcs = ret->get_contents(); - // second, pass the base pointer of the global buffer from "main entry" - // func to all IR functions. We also need to replace the call nodes and - // append a new parameter to pass down the global buffer pointer - std::unordered_map replace_map; - for (auto &f : funcs) { - if (!f->body_.defined()) { - continue; - } - if (f->attr_ && f->attr_->get_or_else(function_attrs::low_level, false)) { - continue; - } - if (any_map_t::fetch_or_else(f->attr_.get(), "device_func", false)) { - continue; - } - auto params = f->params_; - // insert two placeholders - params.insert(params.begin(), 2, expr()); - params[0] = builder::make_var(datatypes::pointer, "__stream"); - params[1] = builder::make_tensor("__module_data", {0UL}, datatypes::s8); - auto retf = builder::make_func( - f->name_, params, builder::make_stmts_unattached({}), f->ret_type_); - if (f->attr_) { - // we added new args, we need to update the comments - if (auto comments = - f->attr_->get_or_null>("comments")) { - if (comments->size() >= 2 && - utils::string_startswith((*comments)[1], "@param")) { - std::vector new_comments = { - comments->at(0), - "@param __stream the stream pointer, usually " - "get_default_stream()", - "@param __module_data the module global data"}; - for (size_t i = 1; i < comments->size(); i++) { - auto &comment = (*comments)[i]; - new_comments.emplace_back(comment); - } - retf->decl_->attr()["comments"] = new_comments; - retf->attr()["comments"] = std::move(new_comments); - } - } - } - auto *funcp = retf.get(); - replace_map[f->name_] = copy_attr(*f, std::move(retf)); - funcp->decl_ = copy_attr(*f, std::move(funcp->decl_)); - } - for (unsigned i = 0; i < funcs.size(); i++) { - if (!funcs[i]->body_.defined()) { - continue; - } - if (funcs[i]->attr_ && - funcs[i]->attr_->get_or_else(function_attrs::low_level, false)) { - continue; - } - if (any_map_t::fetch_or_else(funcs[i]->attr_.get(), "device_func", false)) { - continue; - } - module_globals_resolver_impl_t impl; - if ((int)i == ret->get_entry_func_idx()) { - impl.shared_const_base_tsr_ = std::move(shared_const_base_tsr); - // if it is main func, make sure shared_const_handle is used - auto handle_tsr = ret->get_var_def_from_symbol(shared_const_handle_name); - if (handle_tsr.defined()) { - impl.get_or_create_new_var(handle_tsr->var_); - } - } - impl.absolute_base_ = is_static_global ? baseptr : 0; - impl.map = &replace_map; - auto funcp = replace_map[funcs[i]->name_]; - assert(funcp); - impl.current_rtl_ctx = funcp->params_[0]; - impl.current_base = funcp->params_[1]; - funcp->body_ = impl.dispatch(funcs[i]->body_).remove_const(); - funcs[i] = funcp; - } - return ret; -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/ir/transform/module_globals_resolve.hpp b/legacy/core/src/compiler/ir/transform/module_globals_resolve.hpp deleted file mode 100644 index da24ad861..000000000 --- a/legacy/core/src/compiler/ir/transform/module_globals_resolve.hpp +++ /dev/null @@ -1,46 +0,0 @@ -/******************************************************************************* - * Copyright 2020-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_MODULE_GLOBALS_RESOLVE_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_IR_TRANSFORM_MODULE_GLOBALS_RESOLVE_HPP - -#include "../module_pass.hpp" - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -namespace attr_keys { -constexpr const char *static_global = "static_global"; -constexpr const char *module_global_offset = "module_global_offset"; -} // namespace attr_keys -/** - * Resolves global variables and tensors to module_globals_pointer + offsets. - * Also modifies all function arguments defined in the current module by - * appending an additional argument for module_globals_pointer - * */ -class module_globals_resolver_t : public module_pass_t { -public: - const_ir_module_ptr operator()(const_ir_module_ptr m) override; - SC_DECL_PASS_INFO_FUNC(); -}; -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/ir/transform/nested_parallel_flatten.cpp b/legacy/core/src/compiler/ir/transform/nested_parallel_flatten.cpp deleted file mode 100644 index 496e4a5e7..000000000 --- a/legacy/core/src/compiler/ir/transform/nested_parallel_flatten.cpp +++ /dev/null @@ -1,493 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -#include "nested_parallel_flatten.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -SC_DECL_PASS_INFO(nested_parallel_flattener, - SC_PASS_DEPENDS_ON(validator, - buffer_rescheduling_tensor_hoisting), - SC_PASS_REQUIRE_STATE(CONST_FOLDED, IR_SIMPLIFIED), - SC_PASS_REQUIRE_NOT_STATE(), SC_PASS_SET_STATE(), - SC_PASS_UNSET_STATE(CONST_FOLDED, IR_SIMPLIFIED)); - -// In this pass, we do and only do nested parallel flattening. -// Tensor hoisting is already done in buffer_rescheduling_tensor_hoisting pass. -class nested_parallel_flatten_impl_t : public ir_visitor_t { - struct parallel_info_t { - int num_groups_; - int threads_per_group_; - expr thread_id_; - expr group_id_; - expr barriers_; - parallel_info_t(int num_groups, int threads_per_group) - : num_groups_(num_groups), threads_per_group_(threads_per_group) {} - }; - - std::vector info_; - std::vector *top_level_parallel_seq_ = nullptr; - expr global_tid_; - int runtime_threads_ = runtime_config_t::get().get_num_threads(); - int count_ = 0; - int var_count_ = 0; - int for_count_ = 0; - bool cannot_parallel_ = false; - bool need_pre_barrier_ = false; - bool need_post_barrier_ = false; - -public: - using ir_visitor_t::dispatch; - - std::string make_name(const char *n) { - std::string name = n; - name += std::to_string(count_); - name += '_'; - name += std::to_string(info_.size()); - name += '_'; - name += std::to_string(++var_count_); - return name; - } - - expr get_barrier_for_current_for() { - COMPILE_ASSERT(top_level_parallel_seq_ && info_.size() > 1UL, - "Invalid for-loop"); - int num_barrier = 1; - constexpr uint64_t barrier_size = sizeof(runtime::barrier_t); - expr idx = info_[info_.size() - 2].group_id_ * barrier_size; - for (int64_t i = info_.size() - 2; i >= 0; i--) { - num_barrier *= info_[i].num_groups_; - if (i != 0) { - idx = info_[i - 1].group_id_ * (num_barrier * barrier_size) + idx; - } - } - if (info_.back().barriers_.defined()) { - return builder::tensor_ptr(info_.back().barriers_, {idx}); - } - - info_.back().barriers_ = builder::make_tensor( - make_name("_barrier"), {num_barrier * barrier_size}, datatypes::u8); - top_level_parallel_seq_->emplace_back( - builder::make_var_tensor_def_unattached(info_.back().barriers_)); - top_level_parallel_seq_->emplace_back( - builder::make_evaluate_unattached(builtin::get_init_barrier_func()( - info_.back().barriers_, num_barrier, - uint64_t(info_[info_.size() - 2].threads_per_group_)))); - return builder::tensor_ptr(info_.back().barriers_, {idx}); - } - - void gen_call_to_barrier(std::vector *cur_insert_point, - int post_barrier_id) { - auto b = get_barrier_for_current_for(); - auto the_call = - builtin::get_barrier_arrive_func()(b, get_ir_null(), get_ir_null()); - cur_insert_point->emplace_back(builder::make_evaluate_unattached(the_call)); - if (post_barrier_id >= 0) { - the_call->attr()["post_barrier_id"] = post_barrier_id; - } - } - - bool is_trace_call(const stmt &v) { - return v.cast() - .map([](const evaluate &v) { return v->value_.as(); }) - .map([](const call &v) { - return dynamic_cast(v->func_.get()); - }) - .filter([](func_base *f) { - return f->attr_ && - f->attr_->get_or_else(function_attrs::is_trace_func, false); - }) - .has_value(); - } - - /* -transforming: -void work() { -#pragma omp parallel for num_threads(4) - for (int i = 0; i < 1000; i++) { - aaa(i); - bbb(i); -#pragma omp parallel for num_threads(2) - for (int j = 0; j < 123; j++) { - ccc(j); - ddd(j); -#pragma omp parallel for num_threads(8) - for (int k = 0; k < 56; k++) { - eee(k); - fff(k); - } - } - } -} -===================================== -to: -void work() { - barrier bar1[4]; // 1 is group level. 4 is the thread number of first group, - // each bar sync 16 threads - barrier bar2[4][2]; // 2 is group level. 2 is the thread number of first - // group, each bar sync 8 threads -#pragma omp parallel for num_threads(64) - for (int tid0 = 0; tid0 < 64; tid0++) { - int gid1 = tid0 / 16; // group id level 1, 0-3 - int local_tid1 = tid0 % 16; // thread id in group level 1, 0-15 - int start1, end1; - balance211(gid1, 0, 1000, &start1, &end1); - // simulate the line : for (int i = 0; i < 1000; i++) { - // 64-thread group will execute following - for (int i = start1; i < end1; i++) { - if (local_tid1 == 0) { - // only 1 threads in 16-thread group will execute this - aaa(i); - bbb(i); - } - // start of the code: for (int j = 0; j < 123; j++) { - // sync 16 threads in the group - int gid2 = local_tid1 / 8; // group id level 2, 0-1 - int local_tid2 = local_tid1 % 8; // thread id in group level 2 - int start2, end2; - balance211(gid2, 0, 123, &start2, &end2); - bar1[gid1].enter_and_wait(); - for (int j = start2; j < end2; j++) { - if (local_tid2 == 0) { - ccc(j); - ddd(j); - } - // last level has 8 threads in each group - int gid3 = local_tid1 / 1; - int local_tid3 = local_tid1 % 1; // thread id in group level 3 - int start3, end3; - balance211(gid3, 0, 56, &start3, &end3); - bar2[gid1][gid2].enter_and_wait(); - for (int k = start3; k < end3; k++) { - eee(k); - fff(k); - } - bar2[gid1][gid2].enter_and_wait(); - } - bar1[gid1].enter_and_wait(); - } - } -} - */ - void transform_loop(const for_loop_c &v, int num_threads_parent_group, - std::vector &seq, expr tid0, - bool need_pre_barrier, // NOLINT - bool need_post_barrier) { - int cur_post_barrier_id = for_count_; - for_count_++; - COMPILE_ASSERT( - info_.empty() || info_.front().threads_per_group_ != 0, - "Cannot handle nested parallel-for without num threads in most " - "outer parallel-for"); - int num_threads = v->num_threads_; - if (num_threads == 0) { - num_threads = num_threads_parent_group; - } - bool divisible = num_threads_parent_group % num_threads == 0; - uint64_t threads_per_group = num_threads_parent_group / num_threads; - COMPILE_ASSERT(threads_per_group > 0, - "Too many threads in this parallel: " << v); - - info_.emplace_back(num_threads, threads_per_group); - auto gid1 = builder::make_var(datatypes::index, make_name("_gid")); - info_.back().group_id_ = gid1; - auto tid1 = builder::make_var(datatypes::index, make_name("_tid")); - info_.back().thread_id_ = tid1; - seq.emplace_back(builder::make_var_tensor_def_unattached( - gid1, linkage::local, tid0 / threads_per_group)); - seq.emplace_back(builder::make_var_tensor_def_unattached( - tid1, linkage::local, tid0 % threads_per_group)); - std::vector *cur_insert_point = &seq; - // if parent threads per group is not divisible by the current num of - // threads, gid may excess num_threads, need to skip it - if (!divisible) { - auto gid_ok_body = make_stmt(std::vector{}); - stmts gid_skip_body; - if (need_pre_barrier) { - gid_skip_body = make_stmt(std::vector{}); - gen_call_to_barrier(&gid_skip_body->seq_, -1); - } - seq.emplace_back(builder::make_if_else_unattached( - gid1 < - make_expr(uint64_t(num_threads), datatypes::index), - gid_ok_body, gid_skip_body)); - cur_insert_point = &gid_ok_body->seq_; - // the balance211 & for body code will be omited in the if body - } - expr begin, end; - builtin::generate_balance211( - num_threads, v->iter_begin_, v->iter_end_, v->step_, gid1, - [&](const char *v) { return make_name(v); }, &begin, nullptr, &end, - cur_insert_point); - if (need_pre_barrier) { - gen_call_to_barrier(cur_insert_point, -1); - } - - auto new_body = make_stmt(std::vector{}); - auto step_expr = v->step_->dtype_ == datatypes::index - ? v->step_ - : constant_folder_t()( - builder::make_cast(datatypes::index, v->step_)); - cur_insert_point->emplace_back(builder::make_for_loop_unattached( - v->var_, begin, end, step_expr, new_body, v->incremental_, - for_type::NORMAL)); - - auto &old_body = v->body_.checked_as()->seq_; - stmts single_thread_body; - bool local_need_pre_barrier = false; - // convert old body of v to new_body - for (size_t i = 0; i < old_body.size(); i++) { - if (old_body[i].isa() && - old_body[i].static_as()->kind_ == for_type::PARALLEL) { - cannot_parallel_ = false; - // if there are single-threaded sections in the for-body, we - // need sync the threads - need_pre_barrier_ = local_need_pre_barrier; - need_post_barrier_ = false; - // check if we need to insert post barrier. If the current - // parallel-for is the last statement in parent parallel-for, we - // don't need the barrier - for (size_t n = i + 1; n < old_body.size(); n++) { - // if the next stmt is a pure definition, we can ignore it - if (old_body[n].isa()) { - auto &initv = old_body[n].static_as()->init_; - if (!initv.defined() || initv.isa()) { - continue; - } - } else if (is_trace_call(old_body[n])) { - continue; - } - - // otherwise, we cannot remove the barrier because the stmt - // may depend on the current loop - need_post_barrier_ = true; - break; - } - auto body = dispatch(old_body[i]).remove_const().checked_as(); - new_body->seq_.insert(new_body->seq_.end(), body->seq_.begin(), - body->seq_.end()); - single_thread_body = stmts(); - local_need_pre_barrier = false; - } else if (old_body[i] - .cast() - .filter([](const define &v) { - if (!v->init_.defined()) { - return true; - } - bool is_allowed = - v->init_.isa() || - (v->init_.isa(); - update_liveness({val->cond_, val->l_, val->r_}, index); - } break; - case sc_expr_type::indexing: { - auto val = v.static_as(); - update_liveness(&val->ptr_, index); - update_liveness(val->idx_, index); - if (val->mask_.defined()) { - update_liveness(&val->mask_, index); - } - } break; - case sc_expr_type::call: { - auto val = v.static_as(); - auto func = std::dynamic_pointer_cast(val->func_); - if (func) { - update_liveness({expr(func)}, index); - } - update_liveness(val->args_, index); - } break; - case sc_expr_type::tensorptr: { - auto val = v.static_as(); - update_liveness(&val->base_->ptr_, index); - update_liveness(val->base_->idx_, index); - } break; - case sc_expr_type::intrin_call: { - update_liveness(v.static_as()->args_, index); - } break; - case sc_expr_type::low_level_intrin: { - auto val = v.checked_as(); - update_liveness(val->args_, index); - if (val->modifier_.cond_mask_.defined()) { - update_liveness({val->modifier_.cond_mask_}, index); - } - } break; - } - } -}; - -func_c live_interval_t::operator()(func_c v) { - if (v->name_.find("_should_inline_") != std::string::npos) { - return v; - } - live_interval_impl_t live_interval; - return live_interval.dispatch(std::move(v)); -} - -} // namespace xbyak -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/jit/xbyak/ir/pass/live_interval.hpp b/legacy/core/src/compiler/jit/xbyak/ir/pass/live_interval.hpp deleted file mode 100644 index 1a51c973f..000000000 --- a/legacy/core/src/compiler/jit/xbyak/ir/pass/live_interval.hpp +++ /dev/null @@ -1,46 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_JIT_XBYAK_IR_PASS_LIVE_INTERVAL_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_JIT_XBYAK_IR_PASS_LIVE_INTERVAL_HPP - -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace xbyak { - -/* * - * Calculate the liveness of each ir expr, the live range is determined by first - * and last use of an expr at stmt ir index. - * */ -class live_interval_t : public function_pass_t { -public: - live_interval_t() = default; - func_c operator()(func_c v) override; - -private: -}; - -} // namespace xbyak -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/jit/xbyak/ir/reg_allocation/interval_tree.hpp b/legacy/core/src/compiler/jit/xbyak/ir/reg_allocation/interval_tree.hpp deleted file mode 100644 index f78c13b7f..000000000 --- a/legacy/core/src/compiler/jit/xbyak/ir/reg_allocation/interval_tree.hpp +++ /dev/null @@ -1,164 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_JIT_XBYAK_IR_REG_ALLOCATION_INTERVAL_TREE_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_JIT_XBYAK_IR_REG_ALLOCATION_INTERVAL_TREE_HPP - -#include -#include -#include -#include - -#include "virtual_reg.hpp" - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace xbyak { - -/* * - * Non-overlapping balanced interval tree implemetation based on std::set. - * */ -class interval_tree_t { -public: - // constructor - interval_tree_t() = default; - // destructor - virtual ~interval_tree_t() = default; - - bool empty() { return node_map_.empty(); } - - // insert new interval - void insert(stmt_index_t start, stmt_index_t end, virtual_reg_t *virt_reg) { - node_map_.insert(node_t(start, end, virt_reg)); - } - - // remove existing interval - void remove(stmt_index_t start, stmt_index_t end, virtual_reg_t *virt_reg) { - erase_nodes(start, end, virt_reg, [](node_t node) {}); - } - - // divide existing interval using cut range - void divide(stmt_index_t start, stmt_index_t end, virtual_reg_t *virt_reg) { - std::vector erased_nodes; - auto node_func = [&](node_t node) { erased_nodes.push_back(node); }; - erase_nodes(start, end, virt_reg, node_func); - for (auto &node : erased_nodes) { - // front - stmt_index_t start_front = node.start_; - stmt_index_t end_front = std::min(node.end_, start); - if (start_front < end_front) { - insert(start_front, end_front, virt_reg); - } - // back - stmt_index_t start_back = std::max(node.start_, end); - stmt_index_t end_back = node.end_; - if (start_back < end_back) { - insert(start_back, end_back, virt_reg); - } - } - } - - // search interval for overlap - bool search(stmt_index_t start, stmt_index_t end) { - auto iter = node_map_.lower_bound(node_t(start, end, nullptr)); - if (iter != node_map_.begin()) { - iter--; - } - while (iter != node_map_.end()) { - auto &node = *iter; - if (end <= node.start_) { - break; - } - if (node.intersects(start, end)) { - return true; - } - iter++; - } - return false; - } - - // query interval for overlap - void query(stmt_index_t start, stmt_index_t end, - std::function func) { - auto iter = node_map_.lower_bound(node_t(start, end, nullptr)); - if (iter != node_map_.begin()) { - iter--; - } - while (iter != node_map_.end()) { - auto &node = *iter; - if (end <= node.start_) { - break; - } - if (node.intersects(start, end)) { - func(node.virtual_reg_); - } - iter++; - } - } - -private: - // Internal node - struct node_t { - stmt_index_t start_; - stmt_index_t end_; - virtual_reg_t *virtual_reg_; - - bool intersects(stmt_index_t start, stmt_index_t end) const { - return std::max(start_, start) < std::min(end_, end); - } - - bool operator<(const node_t &b) const { return start_ < b.start_; } - - node_t(stmt_index_t start, stmt_index_t end, virtual_reg_t *virt_reg) - : start_(start), end_(end), virtual_reg_(virt_reg) { - // must contain valid range - assert(start < end); - } - }; - // Internal RB-tree - std::set node_map_; - - // erase interval node - void erase_nodes(stmt_index_t start, stmt_index_t end, - virtual_reg_t *virt_reg, std::function func) { - auto iter = node_map_.lower_bound(node_t(start, end, nullptr)); - if (iter != node_map_.begin()) { - iter--; - } - while (iter != node_map_.end()) { - auto &node = *iter; - if (end <= node.start_) { - break; - } - if (node.virtual_reg_ == virt_reg) { - func(node); - iter = node_map_.erase(iter); - } else { - iter++; - } - } - } -}; - -} // namespace xbyak -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/jit/xbyak/ir/reg_allocation/live_range.hpp b/legacy/core/src/compiler/jit/xbyak/ir/reg_allocation/live_range.hpp deleted file mode 100644 index 96ac0741c..000000000 --- a/legacy/core/src/compiler/jit/xbyak/ir/reg_allocation/live_range.hpp +++ /dev/null @@ -1,84 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_JIT_XBYAK_IR_REG_ALLOCATION_LIVE_RANGE_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_JIT_XBYAK_IR_REG_ALLOCATION_LIVE_RANGE_HPP - -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace xbyak { - -using stmt_index_t = int64_t; - -namespace stmt_index_const { -constexpr stmt_index_t increment = 4; -} - -/* * - * Live range representation for expr, records start and end point for - * virtual_reg based on its def-use at stmt_index - * */ -struct live_range_t { - bool defined_ = false; - - stmt_index_t start_ = -1; - stmt_index_t end_ = -1; - - live_range_t() = default; - - live_range_t(stmt_index_t start) - : defined_(true), start_(start), end_(start) {} - live_range_t(stmt_index_t start, stmt_index_t end) - : defined_(true), start_(start), end_(end) {} - - void update(stmt_index_t index) { end_ = index; } - void update(stmt_index_t init_index, stmt_index_t index) { - start_ = (init_index < start_) ? init_index : start_; - end_ = index; - } - - bool empty() const { return start_ == end_; } - - bool intersects(const live_range_t &b) const { - return std::max(start_, b.start_) < std::min(end_, b.end_); - } - - bool encompasses(const live_range_t &b) const { - return (start_ < b.start_) && (end_ > b.end_); - } - - bool enclose(const stmt_index_t &i) const { - return (start_ < i) && (end_ > i); - } - - friend std::ostream &operator<<(std::ostream &os, const live_range_t &m) { - return os << "[" << m.start_ << ", " << m.end_ << "]"; - } -}; - -} // namespace xbyak -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/jit/xbyak/ir/reg_allocation/reg_allocator.hpp b/legacy/core/src/compiler/jit/xbyak/ir/reg_allocation/reg_allocator.hpp deleted file mode 100644 index c5ef9bf83..000000000 --- a/legacy/core/src/compiler/jit/xbyak/ir/reg_allocation/reg_allocator.hpp +++ /dev/null @@ -1,291 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_JIT_XBYAK_IR_REG_ALLOCATION_REG_ALLOCATOR_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_JIT_XBYAK_IR_REG_ALLOCATION_REG_ALLOCATOR_HPP - -#include -#include -#include -#include -#include -#include - -#include "virtual_slot.hpp" - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace xbyak { - -/* * - * vrtual_reg dequeue priority: - * [spill weight higher] -> [defined earlier] -> [ended earlier] - * */ -struct spill_weight_comparator_t { - bool operator()(const virtual_reg_t *lhs, const virtual_reg_t *rhs) { - assert(lhs != nullptr && rhs != nullptr); - bool def_later = lhs->live_range_.start_ == rhs->live_range_.start_ - ? lhs->live_range_.end_ > rhs->live_range_.end_ - : lhs->live_range_.start_ > rhs->live_range_.start_; - bool weight_equal = lhs->spill_weight_ == rhs->spill_weight_; - bool weight_less = lhs->spill_weight_ < rhs->spill_weight_; - return (weight_equal && def_later) || (!weight_equal && weight_less); - } -}; - -/* * - * Priority based register allocator. - * All unassigned virtual regs need to be enqueued. - * Dequeue each virtual reg based on priority, allocator will try to assign a - * virtual slot for each virtual reg. - * If no slot available, virtual reg will be spilled and if spill resolver find - * the spilled operand is conflicted with intrinsics' instruction format, - * conflicts will be resolved and small load/store intervals with infinite spill - * weight will be created and put back into the priority queue. - * */ -class reg_allocator_t { -public: - reg_allocator_t(const x86_64::target_profile_t &profile) - : target_profile_(profile), require_resolve_(false) { - // Win64 ABI considers registers XMM6-XMM15 nonvolatile, but the - // upper portions of YMM0-YMM15 and ZMM0-ZMM15 still volatile. - // This will introduce unnecessary design complexity, so we treat - // all of them as volatile in xbyak generated callers. - virtual_slots_map_ = - std::make_shared(target_profile_, true); - virtual_slots_array_ = std::make_shared( - virtual_slots_map_->get_slots_sum()); - } - virtual ~reg_allocator_t() = default; - - bool queue_empty() { return virtual_reg_queue_.empty(); } - - void enqueue(virtual_reg_t *virt_reg) { virtual_reg_queue_.push(virt_reg); } - - virtual_reg_t *dequeue() { - if (queue_empty()) { - return nullptr; - } - virtual_reg_t *top = virtual_reg_queue_.top(); - virtual_reg_queue_.pop(); - return top; - } - - // Allocation routine - void run_allocator() { - // Go through queue and allocate every unassigned virtual_regs - while (!queue_empty()) { - // Get the virtual_reg on front of queue - auto virt_reg = dequeue(); - // Try to allocate virtual_reg, if interference exists - std::set evicted; - auto index = try_assign(virt_reg, evicted); - // Evict confilct virt_regs with less spill weight - for (auto &vr : evicted) { - unassign(vr); - } - // Assign or spill virtual reg - if (index == virt_reg_const::invalid) { - spill(virt_reg); - } else { - allocate(virt_reg, index); - } - // Resolve address mode and get new virtual_regs created - if (require_resolve()) { - resolve_spill(virt_reg->live_range_); - } - } - } - - // Heuristics for determining virtual_reg assign/spill - virt_reg_index_t try_assign(virtual_reg_t *virt_reg, - std::set &evicted) { - // Initial value for interference check - virt_reg_index_t index = virt_reg_const::invalid; - spill_weight_t weight = spill_weight_const::infinity; - - // Check interference for hint reg - auto check_interference_hint = [&](bool preserved) { - auto &callee_save = slots_map().get_callee_save_set(virt_reg->type_); - index = virt_reg->index_hint_; - bool check = - preserved ? callee_save.find(index) != callee_save.end() : true; - if (check) { - weight = slots_array().interfered_weights(virt_reg, index); - } - }; - // Check interference for all regs - auto check_interference_all = [&](bool preserved) { - const auto &candidates = - preserved ? slots_map().get_callee_save(virt_reg->type_) - : slots_map().get_slots_index(virt_reg->type_); - for (const auto &i : candidates) { - auto w = slots_array().interfered_weights(virt_reg, i); - if (w <= weight) { - index = i; - weight = w; - } - if (weight == spill_weight_const::null) { - break; - } - } - }; - - // Check interference for different types of virtual reg - switch (virt_reg->hint_) { - case virt_reg_hint::strong: { - check_interference_hint(virt_reg->preserved_); - } break; - case virt_reg_hint::weak: { - check_interference_hint(virt_reg->preserved_); - if (weight == spill_weight_const::null) { - break; - } - check_interference_all(virt_reg->preserved_); - } break; - case virt_reg_hint::none: { - check_interference_all(virt_reg->preserved_); - } break; - } - - // Final result - if (weight == spill_weight_const::null) { - // If slot aviliable - return index; - } else { - if (virt_reg->spill_weight_ <= weight) { - // If slot unaviliable - assert(virt_reg->spill_weight_ < spill_weight_const::infinity); - return virt_reg_const::invalid; - } else { - // If interfered virtual regs on slot have less spill weight - // Evict interfered and assign current virtual reg instead - assert(index != virt_reg_const::invalid); - evicted = slots_array().interfered_regs(virt_reg, index); - return index; - } - } - } - - // Check instruction format and create new load/store interval if needed - void resolve_spill(const live_range_t &spill_range) { - std::vector virtual_regs; - resolve_spill_impl(spill_range, virtual_regs); - for (auto &vr : virtual_regs) { - enqueue(vr); - } - require_resolve_ = false; - } - - // Unassign virtual_reg and put back to the queue - void unassign(virtual_reg_t *virt_reg) { - assert(virt_reg); - switch (virt_reg->stat_) { - case virt_reg_stat::spilled: { - virt_reg->set_unassigned(); - } break; - case virt_reg_stat::allocated: { - slots_array().unassign_slot(virt_reg, virt_reg->index_); - virt_reg->set_unassigned(); - } break; - case virt_reg_stat::disabled: - case virt_reg_stat::buffered: - case virt_reg_stat::unassigned: - case virt_reg_stat::designated: { - assert(0 && "Invalid Unassign."); - } break; - } - enqueue(virt_reg); - } - - // Allocate physical_reg for virtual_reg - void allocate(virtual_reg_t *virt_reg, virt_reg_index_t index) { - assert(virt_reg); - switch (virt_reg->stat_) { - case virt_reg_stat::unassigned: { - slots_array().assign_slot(virt_reg, index); - virt_reg->set_allocated(index); - } break; - case virt_reg_stat::designated: { - assert(index == virt_reg->index_); - slots_array().assign_slot(virt_reg, index); - } break; - case virt_reg_stat::disabled: - case virt_reg_stat::buffered: - case virt_reg_stat::allocated: - case virt_reg_stat::spilled: { - assert(0 && "Invalid Allocate."); - } break; - } - } - - // Spill virtual_reg on stack - void spill(virtual_reg_t *virt_reg) { - assert(virt_reg); - switch (virt_reg->stat_) { - case virt_reg_stat::unassigned: { - virt_reg->set_spilled(); - } break; - case virt_reg_stat::allocated: - case virt_reg_stat::disabled: - case virt_reg_stat::buffered: - case virt_reg_stat::designated: - case virt_reg_stat::spilled: { - assert(0 && "Invalid Spill."); - } break; - } - spilled_virt_regs_.insert(virt_reg); - require_resolve_ = true; - } - - // Current target profile - const x86_64::target_profile_t &target_profile() { return target_profile_; } - // Slots map to map between virtual slots and physical registers - virtual_slots_map_t &slots_map() { return *virtual_slots_map_; } - // Slots array to store allocated virtual registers and check interference - virtual_slots_array_t &slots_array() { return *virtual_slots_array_; } - // Spilled occurred - bool require_resolve() { return require_resolve_; } - // Spilled virt regs - std::set &spilled_virt_regs() { return spilled_virt_regs_; } - - // Virtual function for allocator_impl to create spill resolver - virtual void - resolve_spill_impl(const live_range_t &spill_range, - std::vector &virtual_regs) = 0; - -private: - const x86_64::target_profile_t &target_profile_; - - std::priority_queue, - spill_weight_comparator_t> - virtual_reg_queue_; - - bool require_resolve_; - std::set spilled_virt_regs_; - std::shared_ptr virtual_slots_map_; - std::shared_ptr virtual_slots_array_; -}; - -} // namespace xbyak -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/jit/xbyak/ir/reg_allocation/virtual_reg.hpp b/legacy/core/src/compiler/jit/xbyak/ir/reg_allocation/virtual_reg.hpp deleted file mode 100644 index 583dcfff7..000000000 --- a/legacy/core/src/compiler/jit/xbyak/ir/reg_allocation/virtual_reg.hpp +++ /dev/null @@ -1,230 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_JIT_XBYAK_IR_REG_ALLOCATION_VIRTUAL_REG_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_JIT_XBYAK_IR_REG_ALLOCATION_VIRTUAL_REG_HPP - -#include -#include -#include -#include - -#include "live_range.hpp" - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace xbyak { - -using virt_reg_index_t = int32_t; -using spill_weight_t = int32_t; - -namespace virt_reg_const { -constexpr virt_reg_index_t invalid = -1; -} // namespace virt_reg_const - -namespace spill_weight_const { -constexpr spill_weight_t infinity = 65535; -constexpr spill_weight_t initial = 1; -constexpr spill_weight_t null = 0; -} // namespace spill_weight_const - -enum class virt_reg_stat { - disabled = 0, // no need for register allocation - buffered, // tensor buffer at local stack - designated, // designated physical registers - unassigned, // to be assigned by register allcator - allocated, // allocated by register allcator - spilled, // spilled by register allcator/all addr operands -}; - -enum class virt_reg_type { - gp_reg = 0, // x86 general purpose 64-bit registers - fp_reg, // x86 AVX/AVX512(0-15/0-31) Extended SIMD registers - fp_vex_reg, // x86 SSE/AVX(0-15) SIMD registers - mask_reg, // x86 AVX512 mask registers - tile_reg, // x86 AMX tile registers - NUM_TYPES, -}; - -enum class virt_reg_hint { - none = 0, // no hint - weak, // perfer to be assigned at certain register - strong, // must be assigned at certain register or spill -}; - -/* * - * Xbyak::Reg hashing fucntion for hash map - * */ -struct xbyak_reg_hasher_t { - size_t operator()(const Xbyak::Reg &r) const { - size_t kind_mask = r.getKind() << 16; - return kind_mask | r.getIdx(); - } -}; - -/* * - * Virtual regsister for each expr, containing low level ir info, e.g. status, - * type, allocation hint, spill weight, live range, assigned slot, etc. - * */ -struct virtual_reg_t { - virt_reg_hint hint_ = virt_reg_hint::none; - virt_reg_type type_ = virt_reg_type::gp_reg; - virt_reg_stat stat_ = virt_reg_stat::disabled; - - live_range_t live_range_; - - virt_reg_index_t index_ = virt_reg_const::invalid; - virt_reg_index_t index_hint_ = virt_reg_const::invalid; - - spill_weight_t spill_weight_ = spill_weight_const::initial; - - // need to be preserved across function calls - bool preserved_ = false; - // force fp_reg to only use sse/avx fp_reg_vex XMM0-15 - bool force_fp_vex_ = false; - - spill_weight_t extra_weight() { - auto range = live_range_.end_ - live_range_.start_; - spill_weight_t range_weight = - range > 0 // - ? 1 + (stmt_index_const::increment * 256 / range) - : 0; - spill_weight_t preserve_weight = preserved_ // - ? stmt_index_const::increment * 128 - : 0; - spill_weight_t hint_weight = hint_ != virt_reg_hint::none // - ? stmt_index_const::increment * 16 - : 0; - spill_weight_t fp_reg_weight = force_fp_vex_ // - ? stmt_index_const::increment * 64 - : 0; - return range_weight + preserve_weight + hint_weight + fp_reg_weight; - } - - bool intersects(const virtual_reg_t &b) { - return live_range_.intersects(b.live_range_); - } - - bool disabled() { return stat_ == virt_reg_stat::disabled; } - - bool buffered() { return stat_ == virt_reg_stat::buffered; } - - bool spilled() { return stat_ == virt_reg_stat::spilled; } - - bool allocated() { - return stat_ == virt_reg_stat::designated || - stat_ == virt_reg_stat::allocated; - } - - void set_preserved() { preserved_ = true; } - void set_force_fp_vex() { force_fp_vex_ = true; } - - void set_type(virt_reg_type type) { type_ = type; } - - void set_hint(virt_reg_hint hint, virt_reg_index_t index) { - if (static_cast(hint) > static_cast(hint_)) { - hint_ = hint; - index_hint_ = index; - } - } - - void reset_hint() { - hint_ = virt_reg_hint::none; - index_hint_ = virt_reg_const::invalid; - } - - void set_buffered() { - stat_ = virt_reg_stat::buffered; - index_ = virt_reg_const::invalid; - } - - void set_unassigned() { - stat_ = virt_reg_stat::unassigned; - index_ = virt_reg_const::invalid; - } - - void set_designated(virt_reg_index_t index) { - stat_ = virt_reg_stat::designated; - index_ = index; - spill_weight_ = spill_weight_const::infinity + 1; - set_hint(virt_reg_hint::strong, index); - } - - void set_allocated(virt_reg_index_t index) { - stat_ = virt_reg_stat::allocated; - index_ = index; - } - - void set_spilled() { - stat_ = virt_reg_stat::spilled; - index_ = virt_reg_const::invalid; - } - - void add_weight(spill_weight_t weight) { - spill_weight_ = - std::min(spill_weight_const::infinity - 1, spill_weight_ + weight); - assert(spill_weight_ > 0); - } - - friend std::ostream &operator<<(std::ostream &os, const virtual_reg_t &m) { - static const char *type_enum_str[] = {"gp", "fp", "fpvex", "k", "tmm"}; - static const char *stat_enum_str[] = {"x", "B", "D", "U", "A", "S"}; - static const char *hint_enum_str[] = {"", "h", "H"}; - os << m.live_range_ << ": SW-" // - << m.spill_weight_ << ": CP-" // - << m.preserved_ << ": FV-" // - << m.force_fp_vex_ << ": " // - << stat_enum_str[static_cast(m.stat_)] - << hint_enum_str[static_cast(m.hint_)]; - switch (m.stat_) { - case virt_reg_stat::designated: - case virt_reg_stat::allocated: { - os << ": %" << type_enum_str[static_cast(m.type_)] << m.index_; - } break; - default: - break; - } - return os; - } - - virtual_reg_t() = default; - virtual_reg_t(virt_reg_type type) : type_(type) {} -}; - -inline virt_reg_type get_virt_reg_type(const sc_data_type_t &t, bool is_avx512, - bool force_fp_vex = false) { - if (t.type_code_ == sc_data_etype::BOOLEAN && t.lanes_ > 1) { - return is_avx512 ? virt_reg_type::mask_reg : virt_reg_type::gp_reg; - } else if (force_fp_vex && is_x86_simd(t)) { - return virt_reg_type::fp_vex_reg; - } else if (is_x86_simd(t)) { - return virt_reg_type::fp_reg; - } else if (t.is_tile()) { - return virt_reg_type::tile_reg; - } - - return virt_reg_type::gp_reg; -} - -} // namespace xbyak -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/jit/xbyak/ir/reg_allocation/virtual_slot.hpp b/legacy/core/src/compiler/jit/xbyak/ir/reg_allocation/virtual_slot.hpp deleted file mode 100644 index bdc3130b0..000000000 --- a/legacy/core/src/compiler/jit/xbyak/ir/reg_allocation/virtual_slot.hpp +++ /dev/null @@ -1,317 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_JIT_XBYAK_IR_REG_ALLOCATION_VIRTUAL_SLOT_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_JIT_XBYAK_IR_REG_ALLOCATION_VIRTUAL_SLOT_HPP - -#include -#include -#include -#include -#include -#include - -#include - -#include "interval_tree.hpp" - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace xbyak { - -/* * - * Represent each physical rigister and store live ranges using non-overlapping - * balanced interval tree - * */ -class virtual_slot_t : public interval_tree_t { -public: - virtual_slot_t() = default; - virtual ~virtual_slot_t() = default; - - void insert(virtual_reg_t *virt_reg) { - assert(virt_reg); - auto &live_range = virt_reg->live_range_; - interval_tree_t::insert(live_range.start_, live_range.end_, virt_reg); - } - - void remove(virtual_reg_t *virt_reg) { - assert(virt_reg); - auto &live_range = virt_reg->live_range_; - interval_tree_t::remove(live_range.start_, live_range.end_, virt_reg); - } - - void divide(virtual_reg_t *virt_reg, const live_range_t &range) { - assert(virt_reg); - interval_tree_t::divide(range.start_, range.end_, virt_reg); - } - - bool intersects(virtual_reg_t *virt_reg) { - assert(virt_reg); - auto &live_range = virt_reg->live_range_; - return interval_tree_t::search(live_range.start_, live_range.end_); - } - - spill_weight_t intersect_weights(virtual_reg_t *virt_reg) { - assert(virt_reg); - auto &live_range = virt_reg->live_range_; - spill_weight_t weight = spill_weight_const::null; - auto query_func = [&](virtual_reg_t *virt_reg) { - // Sum of intersected weight - weight = std::min(spill_weight_const::infinity, - weight + virt_reg->spill_weight_); - }; - query(live_range.start_, live_range.end_, query_func); - return weight; - } - - std::set intersect_regs(virtual_reg_t *virt_reg) { - assert(virt_reg); - auto &live_range = virt_reg->live_range_; - std::set virtual_regs; - auto query_func = [&](virtual_reg_t *vreg) { - // Set of intersected virtual regs - virtual_regs.insert(vreg); - }; - query(live_range.start_, live_range.end_, query_func); - return virtual_regs; - } - - virtual_reg_t *encompassing(const live_range_t &range) { - std::set virtual_regs; - auto query_func = [&](virtual_reg_t *vreg) { - // Set of virtual_regs encompassing range - auto &live_range = vreg->live_range_; - if (live_range.encompasses(range)) { - virtual_regs.insert(vreg); - } - }; - query(range.start_, range.end_, query_func); - if (virtual_regs.empty()) { - return nullptr; - } - assert(virtual_regs.size() == 1); - return *virtual_regs.begin(); - } -}; - -/* * - * Represent all virtual slots available for assign - * */ -class virtual_slots_array_t { -public: - virtual_slots_array_t(virt_reg_index_t slots_sum) - : virtual_slots_sum_(slots_sum) { - virtual_slots_.resize(virtual_slots_sum_); - } - virtual ~virtual_slots_array_t() = default; - - void assign_slot(virtual_reg_t *virt_reg, virt_reg_index_t index) { - assert(virt_reg); - assert(index >= 0 && index < virtual_slots_sum_); - virtual_slots_[index].insert(virt_reg); - } - - void unassign_slot(virtual_reg_t *virt_reg, virt_reg_index_t index) { - assert(virt_reg); - assert(index >= 0 && index < virtual_slots_sum_); - virtual_slots_[index].remove(virt_reg); - } - - void divide_interval(virtual_reg_t *virt_reg, const live_range_t &range, - virt_reg_index_t index) { - assert(virt_reg); - assert(index >= 0 && index < virtual_slots_sum_); - virtual_slots_[index].divide(virt_reg, range); - } - - bool interfered_with(virtual_reg_t *virt_reg, virt_reg_index_t index) { - assert(virt_reg); - assert(index >= 0 && index < virtual_slots_sum_); - return virtual_slots_[index].intersects(virt_reg); - } - - spill_weight_t interfered_weights(virtual_reg_t *virt_reg, - virt_reg_index_t index) { - assert(virt_reg); - assert(index >= 0 && index < virtual_slots_sum_); - return virtual_slots_[index].intersect_weights(virt_reg); - } - - std::set interfered_regs(virtual_reg_t *virt_reg, - virt_reg_index_t index) { - assert(virt_reg); - assert(index >= 0 && index < virtual_slots_sum_); - return virtual_slots_[index].intersect_regs(virt_reg); - } - - virtual_reg_t *encompassing_reg(const live_range_t &range, - virt_reg_index_t index) { - assert(!range.empty()); - assert(index >= 0 && index < virtual_slots_sum_); - return virtual_slots_[index].encompassing(range); - } - - std::set utilized_slots() { - std::set ret_set; - for (virt_reg_index_t index = 0; index < virtual_slots_sum_; index++) { - if (!virtual_slots_[index].empty()) { - ret_set.insert(index); - } - } - return ret_set; - } - -private: - virt_reg_index_t virtual_slots_sum_; - std::vector virtual_slots_; -}; - -/* * - * Map between physical registers and vistual slots. - * */ -class virtual_slots_map_t { -public: - // When fp_regs_volatile is true - // Ignore abi profile and does not treat fp regs as callee-saved - virtual_slots_map_t(const x86_64::target_profile_t &profile, - bool fp_regs_volatile = false) { - // Map all allocatable regs to virtual slot indexes - virt_reg_index_t virt_index = 0; - // Allocatable slot indexes - allocatable_indexes_.resize(static_cast(virt_reg_type::NUM_TYPES)); - callee_save_indexes_.resize(static_cast(virt_reg_type::NUM_TYPES)); - callee_save_sets_.resize(static_cast(virt_reg_type::NUM_TYPES)); - // ======================================== - // Get allocatable regs from target profile - // ======================================== - auto get_allocatable_regs = [&](const virt_reg_type ®_type, - const std::vector &alloc_regs) { - auto type_index = static_cast(reg_type); - for (size_t i = 0; i < alloc_regs.size(); i++) { - allocatable_indexes_[type_index].push_back(virt_index); - allocatable_regs_.push_back(alloc_regs[i]); - allocatable_regs_name_.push_back(alloc_regs[i].toString()); - xbyak_regs_map_[alloc_regs[i]] = virt_index; - virt_index++; - } - }; - auto get_allocatable_fp_vex_regs = - [&](const virt_reg_type ®_type, - const std::vector &alloc_regs) { - auto type_index = static_cast(reg_type); - for (size_t i = 0; i < alloc_regs.size(); i++) { - auto idx = get_reg_index(alloc_regs[i]); - allocatable_indexes_[type_index].push_back(idx); - } - }; - - // Allocatable gp regs to virtual indexes - get_allocatable_regs(virt_reg_type::gp_reg, profile.alloc_gp_regs_); - // Allocatable fp regs to virtual indexes - get_allocatable_regs(virt_reg_type::fp_reg, profile.alloc_xmm_regs_); - // Allocatable mask regs to virtual indexes - get_allocatable_regs(virt_reg_type::mask_reg, profile.alloc_mask_regs_); - // Allocatable tile regs to virtual indexes - get_allocatable_regs(virt_reg_type::tile_reg, profile.alloc_tile_regs_); - // Allocatable fp_vex regs, without create new virt_index - get_allocatable_fp_vex_regs(virt_reg_type::fp_vex_reg, - profile.alloc_xmm_vex_regs_); - - // Check consitancy - assert((size_t)virt_index == allocatable_regs_.size()); - slots_sum_ = virt_index; - - // ========================================= - // Get callee saved regs from target profile - // ========================================= - auto get_callee_saved_regs = - [&](const virt_reg_type ®_type, - const std::vector &callee_saved_regs) { - auto type_index = static_cast(reg_type); - for (auto ® : callee_saved_regs) { - auto index = get_reg_index(reg); - callee_save_indexes_[type_index].push_back(index); - callee_save_sets_[type_index].insert(index); - } - }; - // Allocatable gp regs to virtual indexes - get_callee_saved_regs(virt_reg_type::gp_reg, profile.callee_saved_gp_regs_); - // Allocatable fp regs to virtual indexes - if (!fp_regs_volatile) { - get_callee_saved_regs(virt_reg_type::fp_reg, - profile.callee_saved_xmm_regs_); - } - } - - Xbyak::Reg get_reg_physical(virt_reg_index_t idx) { - assert(idx >= 0 && idx < slots_sum_); - return allocatable_regs_[idx]; - } - - std::string get_reg_name(virt_reg_index_t idx) { - if (idx >= 0 && idx < slots_sum_) { - return allocatable_regs_name_[idx]; - } else { - return ""; - } - } - - virt_reg_index_t get_reg_index(const Xbyak::Reg ®) { - auto iter = xbyak_regs_map_.find(reg); - if (iter == xbyak_regs_map_.end()) { - assert(false && "Not valid reg."); - return virt_reg_const::invalid; - } - return iter->second; - } - - const std::vector &get_slots_index(virt_reg_type type) { - assert(type != virt_reg_type::NUM_TYPES); - return allocatable_indexes_[static_cast(type)]; - } - - const std::vector &get_callee_save(virt_reg_type type) { - assert(type != virt_reg_type::NUM_TYPES); - return callee_save_indexes_[static_cast(type)]; - } - - const std::set &get_callee_save_set(virt_reg_type type) { - assert(type != virt_reg_type::NUM_TYPES); - return callee_save_sets_[static_cast(type)]; - } - - virt_reg_index_t get_slots_sum() { return slots_sum_; } - -private: - std::vector> allocatable_indexes_; - std::vector> callee_save_indexes_; - std::vector> callee_save_sets_; - std::vector allocatable_regs_name_; - std::vector allocatable_regs_; - std::unordered_map - xbyak_regs_map_; - virt_reg_index_t slots_sum_; -}; - -} // namespace xbyak -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/jit/xbyak/ir/transform/avx2_legalizer.cpp b/legacy/core/src/compiler/jit/xbyak/ir/transform/avx2_legalizer.cpp deleted file mode 100644 index 5014b2069..000000000 --- a/legacy/core/src/compiler/jit/xbyak/ir/transform/avx2_legalizer.cpp +++ /dev/null @@ -1,548 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include -#include -#include -#include - -#include -#include -#include - -#include "avx2_legalizer.hpp" - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace xbyak { - -class avx2_legalizer_impl_t : public ir_visitor_t { -public: - using ir_visitor_t::dispatch; - using ir_visitor_t::visit; - - avx2_legalizer_impl_t() = default; - - bool is_base_type_less_than_32bit(const sc_data_type_t &dtype) { - return utils::get_sizeof_etype(dtype.type_code_) * 8 < 32; - } - - expr_c visit(tensor_c v) override { - // avoid dispatch into for loop index dependent tensor - return v; - } - - expr_c visit(cast_c v) override { - auto vv = ir_visitor_t::visit(std::move(v)).dyn_as(); - assert(vv.defined()); - auto src = vv->in_; - const auto src_dtype = src->dtype_; - const auto dst_dtype = vv->dtype_; - auto is_32bit_avx2_cast = [&](const sc_data_etype &cmp_dst_type, - const sc_data_etype &cmp_src_type) { - return src_dtype.lanes_ > 1 && (dst_dtype.type_code_ == cmp_dst_type) && - (src_dtype.type_code_ == cmp_src_type); - }; - auto s32_smid_cast = [&](bool is_u16 = false) { - expr cast_u16 = - builder::make_cast(sc_data_type_t::u16(src_dtype.lanes_ * 2), src); - expr permutexvar_expr = builder::make_permutexvar( - builder::make_constant(UINT64_C(0b11011000)), cast_u16, 4); - expr extract_u16_expr = - builder::make_extract(permutexvar_expr, 0, src_dtype.lanes_); - if (is_u16) { - return extract_u16_expr; - } - expr cast_dst = builder::make_cast( - sc_data_type_t(dst_dtype.type_code_, src_dtype.lanes_ * 2), - extract_u16_expr); - // must cast to index, inorder to use reg64 in extract instruction - expr reinpt_1 = builder::make_reinterpret( - cast_dst, sc_data_type_t(sc_data_etype::INDEX, 2)); - expr extract_64bit = builder::make_extract(reinpt_1, 0); - expr reinpt_2 = builder::make_reinterpret(extract_64bit, dst_dtype); - return reinpt_2; - }; - if ((is_32bit_avx2_cast(sc_data_etype::U8, sc_data_etype::S32) || - is_32bit_avx2_cast(sc_data_etype::S8, sc_data_etype::S32) || - is_32bit_avx2_cast(sc_data_etype::S8, sc_data_etype::U32) || - is_32bit_avx2_cast(sc_data_etype::U8, sc_data_etype::U32))) { - return s32_smid_cast(); - } else if ((is_32bit_avx2_cast(sc_data_etype::U16, sc_data_etype::U32) || - is_32bit_avx2_cast(sc_data_etype::U16, sc_data_etype::S32))) { - return s32_smid_cast(true); - } - return vv; - } - - expr_c visit(intrin_call_c v) override { - auto vv = ir_visitor_t::visit(std::move(v)).static_as(); - auto dst_dtype = vv->dtype_; - switch (vv->type_) { - case intrin_type::rsqrt: { - if (dst_dtype.is_etype(sc_data_etype::F32)) { - // AVX2 rsqrt have low precision. Do as llvm - // (rcpps(sqrtps)), 1/sqrt. - std::vector init_val(dst_dtype.lanes_, 1.f); - return builder::make_constant(init_val, dst_dtype) / - builder::make_sqrt(vv->args_[0]); - } - } break; - case intrin_type::saturated_cast: { - // Currently our project just need u32s32 to u8s8 or u16. - assert(utils::is_one_of(dst_dtype.type_code_, sc_data_etype::S8, - sc_data_etype::U16, sc_data_etype::U8)); - assert(utils::is_one_of(vv->args_[0]->dtype_.type_code_, - sc_data_etype::S32, sc_data_etype::U32)); - // In avx2, cast is saturated cast. - return builder::make_cast(dst_dtype, vv->args_[0]); - }; break; - default: - break; - } - return vv; - } - - expr_c visit(cmp_c v) override { - auto vv = ir_visitor_t::visit(std::move(v)).dyn_as(); - assert(vv.defined()); - // AVX2 uint have no cmp other than EQ - // AVX2 sint have no cmp other than EQ/GT - auto transform_cmp = [this](const cmp_c &v) -> expr { - const auto src_dtype = v->l_->dtype_; - if (src_dtype.lanes_ > 1) { - switch (get_etype_category(src_dtype)) { - case type_category::CATE_UINT: { - return avx_uint_cmp(v); - } break; - case type_category::CATE_INT: { - return avx_sint_cmp(v); - } break; - case type_category::CATE_FLOAT: { - return avx_float_cmp(v); - } break; - default: - break; // No need to transform. - } - } - return v.remove_const(); - }; - // - const auto cmp_dtype = vv->dtype_; - return avx_mask_cast(transform_cmp(vv), cmp_dtype); - } - - expr_c visit(select_c v) override { - // AVX2 mask must cast to the data type it is masking - auto vv = ir_visitor_t::visit(std::move(v)).dyn_as(); - assert(vv.defined()); - auto dtype = vv->dtype_; - if (dtype != vv->cond_->dtype_) { - return builder::make_select(avx_mask_cast(vv->cond_, dtype), vv->l_, - vv->r_); - } else { - return vv; - } - } - - expr_c visit(indexing_c v) override { - // AVX2 mask must cast to the data type it is masking - auto vv = ir_visitor_t::visit(std::move(v)).dyn_as(); - assert(vv.defined()); - auto dtype = vv->dtype_; - if (vv->mask_.defined() && dtype != vv->mask_->dtype_) { - // We process u8s8 and bf16 datatype one by one like llvm. - if (is_base_type_less_than_32bit(dtype)) { - return vv; - } else { - return builder::make_indexing(vv->ptr_, vv->idx_, dtype.lanes_, - avx_mask_cast(vv->mask_, dtype)); - } - } else { - return vv; - } - } - - void assign_var_stmt(expr &target_var, expr &data, const int imm, - const int elem_bits, indexing_c &vv, - std::vector &then_block_list, - std::vector &tmp_offset) const { - // insert - if (INTRIN_TYPE == INSERT_INTRIN) { - then_block_list.emplace_back(builder::make_assign_unattached( - target_var, builder::make_insert(target_var, data, imm))); - } else if (INTRIN_TYPE == EXTRACT_INTRIN) { // extract - then_block_list.emplace_back(builder::make_assign_unattached( - builder::make_indexing(vv->ptr_, tmp_offset), - builder::make_extract(data, imm))); - } else { - assert(false && "Expect insert or extract intrin type."); - } - } - -#define PARAM(X) X -#define DEFINE_OFFSET_ITER_VAR() \ - auto offset_var = builder::make_var( \ - datatypes::index, "offset_var" + std::to_string(var_index++)); \ - cur_list.emplace_back(builder::make_var_tensor_def_unattached( \ - offset_var, linkage::local, \ - builder::make_constant({0UL}, datatypes::index))); \ - std::vector base = vv->idx_; \ - std::vector tmp_offset(base); \ - tmp_offset[tmp_offset.size() - 1] = \ - tmp_offset[tmp_offset.size() - 1] + offset_var; \ - expr iter_var = builder::make_var(datatypes::s32, \ - "iter_var" + std::to_string(var_index++)); \ - cur_list.emplace_back( \ - builder::make_var_tensor_def_unattached(iter_var, linkage::local, 1)); \ - auto mask = builder::make_var(datatypes::s32, \ - "mask_var" + std::to_string(var_index++)); \ - cur_list.emplace_back(builder::make_var_tensor_def_unattached( \ - mask, linkage::local, builder::make_cast(datatypes::s32, vv->mask_))); - -#define INSERT_EXTRACT_DATA_TO_VAR(target_var, data, elembits, imm) \ - assign_var_stmt(target_var, data, imm, elem_bits, vv, then_block_list, \ - tmp_offset); \ - then_block = builder::make_stmts_unattached(then_block_list); \ - cur_list.emplace_back( \ - builder::make_if_else_unattached(mask >= iter_var, then_block, stmt())); \ - then_block_list.clear(); \ - cur_list.emplace_back(builder::make_assign_unattached( \ - offset_var, \ - builder::make_add(offset_var, \ - builder::make_constant({1UL}, datatypes::index)))); \ - cur_list.emplace_back(builder::make_assign_unattached( \ - iter_var, builder::make_add(builder::make_shl(iter_var, 1), 1))); - -#define INSERT_EXTRACT_DATA_bf16(var_name, data) \ - for (size_t di = 0; di < 8; di++) { \ - INSERT_EXTRACT_DATA_TO_VAR(var_name, data, elem_bits, di) \ - } - -#define INSERT_EXTRACT_DATA_u8s8(var_name, data) \ - for (size_t di = 0; di < 16; di++) { \ - INSERT_EXTRACT_DATA_TO_VAR(var_name, data, elem_bits, di) \ - } - - stmt_c insert_byte_by_byte(const expr &value, const expr &var) { - auto dtype = var->dtype_; - if (value.defined() && (value->node_type_ == sc_expr_type::indexing)) { - auto vv = value.static_as(); - if (vv->mask_.defined() && is_base_type_less_than_32bit(dtype)) { - INTRIN_TYPE = INSERT_INTRIN; - std::vector cur_list, body_list; - std::vector then_block_list; - stmt then_block; - expr insert_var_1, insert_var_2; - - const int lanes = dtype.lanes_; - bool is_s8 = dtype.is_etype(sc_data_etype::S8); - bool is_bf16 = dtype.is_etype(sc_data_etype::BF16); - const int elem_bits = is_bf16 ? 16 : 8; - const int type_bits = 128; - bool need_2_xmm = is_bf16 ? lanes > 8 : lanes > 16; - const int type_lanes = need_2_xmm ? lanes / 2 : lanes; - sc_data_type_t insert_type = is_bf16 ? sc_data_type_t::bf16(type_lanes) - : is_s8 ? sc_data_type_t::s8(type_lanes) - : sc_data_type_t::u8(type_lanes); - - // original defined var - cur_list.emplace_back(builder::make_var_tensor_def_unattached( - var, linkage::local, builder::make_constant({0UL}, dtype))); - - DEFINE_OFFSET_ITER_VAR(); - - auto data = builder::make_indexing(vv->ptr_, tmp_offset); - - auto insert_kernel = [&](expr &insert_var, expr &data) { - if (is_bf16) { - INSERT_EXTRACT_DATA_bf16(insert_var, data); - } else { - INSERT_EXTRACT_DATA_u8s8(insert_var, data); - } - }; - - auto make_insert_var = [&](expr &var) { - var = builder::make_var(insert_type, - "inser_var" + std::to_string(var_index++)); - cur_list.emplace_back(builder::make_var_tensor_def_unattached( - var, linkage::local, builder::make_constant({0UL}, insert_type))); - }; - - make_insert_var(insert_var_1); - if (need_2_xmm) { - make_insert_var(insert_var_2); - } - insert_kernel(insert_var_1, data); - if (!need_2_xmm) { - cur_list.emplace_back( - builder::make_assign_unattached(var, insert_var_1)); - } else { - cur_list.emplace_back(builder::make_assign_unattached( - var, builder::make_insert(var, insert_var_1, 0))); - insert_kernel(insert_var_2, data); - cur_list.emplace_back(builder::make_assign_unattached( - var, builder::make_insert(var, insert_var_2, 1))); - } - - auto cur_body = builder::make_stmts_unattached(cur_list); - return cur_body; - } else { - return stmt(); - } - } - return stmt(); - } - - stmt_c extract_byte_by_byte(const expr &value, const expr &var) { - auto dtype = value->dtype_; - if (var.defined() && (var->node_type_ == sc_expr_type::indexing)) { - auto vv = var.static_as(); - if (vv->mask_.defined() && is_base_type_less_than_32bit(dtype)) { - INTRIN_TYPE = EXTRACT_INTRIN; - const int extract_bits = 128; - std::vector cur_list, body_list; - std::vector then_block_list; - stmt then_block; - const int lanes = dtype.lanes_; - bool is_s8 = dtype.is_etype(sc_data_etype::S8); - bool is_bf16 = dtype.is_etype(sc_data_etype::BF16); - const int elem_bits = is_bf16 ? 16 : 8; - const int type_bits = 128; - bool need_2_xmm = is_bf16 ? lanes > 8 : lanes > 16; - const int type_lanes = need_2_xmm ? lanes / 2 : lanes; - sc_data_type_t extract_type = is_bf16 ? sc_data_type_t::bf16(type_lanes) - : is_s8 ? sc_data_type_t::s8(type_lanes) - : sc_data_type_t::u8(type_lanes); - expr extract_var_1, extract_var_2; - - auto make_extract_var = [&](expr &var, const int imm) { - var = builder::make_var(extract_type, - "extract_var" + std::to_string(var_index++)); - cur_list.emplace_back(builder::make_var_tensor_def_unattached(var)); - cur_list.emplace_back(builder::make_assign_unattached( - var, builder::make_extract(value, imm, type_lanes))); - }; - - // We need to extract value to xmm register. - if (need_2_xmm) { - make_extract_var(extract_var_1, 0); - make_extract_var(extract_var_2, 1); - } else { - extract_var_1 = builder::make_var( - extract_type, "extract_var" + std::to_string(var_index++)); - cur_list.emplace_back( - builder::make_var_tensor_def_unattached(extract_var_1)); - cur_list.emplace_back( - builder::make_assign_unattached(extract_var_1, value)); - } - - DEFINE_OFFSET_ITER_VAR(); - - if (is_bf16) { - INSERT_EXTRACT_DATA_bf16(extract_var_1, extract_var_1); - if (need_2_xmm) { - INSERT_EXTRACT_DATA_bf16(extract_var_2, extract_var_2); - } - } else { - INSERT_EXTRACT_DATA_u8s8(extract_var_1, extract_var_1); - if (need_2_xmm) { - INSERT_EXTRACT_DATA_u8s8(extract_var_2, extract_var_2); - } - } - - auto cur_body = builder::make_stmts_unattached(cur_list); - return cur_body; - } else { - return stmt(); - } - } - return stmt(); - } - - stmt_c visit(define_c v) override { - auto defined = ir_visitor_t::visit(std::move(v)).dyn_as(); - assert(defined.defined()); - auto var = defined->var_; - auto value = defined->init_; - if (value.defined() && (value->node_type_ == sc_expr_type::indexing)) { - auto res = insert_byte_by_byte(value, var); - return res.defined() ? res : defined; - } - return defined; - } - - stmt_c visit(assign_c v) override { - auto assign = ir_visitor_t::visit(std::move(v)).dyn_as(); - assert(assign.defined()); - auto var = assign->var_; - auto value = assign->value_; - if (var.defined() && (var->node_type_ == sc_expr_type::indexing)) { - auto res = extract_byte_by_byte(value, var); - return res.defined() ? res : assign; - } else if (value.defined() && value->node_type_ == sc_expr_type::indexing) { - auto res = insert_byte_by_byte(value, var); - return res.defined() ? res : assign; - } - return assign; - } - -protected: - // repeat var index - uint32_t var_index = 1; - const int INSERT_INTRIN = 0; - const int EXTRACT_INTRIN = 1; - int INTRIN_TYPE = -1; - - // Cast AVX2 mask - expr avx_mask_cast(const expr &v, sc_data_type_t dtype) { - auto nested = v.cast().filter( - [dtype](const low_level_intrin_c &v) { - return v->kind_ == low_level_intrin_kind::x86_general && - v->type_ == x86_intrin_type::avx_mask_cast && - v->args_[0]->dtype_ == dtype; - }); - if (nested.has_value()) { - // eliminate unnecessary nested mask cast - return nested.get()->args_[0]; - } else if (dtype.lanes_ > 1 || v->dtype_.lanes_ > 1) { - return builder::make_x86_intrin(x86_intrin_type::avx_mask_cast, {v}, - {{"dtype", dtype}}); - } else { - return v; - } - } - - // Transform for AVX2 uint compare - expr avx_uint_cmp(const cmp_c &v) { - using sc_etype = sc_expr_type; - const auto &t = v->node_type_; - const auto &l = v->l_; - const auto &r = v->r_; - switch (t) { - case sc_etype::cmp_eq: - return transform_uint_eq(l, r); - case sc_etype::cmp_lt: - return transform_uint_lt(l, r); - case sc_etype::cmp_le: - return transform_uint_le(l, r); - case sc_etype::cmp_ne: - return transform_uint_ne(l, r); - case sc_etype::cmp_ge: - return transform_uint_ge(l, r); - case sc_etype::cmp_gt: - return transform_uint_gt(l, r); - default: - COMPILE_ASSERT(false, "Invalid compare type: " << t); - } - return v.remove_const(); - } - expr transform_uint_eq(const expr &l, const expr &r) { - auto code = static_cast(xbyak_condition::eq); - return builder::make_x86_intrin(x86_intrin_type::avx_compare, - {l, r, builder::make_constant(code)}); - } - expr transform_uint_ne(const expr &l, const expr &r) { - auto ones = builder::make_constant({INT64_C(-1)}, l->dtype_); - return builder::make_int_xor(transform_uint_eq(l, r), ones); - } - expr transform_uint_ge(const expr &l, const expr &r) { - return transform_uint_eq(builder::make_max(l, r), l); - } - expr transform_uint_le(const expr &l, const expr &r) { - return transform_uint_ge(r, l); - } - expr transform_uint_gt(const expr &l, const expr &r) { - auto ones = builder::make_constant({INT64_C(-1)}, l->dtype_); - return builder::make_int_xor(transform_uint_le(l, r), ones); - } - expr transform_uint_lt(const expr &l, const expr &r) { - return transform_uint_gt(r, l); - } - - // Transform for AVX2 sint compare - expr avx_sint_cmp(const cmp_c &v) { - using sc_etype = sc_expr_type; - const auto &t = v->node_type_; - const auto &l = v->l_; - const auto &r = v->r_; - switch (t) { - case sc_etype::cmp_eq: - return transform_sint_eq(l, r); - case sc_etype::cmp_lt: - return transform_sint_lt(l, r); - case sc_etype::cmp_le: - return transform_sint_le(l, r); - case sc_etype::cmp_ne: - return transform_sint_ne(l, r); - case sc_etype::cmp_ge: - return transform_sint_ge(l, r); - case sc_etype::cmp_gt: - return transform_sint_gt(l, r); - default: - COMPILE_ASSERT(false, "Invalid compare type: " << t); - } - return v.remove_const(); - } - expr transform_sint_eq(const expr &l, const expr &r) { - auto code = static_cast(xbyak_condition::eq); - return builder::make_x86_intrin(x86_intrin_type::avx_compare, - {l, r, builder::make_constant(code)}); - } - expr transform_sint_ne(const expr &l, const expr &r) { - auto ones = builder::make_constant({INT64_C(-1)}, l->dtype_); - return builder::make_int_xor(transform_sint_eq(l, r), ones); - } - expr transform_sint_ge(const expr &l, const expr &r) { - return transform_sint_eq(builder::make_max(l, r), l); - } - expr transform_sint_le(const expr &l, const expr &r) { - return transform_sint_ge(r, l); - } - expr transform_sint_gt(const expr &l, const expr &r) { - auto code = static_cast(xbyak_condition::gt); - return builder::make_x86_intrin(x86_intrin_type::avx_compare, - {l, r, builder::make_constant(code)}); - } - expr transform_sint_lt(const expr &l, const expr &r) { - return transform_sint_gt(r, l); - } - - // Transform for AVX2 float compare - expr avx_float_cmp(const cmp_c &v) { - auto code = static_cast(get_xbyak_condition(v->node_type_)); - return builder::make_x86_intrin( - x86_intrin_type::avx_compare, - {v->l_, v->r_, builder::make_constant(code)}); - } -}; - -func_c avx2_legalizer_t::operator()(func_c v) { - // No need for AVX2 legalization when AVX512 is available - if (target_machine_.cpu_flags_.fAVX512F) { - return v; - } - avx2_legalizer_impl_t avx2_legalizer; - return avx2_legalizer.dispatch(std::move(v)); -} - -} // namespace xbyak -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/jit/xbyak/ir/transform/avx2_legalizer.hpp b/legacy/core/src/compiler/jit/xbyak/ir/transform/avx2_legalizer.hpp deleted file mode 100644 index 3e5583612..000000000 --- a/legacy/core/src/compiler/jit/xbyak/ir/transform/avx2_legalizer.hpp +++ /dev/null @@ -1,49 +0,0 @@ -/******************************************************************************* - * Copyright 2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_JIT_XBYAK_IR_TRANSFORM_AVX2_LEGALIZER_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_JIT_XBYAK_IR_TRANSFORM_AVX2_LEGALIZER_HPP - -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace xbyak { - -/* * - * avx2 legalizer for xbyak backend, make sure data type and operation is - * compatible with avx2 intrinsic requirement. - * */ -class avx2_legalizer_t : public function_pass_t { -public: - avx2_legalizer_t(const runtime::target_machine_t &target_machine) - : target_machine_(target_machine) {} - func_c operator()(func_c v) override; - -private: - const runtime::target_machine_t &target_machine_; -}; - -} // namespace xbyak -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/jit/xbyak/ir/transform/avx2_mask_indexing_transform.cpp b/legacy/core/src/compiler/jit/xbyak/ir/transform/avx2_mask_indexing_transform.cpp deleted file mode 100644 index 8f27b3ab0..000000000 --- a/legacy/core/src/compiler/jit/xbyak/ir/transform/avx2_mask_indexing_transform.cpp +++ /dev/null @@ -1,131 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include -#include -#include - -#include -#include -#include - -#include "avx2_mask_indexing_transform.hpp" -#include "util/utils.hpp" - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace xbyak { - -class avx2_mask_indexing_transform_impl_t : public ir_visitor_t { -public: - using ir_visitor_t::dispatch; - using ir_visitor_t::visit; - - std::vector idx_transform_stmt; - std::unordered_map idx_var_map; - - avx2_mask_indexing_transform_impl_t() = default; - - bool is_base_type_less_than_32bit(const sc_data_type_t &dtype) { - return utils::get_sizeof_etype(dtype.type_code_) * 8 < 32; - } - - stmt_c process_scope_indexing(const expr &value, const expr &var, - const stmt_c &v) { - auto is_mask_indexing = [&](const expr &inp) { - return (inp.defined() && inp->node_type_ == sc_expr_type::indexing && - inp.dyn_as()->mask_.defined() && - is_base_type_less_than_32bit(inp->dtype_)); - }; - // a = min(b, u16x8(A[i@M])) or A[i@M] = B[i@M] - if ((value.defined() && value->node_type_ != sc_expr_type::indexing) || - (is_mask_indexing(var) && is_mask_indexing(value))) { - assert(idx_transform_stmt.empty()); - auto vv = dispatch(value); - if (!idx_transform_stmt.empty()) { - std::vector cur_list(std::move(idx_transform_stmt)); - if (v.isa()) { - cur_list.emplace_back(builder::make_var_tensor_def_unattached( - var, v.dyn_as()->linkage_, vv)); - } else { - cur_list.emplace_back(builder::make_assign_unattached(var, vv)); - } - idx_var_map.clear(); - idx_transform_stmt.clear(); - return builder::make_stmts_unattached(cur_list); - } - } - return v; - } - - expr_c visit(tensor_c v) override { - // avoid dispatch into for loop index dependent tensor - return v; - } - - expr_c visit(indexing_c v) override { - auto vv = ir_visitor_t::visit(v).dyn_as(); - if (vv.defined() && is_base_type_less_than_32bit(vv->dtype_) && - vv->mask_.defined()) { - if (idx_var_map.find(vv) != idx_var_map.end()) { - return idx_var_map[vv]; - } - auto nested_idx_var = builder::make_var( - vv->dtype_, "indexing_nested" + std::to_string(count_++)); - auto nested_var_define = builder::make_var_tensor_def_unattached( - nested_idx_var, linkage::local); - auto nested_assign = builder::make_assign_unattached(nested_idx_var, vv); - std::vector cur_list; - cur_list.emplace_back(nested_var_define); - cur_list.emplace_back(nested_assign); - auto nested_stmts = builder::make_stmts_unattached(cur_list); - idx_transform_stmt.emplace_back(nested_stmts); - idx_var_map.insert(std::make_pair(vv, nested_idx_var)); - return nested_idx_var; - } - return vv; - } - - stmt_c visit(define_c v) override { - // multiple nested expr has indexing - return process_scope_indexing(v->init_, v->var_, v); - } - - stmt_c visit(assign_c v) override { - // multiple nested expr has indexing - return process_scope_indexing(v->value_, v->var_, v); - } - -private: - size_t count_ = 0; -}; - -func_c avx2_mask_indexing_t::operator()(func_c v) { - // No need for AVX2 legalization when AVX512 is available - if (target_machine_.cpu_flags_.fAVX512F) { - return v; - } - avx2_mask_indexing_transform_impl_t avx2_indexing_transform; - return avx2_indexing_transform.dispatch(std::move(v)); -} - -} // namespace xbyak -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/jit/xbyak/ir/transform/avx2_mask_indexing_transform.hpp b/legacy/core/src/compiler/jit/xbyak/ir/transform/avx2_mask_indexing_transform.hpp deleted file mode 100644 index 56d78f43b..000000000 --- a/legacy/core/src/compiler/jit/xbyak/ir/transform/avx2_mask_indexing_transform.hpp +++ /dev/null @@ -1,49 +0,0 @@ -/******************************************************************************* - * Copyright 2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_JIT_XBYAK_IR_TRANSFORM_AVX2_MASK_INDEXING_TRANSFORM_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_JIT_XBYAK_IR_TRANSFORM_AVX2_MASK_INDEXING_TRANSFORM_HPP - -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace xbyak { - -/* * - * avx2 mask indexing transformation for xbyak backend, make sure indexing is - * not nested. So that avx2 legalize pass can handle indexing correctly. - * */ -class avx2_mask_indexing_t : public function_pass_t { -public: - avx2_mask_indexing_t(const runtime::target_machine_t &target_machine) - : target_machine_(target_machine) {} - func_c operator()(func_c v) override; - -private: - const runtime::target_machine_t &target_machine_; -}; - -} // namespace xbyak -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/jit/xbyak/ir/transform/call_transform.cpp b/legacy/core/src/compiler/jit/xbyak/ir/transform/call_transform.cpp deleted file mode 100644 index 0526930c2..000000000 --- a/legacy/core/src/compiler/jit/xbyak/ir/transform/call_transform.cpp +++ /dev/null @@ -1,242 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include -#include -#include -#include - -#include -#include -#include - -#include "call_transform.hpp" - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace xbyak { -using namespace xbyak::x86_64; - -// Cached ABI infomation inside func node -abi_function_interface::ptr cached_func_abi_interface(const func_t &v) { - assert(v->attr_ && v->attr_->has_key(attr_keys::abi_interface)); - auto func_iface = - v->attr_->get(attr_keys::abi_interface); - return func_iface; -} - -// Cached ABI infomation inside call node -abi_function_interface::ptr cached_call_abi_interface(const call_c &v) { - return cached_func_abi_interface(v->get_prototype()); -} - -/* * - * Extract call node as root expr node, so it is easier to transform individual - * call into a call scope stmts. - * */ -class call_extract_impl_t : public ir_visitor_t { -public: - using ir_visitor_t::dispatch; - using ir_visitor_t::visit; - - call_extract_impl_t() = default; - - size_t index_ = 0; - size_t expr_depth_ = 0; - - std::vector stmt_seq_; - - expr_c dispatch(expr_c v) override { - expr_depth_++; - auto ret = ir_visitor_t::dispatch(std::move(v)); - expr_depth_--; - if (ret.isa() && expr_depth_ > 0) { - return extract_call(ret.remove_const()); - } - return ret; - } - - stmt_c visit(stmts_c v) override { - std::vector new_seq; - for (auto &s : v->seq_) { - auto ss = ir_visitor_t::dispatch(s); - for (auto &&ts : stmt_seq_) { - new_seq.push_back(std::move(ts)); - } - new_seq.emplace_back(ss.remove_const()); - stmt_seq_.clear(); - } - return copy_attr(*v, make_stmt(std::move(new_seq))); - } - - expr extract_call(expr v) { - auto call_var = builder::make_var(v->dtype_, "tmp_call_var_" + - std::to_string(index_++)); - stmt_seq_.emplace_back( - make_stmt(call_var, linkage::local, std::move(v))); - return call_var; - } -}; - -/* * - * Transform each call node to dedicated stmts node, mark the node as function - * call scope, add reverse order args inside scope. And make arg passed to %rdx - * reg to be the last arg assigned (%rdx is a special reg, may used by certain - * operations inside call scope) - * */ -class call_transform_impl_t : public ir_visitor_t { -public: - using ir_visitor_t::dispatch; - using ir_visitor_t::visit; - - call_transform_impl_t(const x86_64::target_profile_t &profile) - : profile_(profile) {} - - const x86_64::target_profile_t &profile_; - - abi_function_interface::ptr func_iface_; - - std::vector new_seq_; - std::vector new_rdx_; - - size_t index_ = 0; - - func_c dispatch(func_c v) override { - func_t func = std::const_pointer_cast(v); - cache_abi_interface(func); - return ir_visitor_t::dispatch(std::move(v)); - } - - // Reverse call args order - expr_c dispatch(expr_c v) override { - if (v.isa()) { - auto vv = v.static_as(); - func_t callee = vv->get_prototype(); - func_iface_ = cache_abi_interface(callee); - auto &old_args = vv->args_; - std::vector new_args; - for (size_t i = 0; i < old_args.size(); i++) { - bool is_reg_rdx = false; - auto &arg = old_args[i]; - auto &initial_loc = func_iface_->param_locs_[i]; - // Create new arg placeholder to be replace args in call node - auto arg_placeholder = builder::make_var( - arg->dtype_, - "call_arg_" + std::to_string(index_) + "_" + std::to_string(i + 1)); - // Create wrapper for actural arg value - auto arg_wrapper = - make_xbyak_intrin(arg->dtype_, {arg}, xbyak_intrin_type::call_arg); - // Get currrent arg location, %rdx is spcical arg - if (initial_loc.get_type() == abi_value_location::tag_type::REGISTER) { - Xbyak::Reg arg_reg = initial_loc.get_register(); - if (arg_reg.isREG() && arg_reg.getIdx() == Xbyak::Operand::RDX) { - is_reg_rdx = true; - } - } - // Arg passed to %rdx reg need to be the last assigned - if (is_reg_rdx) { - new_rdx_.emplace_back(make_stmt( - arg_placeholder, linkage::local, arg_wrapper)); - } else { - new_seq_.emplace_back(make_stmt( - arg_placeholder, linkage::local, arg_wrapper)); - } - // Add to new call node args - new_args.emplace_back(std::move(arg_placeholder)); - } - index_++; - return copy_attr(*vv, - make_expr(vv->func_, std::move(new_args))); - } else { - // After extraction, call node only exists in root expr node. - return v; - } - } - - stmt_c transform_call_stmt(bool is_call, stmt_c v) { - if (is_call) { - std::reverse(new_seq_.begin(), new_seq_.end()); - if (!new_rdx_.empty()) { - new_seq_.emplace_back(std::move(new_rdx_.back())); - } - new_seq_.emplace_back(v.remove_const()); - auto ret = make_stmt(std::move(new_seq_)); - ret->attr().set(attr_keys::abi_interface, func_iface_); - new_seq_.clear(); - new_rdx_.clear(); - return ret; - } else { - return v; - } - } - - stmt_c visit(define_c v) override { - bool is_call = v->init_.defined() && v->init_.isa(); - auto vv = ir_visitor_t::visit(std::move(v)); - return transform_call_stmt(is_call, std::move(vv)); - } - - stmt_c visit(assign_c v) override { - bool is_call = v->value_.defined() && v->value_.isa(); - auto vv = ir_visitor_t::visit(std::move(v)); - return transform_call_stmt(is_call, std::move(vv)); - } - - stmt_c visit(evaluate_c v) override { - bool is_call = v->value_.defined() && v->value_.isa(); - auto vv = ir_visitor_t::visit(std::move(v)); - return transform_call_stmt(is_call, std::move(vv)); - } - - stmt_c visit(returns_c v) override { - bool is_call = v->value_.defined() && v->value_.isa(); - auto vv = ir_visitor_t::visit(std::move(v)); - return transform_call_stmt(is_call, std::move(vv)); - } - - abi_function_interface::ptr cache_abi_interface(func_t &v) { - if (v->attr_ && v->attr_->has_key(attr_keys::abi_interface)) { - auto func_iface = - v->attr_->get(attr_keys::abi_interface); - return func_iface; - } else { - auto func_iface = abi_function_interface::make_interface( - profile_, v->params_, v->ret_type_); - v->attr().set(attr_keys::abi_interface, func_iface); - return func_iface; - } - } -}; - -func_c call_transform_t::operator()(func_c v) { - if (v->name_.find("_should_inline_") != std::string::npos) { - return v; - } - call_extract_impl_t call_extract; - call_transform_impl_t call_transform(profile_); - return call_transform.dispatch(call_extract.dispatch(std::move(v))); -} - -call_transform_t::call_transform_t(const x86_64::target_profile_t &profile) - : profile_(profile) {} - -} // namespace xbyak -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/jit/xbyak/ir/transform/call_transform.hpp b/legacy/core/src/compiler/jit/xbyak/ir/transform/call_transform.hpp deleted file mode 100644 index 624997630..000000000 --- a/legacy/core/src/compiler/jit/xbyak/ir/transform/call_transform.hpp +++ /dev/null @@ -1,64 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_JIT_XBYAK_IR_TRANSFORM_CALL_TRANSFORM_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_JIT_XBYAK_IR_TRANSFORM_CALL_TRANSFORM_HPP - -#include -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace xbyak { - -namespace attr_keys { -// attr for func and call scope, contains x86 calling convention info -// data type: x86_64::abi_function_interface::ptr -constexpr const char *abi_interface = "abi_interface"; -} // namespace attr_keys - -#define TRANSFORMED_CALL(STMT) \ - (STMT->attr_ && STMT->attr_->has_key(attr_keys::abi_interface)) - -// Cached ABI infomation inside func and call node -x86_64::abi_function_interface::ptr cached_func_abi_interface(const func_t &v); -x86_64::abi_function_interface::ptr cached_call_abi_interface(const call_c &v); - -/* * - * Extract call node as root expr node and make dedicated stmts for each call - * node, mark the stmts as function call scope, add reverse order args inside - * scpoe. The process is aimed to transform all function call temp calculations - * into a local scope, so that the register utilization can be much higher. - * */ -class call_transform_t : public function_pass_t { -public: - call_transform_t(const x86_64::target_profile_t &profile); - func_c operator()(func_c v) override; - -private: - const x86_64::target_profile_t &profile_; -}; - -} // namespace xbyak -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/jit/xbyak/ir/transform/constant_optimizer.cpp b/legacy/core/src/compiler/jit/xbyak/ir/transform/constant_optimizer.cpp deleted file mode 100644 index 575ebc93c..000000000 --- a/legacy/core/src/compiler/jit/xbyak/ir/transform/constant_optimizer.cpp +++ /dev/null @@ -1,273 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include -#include - -#include -#include -#include -#include -#include - -#include "constant_optimizer.hpp" - -#define HAS_KEY_ENCODE(EXPR) \ - ((EXPR)->attr_ && (EXPR)->attr_->has_key(attr_keys::force_simd_encode)) - -#define HAS_KEY_LOAD(EXPR) \ - ((EXPR)->attr_ && (EXPR)->attr_->has_key(attr_keys::load_simd_value)) - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace xbyak { - -class constant_optimizer_impl_t : public ir_visitor_t { -public: - using ir_visitor_t::dispatch; - using ir_visitor_t::visit; - - stmt_c visit(define_c v) override { - if (v->init_.defined() && v->init_.isa() && - is_x86_simd(v->init_->dtype_)) { - v->init_->attr().set(attr_keys::load_simd_value, true); - } - return ir_visitor_t::visit(std::move(v)); - } - - stmt_c visit(assign_c v) override { - if (v->value_.isa() && is_x86_simd(v->value_->dtype_)) { - v->value_->attr().set(attr_keys::load_simd_value, true); - } - return ir_visitor_t::visit(std::move(v)); - } - - expr_c visit(tensor_c v) override { - // avoid constant_optimizer for loop index dependent tensor - return v; - } - - expr_c visit(constant_c v) override { - const auto &dtype = v->dtype_; - const auto &value = v->value_; - // Transform multi-lane constant to broadcasted single-lane constant - if (!HAS_KEY_ENCODE(v) && dtype.lanes_ > 1 && value.size() == 1) { - auto new_const = single_lane_constant(v); - new_const->attr().set(attr_keys::force_simd_encode, true); - return builder::make_broadcast(new_const, dtype.lanes_); - } else if (HAS_KEY_LOAD(v)) { - return make_expr(intrin_type::constant_load, - std::vector{v.remove_const()}, - any_map_t()); - } else { - return v; - } - } - - expr_c visit(mul_c v) override { - // If constant is 2^N, transform mul(lhs, 2^n) to shl(rhs, n) - auto transform_const_mul = [this](const expr &lhs, const expr &rhs) { - auto const_rhs = rhs.static_as(); - if (is_power_of_2(const_rhs)) { - auto imm = power_of_2_exponent(const_rhs); - return builder::make_shl(lhs, imm); - } else { - return builder::make_mul(lhs, rhs); - } - }; - // Make constant in multiply always positioned in rhs - auto vv = ir_visitor_t::visit(std::move(v)).static_as(); - if (vv->r_.isa()) { - return transform_const_mul(vv->l_, vv->r_); - } else if (vv->l_.isa()) { - return transform_const_mul(vv->r_, vv->l_); - } else { - return vv; - } - } - - expr_c visit(div_c v) override { - // Transform unsigned div if divisor is a const of power of 2 - if (v->r_.isa() && v->r_->dtype_.is_etype(sc_data_etype::S32)) { - v->r_->attr().set(attr_keys::force_simd_encode, false); - } - auto vv = ir_visitor_t::visit(std::move(v)).static_as(); - if (vv->r_.isa()) { - auto const_rhs = vv->r_.static_as(); - bool transform_pow_2 = - utils::is_one_of(vv->l_->dtype_, datatypes::u8, datatypes::u16, - datatypes::u32, datatypes::index) && - is_power_of_2(const_rhs); - // If constant is 2^N, transform div(x, 2^n) to shr(x, n) - if (transform_pow_2) { - auto imm = power_of_2_exponent(const_rhs); - return builder::make_shr(vv->l_, imm); - } - } - return vv; - } - - expr_c visit(mod_c v) override { - // Transform unsigned mod if divisor is a const of power of 2 - auto vv = ir_visitor_t::visit(std::move(v)).static_as(); - if (vv->r_.isa()) { - auto const_rhs = vv->r_.static_as(); - bool transform_pow_2 = - utils::is_one_of(vv->l_->dtype_, datatypes::u8, datatypes::u16, - datatypes::u32, datatypes::index) && - is_power_of_2(const_rhs); - // If constant is 2^N, transform mod(x, 2^n) to int_and(x, 2^n - 1) - if (transform_pow_2) { - auto imm = power_of_2_minus_1(const_rhs); - return builder::make_int_and(vv->l_, imm); - } - } - return vv; - } - - expr_c visit(select_c v) override { - // Keep zero blend for potential simd optimization - if (v->r_.isa()) { - auto const_rhs = v->r_.static_as(); - if (is_zero(const_rhs)) { - const_rhs->attr().set(attr_keys::force_simd_encode, false); - } - } - return ir_visitor_t::visit(std::move(v)); - } - - expr_c visit(intrin_call_c v) override { - // convert same value constant simd shift to scalar value - // so xbyak can use imm in shift function - auto transform_shift = [this](intrin_call_c v) -> expr_c { - assert(v->args_.size() == 2); - auto &l = v->args_[0]; - auto &r = v->args_[1]; - if (r.isa()) { - auto const_rhs = r.static_as(); - if (const_rhs->value_.size() == 1 && const_rhs->dtype_.lanes_ > 1) { - auto lhs = ir_visitor_t::dispatch(l).remove_const(); - auto rhs = single_lane_constant(const_rhs); - return make_expr( - v->type_, std::vector{lhs, rhs}, any_map_t()); - } - } - return ir_visitor_t::visit(std::move(v)); - }; - // mark constant broadcast value as simd encode - auto transform_broadcast = [this](intrin_call_c v) -> expr_c { - assert(v->args_.size() == 1); - auto &r = v->args_[0]; - if (r.isa()) { - r->attr().set(attr_keys::force_simd_encode, true); - } - return ir_visitor_t::visit(std::move(v)); - }; - // Transform intrin_call node with const value - switch (v->type_) { - case intrin_type::shl: - case intrin_type::shr: { - return transform_shift(std::move(v)); - } break; - case intrin_type::broadcast: { - return transform_broadcast(std::move(v)); - } break; - default: - break; - } - return ir_visitor_t::visit(std::move(v)); - } - - // TODO(XXX): Maybe create a dedicated x86 legalize pass in the future - // make sure x86 imm always rhs, just like expr_c visit(mul_c v) - expr_c visit(cmp_c v) override { - auto make_swap_cmp = [&](sc_expr_type t, const expr_c &lhs, - const expr_c &rhs) { - using sc_etype = sc_expr_type; - switch (t) { - case sc_etype::cmp_eq: - return builder::make_cmp_eq(rhs, lhs); - case sc_etype::cmp_lt: - return builder::make_cmp_gt(rhs, lhs); - case sc_etype::cmp_le: - return builder::make_cmp_ge(rhs, lhs); - case sc_etype::cmp_ne: - return builder::make_cmp_ne(rhs, lhs); - case sc_etype::cmp_ge: - return builder::make_cmp_le(rhs, lhs); - case sc_etype::cmp_gt: - return builder::make_cmp_lt(rhs, lhs); - default: - COMPILE_ASSERT(false, "Invalid compare type: " << t); - } - return expr(); - }; - // swap cmp(l, r) to equivalent cmp(r, l) - if (v->l_.isa()) { - return make_swap_cmp(v->node_type_, v->l_, ir_visitor_t::dispatch(v->r_)); - } else { - return ir_visitor_t::visit(std::move(v)); - } - } - - bool is_zero(const constant &v) { - uint64_t val = v->value_[0].u64; - return (v->value_.size() == 1) && (val == 0); - } - - bool is_power_of_2(const constant &v) { - const auto dtype = v->dtype_; - if (utils::is_one_of(dtype, datatypes::u8, datatypes::u16, datatypes::u32, - datatypes::index)) { - uint64_t val = v->value_[0].u64; - return utils::is_power_of_2(val); - } - return false; - } - - expr power_of_2_exponent(const constant_c &v) { - // Calculate integer log2(2^n) - uint64_t val = v->value_[0].u64; - return builder::make_constant({uint64_t(utils::ctz(val))}, v->dtype_); - } - - expr power_of_2_minus_1(const constant_c &v) { - // Calculate integer 2^n - 1 - uint64_t val = v->value_[0].u64; - return builder::make_constant({val - 1}, v->dtype_); - } - - expr single_lane_constant(const constant_c &v) { - // Transform simd constant to scalar constant - uint64_t val = v->value_[0].u64; - return builder::make_constant({val}, - sc_data_type_t(v->dtype_.type_code_, 1)); - } -}; - -func_c constant_optimizer_t::operator()(func_c v) { - constant_optimizer_impl_t constant_optimizer; - - return constant_optimizer.dispatch(std::move(v)); -} - -} // namespace xbyak -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/jit/xbyak/ir/transform/constant_optimizer.hpp b/legacy/core/src/compiler/jit/xbyak/ir/transform/constant_optimizer.hpp deleted file mode 100644 index 3a47a14dc..000000000 --- a/legacy/core/src/compiler/jit/xbyak/ir/transform/constant_optimizer.hpp +++ /dev/null @@ -1,58 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_JIT_XBYAK_IR_TRANSFORM_CONSTANT_OPTIMIZER_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_JIT_XBYAK_IR_TRANSFORM_CONSTANT_OPTIMIZER_HPP - -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace xbyak { - -namespace attr_keys { -// attr for constant value, mark if constant value need to be encoded -// data type: bool -constexpr const char *force_simd_encode = "force_simd_encode"; -// attr for constant value, mark if simd will be directly load from memory -// data type: bool -constexpr const char *load_simd_value = "load_simd_value"; -} // namespace attr_keys - -#define FORCE_SIMD_ENCODE(EXPR) \ - (EXPR->attr_ && EXPR->attr_->get_or_else(attr_keys::force_simd_encode, false)) - -/* * - * Constant optimizer, mark simd constant and insert broadcast when needed. - * Add simpile strength reduction for constant div/mod/mul. - * */ -class constant_optimizer_t : public function_pass_t { -public: - constant_optimizer_t() = default; - func_c operator()(func_c v) override; - -private: -}; - -} // namespace xbyak -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/jit/xbyak/ir/transform/fp16_legalizer.cpp b/legacy/core/src/compiler/jit/xbyak/ir/transform/fp16_legalizer.cpp deleted file mode 100644 index 6e3b32897..000000000 --- a/legacy/core/src/compiler/jit/xbyak/ir/transform/fp16_legalizer.cpp +++ /dev/null @@ -1,96 +0,0 @@ -/******************************************************************************* - * Copyright 2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include -#include -#include - -#include -#include - -#include "compiler/ir/graph/fusible_op_utils.hpp" -#include "fp16_legalizer.hpp" -#include "util/fp16.hpp" - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -class fp16_legalizer_impl_t : public ir_visitor_t { -public: - using ir_visitor_t::dispatch; - using ir_visitor_t::visit; - - fp16_legalizer_impl_t() = default; - - expr_c visit(tensorptr_c v) override { return v; } - - stmt make_assign(const expr_c &l, const expr_c &r) { - return builder::make_assign_unattached(l, r); - } - - expr make_u16_indexing(const indexing_c &v) { - auto tensor_pointer = - builder::make_reinterpret(v->ptr_, datatypes::u16.get_pointerof()); - return copy_attr(*v, builder::make_indexing(tensor_pointer, v->idx_)); - } - - expr make_reinterpret_f16_u16(const expr_c &v) { - assert(v->dtype_ == datatypes::f16); - return builder::make_reinterpret(v, datatypes::u16); - } - - expr make_reinterpret_u16_f16(const expr_c &v) { - assert(v->dtype_ == datatypes::u16); - return builder::make_reinterpret(v, datatypes::f16); - } - - expr_c visit(indexing_c v) override { - if (v->dtype_ == datatypes::f16) { - return make_reinterpret_u16_f16(make_u16_indexing(v)); - } - return v; - } - - stmt_c visit(assign_c v) override { - auto var = v->var_; - auto value = ir_visitor_t::dispatch(v->value_); - if (var.isa() && var->dtype_ == datatypes::f16) { - var = make_u16_indexing(var.static_as()); - value = make_reinterpret_f16_u16(value); - } - if (!value.ptr_same(v->value_) || !var.ptr_same(v->var_)) { - auto new_stmt = make_assign(var, value); - return copy_attr(*v, std::move(new_stmt)); - } - return v; - } -}; - -func_c fp16_legalizer_t::operator()(func_c v) { - if (target_machine_.cpu_flags_.fAVX512FP16) { - return v; - } - fp16_legalizer_impl_t legalize_transform; - auto ret = legalize_transform.dispatch(std::move(v)); - return ret; -} - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/jit/xbyak/ir/transform/fp16_legalizer.hpp b/legacy/core/src/compiler/jit/xbyak/ir/transform/fp16_legalizer.hpp deleted file mode 100644 index c5ff7a684..000000000 --- a/legacy/core/src/compiler/jit/xbyak/ir/transform/fp16_legalizer.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/******************************************************************************* - * Copyright 2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_JIT_XBYAK_IR_TRANSFORM_FP16_LEGALIZER_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_JIT_XBYAK_IR_TRANSFORM_FP16_LEGALIZER_HPP - -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { - -/* * - * When fp16 must be used on a machine without avx512fp16, the IR must be - * legalized through this pass. - * */ -class fp16_legalizer_t : public function_pass_t { -public: - fp16_legalizer_t(const runtime::target_machine_t &target_machine) - : target_machine_(target_machine) {} - func_c operator()(func_c v) override; - -private: - const runtime::target_machine_t &target_machine_; -}; - -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/jit/xbyak/ir/transform/indexing_transform.cpp b/legacy/core/src/compiler/jit/xbyak/ir/transform/indexing_transform.cpp deleted file mode 100644 index 45475dd7b..000000000 --- a/legacy/core/src/compiler/jit/xbyak/ir/transform/indexing_transform.cpp +++ /dev/null @@ -1,137 +0,0 @@ -/******************************************************************************* - * Copyright 2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "indexing_transform.hpp" - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace xbyak { - -using namespace passlet; - -class indexing_transform_impl_t : public ir_visitor_t { -private: - constant_folder_t constant_folder_; - -public: - using ir_visitor_t::dispatch; - using ir_visitor_t::visit; - - bool is_read_only_tensor(const expr_c &v) { - assert(v.isa()); - // if tesnor is read buffer and has no alias - auto info = alias_info::get_alias_info(*v); - return any_map_t::fetch_or_else(v->attr_.get(), "read_buffer", false) && - (info == nullptr || info->has_no_alias()); - } - - expr_c expand_polynomial_offset(expr_c f) { - return constant_folder_.expand_polynomial(std::move(f), 1, true); - } - - expr_c visit(indexing_c v) override { - auto vv = ir_visitor_t::visit(std::move(v)).checked_as(); - auto &ptr = vv->ptr_; - auto &idx = vv->idx_.back(); - auto idx_type = datatypes::index; - auto elem_type = ptr->dtype_.get_pointer_element(); - auto elem_size = utils::get_sizeof_type(elem_type); - auto is_var_or_const = [](const expr_c &i) { - return i.isa() || i.isa(); - }; - // set read only to the base tensor - auto read_only = is_read_only_tensor(ptr); - ptr->attr().set(attr_keys::read_only_tensor, read_only); - // transform complex indexing to pointer calculation - if (is_var_or_const(idx)) { - // If indexing A[i] already simple, no need to transform - return vv; - } else if (idx.isa() && is_var_or_const(idx.static_as()->r_)) { - // This requires canonicalized expr to work - // The right add op must be most inner var - auto node = idx.static_as(); - auto scale = builder::make_constant({elem_size}, idx_type); - auto offset = expand_polynomial_offset(scale * node->l_); - auto new_ptr = builder::make_cast( - ptr->dtype_, builder::make_cast(idx_type, ptr) + offset); - // transform A[l + r] to (A + (s * l))[r] - auto new_idx = node->r_; - return builder::make_indexing(new_ptr, new_idx, vv->dtype_.lanes_, - vv->mask_); - } else { - auto scale = builder::make_constant({elem_size}, idx_type); - auto offset = expand_polynomial_offset(scale * idx); - auto new_ptr = builder::make_cast( - ptr->dtype_, builder::make_cast(idx_type, ptr) + offset); - // transform A[i] to (A + (s * i))[0] - auto new_idx = builder::make_constant(UINT64_C(0)); - return builder::make_indexing(new_ptr, new_idx, vv->dtype_.lanes_, - vv->mask_); - } - } - - expr_c visit(tensorptr_c v) override { - auto &ptr = v->base_->ptr_; - auto &idx = v->base_->idx_.back(); - auto elem_type = ptr->dtype_.get_pointer_element(); - auto elem_size = utils::get_sizeof_type(elem_type); - auto idx_zero = idx.cast().filter([](const constant &v) { - uint64_t val = v->value_[0].u64; - return (v->value_.size() == 1) && (val == 0); - }); - // transform tensorptr &A[i] to void*(A + (s * i)) - if (idx_zero.has_value()) { - return builder::make_cast(v->dtype_, ptr); - } else { - auto scale = builder::make_constant({elem_size}, idx->dtype_); - auto offset = expand_polynomial_offset(scale * idx); - return builder::make_cast( - v->dtype_, builder::make_cast(datatypes::index, ptr) + offset); - } - return v; - } -}; - -func_c indexing_transform_t::operator()(func_c v) { - indexing_transform_impl_t indexing_transform; - auto vv = indexing_transform.dispatch(std::move(v)); - return vv; -} - -} // namespace xbyak -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/jit/xbyak/ir/transform/indexing_transform.hpp b/legacy/core/src/compiler/jit/xbyak/ir/transform/indexing_transform.hpp deleted file mode 100644 index 31fc54f05..000000000 --- a/legacy/core/src/compiler/jit/xbyak/ir/transform/indexing_transform.hpp +++ /dev/null @@ -1,42 +0,0 @@ -/******************************************************************************* - * Copyright 2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_JIT_XBYAK_IR_TRANSFORM_INDEXING_TRANSFORM_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_JIT_XBYAK_IR_TRANSFORM_INDEXING_TRANSFORM_HPP - -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace xbyak { - -class indexing_transform_t : public function_pass_t { -public: - indexing_transform_t() = default; - func_c operator()(func_c v) override; - -private: -}; - -} // namespace xbyak -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/jit/xbyak/ir/transform/intrinsics_combine.cpp b/legacy/core/src/compiler/jit/xbyak/ir/transform/intrinsics_combine.cpp deleted file mode 100644 index 25329df16..000000000 --- a/legacy/core/src/compiler/jit/xbyak/ir/transform/intrinsics_combine.cpp +++ /dev/null @@ -1,192 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "intrinsics_combine.hpp" - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace xbyak { - -using namespace passlet; - -struct use_count_t { - size_t val_ = 0; -}; - -class use_count_analysis_viewer_t : public ssa_viewer_t { -public: - using ssa_viewer_t::dispatch; - using ssa_viewer_t::view; - - use_count_analysis_t uc_ana_; - - use_count_analysis_viewer_t() - : uc_ana_{ - temp_data_addresser()} {} - - func_c dispatch(func_c f) override { - uc_ana_.view(f, pass_phase::PRE_VISIT); - ssa_viewer_t::dispatch(f->body_); - uc_ana_.view(f, pass_phase::POST_VISIT); - return f; - } - - expr_c dispatch(expr_c v) override { - uc_ana_.view(v, pass_phase::PRE_VISIT); - uc_ana_.view(v, pass_phase::POST_VISIT); - return v; - } - - void view(define_c v) override { - uc_ana_.view(v, pass_phase::PRE_VISIT); - if (v->init_.defined()) { - dispatch(v->init_); - } - uc_ana_.view(v, pass_phase::POST_VISIT); - } - - void view(for_loop_c v) override { - uc_ana_.view(v, pass_phase::PRE_VISIT); - dispatch(v->iter_begin_); - dispatch(v->iter_end_); - dispatch(v->step_); - dispatch(v->body_); - uc_ana_.view(v, pass_phase::POST_VISIT); - } -}; - -class intrinsics_combine_impl_t : public ssa_visitor_t { -public: - using ssa_visitor_t::dispatch; - using ssa_visitor_t::visit; - - std::unordered_map *> define_scope_map_; - - expr_c combine_to_fmadd(const expr &l, const expr &r, bool is_sub = false) { - if (l->dtype_.is_etype(sc_data_etype::F32) || - l->dtype_.is_etype(sc_data_etype::F16)) { - stmt owner{l->ssa_data_->owner_.lock()}; - return owner.cast() - .filter([this, l](const define &v) { - const auto iter = define_scope_map_.find(v); - return l->get_temp_data().get().val_ == 1 && - iter != define_scope_map_.end() && - iter->second == get_current_scope(); - }) - .map([](const define &v) { return v->init_; }) - .map([](const expr &v) { return v.as(); }) - .map([r, is_sub](const mul &v) { - if (is_sub) { - return builder::make_fnmadd(v->l_, v->r_, r); - } - return builder::make_fmadd(v->l_, v->r_, r); - }) - .get_or_else(expr()); - } - return expr_c(); - } - - stmt_c visit(define_c v) override { - define_scope_map_[v] = get_current_scope(); - return ssa_visitor_t::visit(std::move(v)); - } - - expr_c visit(add_c v) override { - // combine (a * b) + c to fmadd(a, b, c) - // if (a * b) only used in fmadd and defined in same scope - auto ret = combine_to_fmadd(v->l_, v->r_); - if (ret.defined()) { - return ret; - } - ret = combine_to_fmadd(v->r_, v->l_); - if (ret.defined()) { - return ret; - } - return v; - } - - expr_c visit(sub_c v) override { - // combine c - (a * b) to fnmadd(a, b, c) - // if (a * b) only used in fnmadd and defined in same scope - // Sub node only appears like: c - (a * b) - auto ret = combine_to_fmadd(v->r_, v->l_, true); - return ret.defined() ? ret : v; - } - - expr_c visit(intrin_call_c v) override { - // combine broadcast(A[i]) to broadcast_idx(A, i) - // if A[i] only used in broadcast - auto combine_to_broadcast = [](const expr &e, int dst_lanes) { - stmt owner{e->ssa_data_->owner_.lock()}; - return owner.cast() - .filter([e](const define &v) { - return e->get_temp_data().get().val_ == 1; - }) - .map([](const define &v) { return v->init_; }) - .map([](const expr &v) { return v.as(); }) - .map([dst_lanes](const indexing &idxn) -> expr_c { - if (idxn->mask_.defined()) { - return expr(); - } - auto src_lanes = builder::make_constant((int)idxn->dtype_.lanes_); - src_lanes->ssa_data_ = utils::make_unique(); - assert(idxn->idx_.size() == 1); - return builder::make_x86_intrin( - x86_intrin_type::avx_broadcast_idx, - {idxn->ptr_, idxn->idx_.back(), src_lanes}, - {{"lanes", dst_lanes}}); - }); - }; - switch (v->type_) { - case intrin_type::broadcast: { - assert(v->args_.size() == 1); - return combine_to_broadcast(v->args_[0], v->dtype_.lanes_).get_or_else(v); - } break; - default: - break; - } - return v; - } -}; - -func_c intrinsics_combine_t::operator()(func_c v) { - use_count_analysis_viewer_t use_count_analysis; - use_count_analysis.dispatch(v); - intrinsics_combine_impl_t intrinsics_combine; - return intrinsics_combine.top_level_dispatch(std::move(v)); -} - -} // namespace xbyak -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/jit/xbyak/ir/transform/intrinsics_combine.hpp b/legacy/core/src/compiler/jit/xbyak/ir/transform/intrinsics_combine.hpp deleted file mode 100644 index cde332140..000000000 --- a/legacy/core/src/compiler/jit/xbyak/ir/transform/intrinsics_combine.hpp +++ /dev/null @@ -1,42 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_JIT_XBYAK_IR_TRANSFORM_INTRINSICS_COMBINE_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_JIT_XBYAK_IR_TRANSFORM_INTRINSICS_COMBINE_HPP - -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace xbyak { - -class intrinsics_combine_t : public function_pass_t { -public: - intrinsics_combine_t() = default; - func_c operator()(func_c v) override; - -private: -}; - -} // namespace xbyak -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/jit/xbyak/ir/transform/live_range_split.cpp b/legacy/core/src/compiler/jit/xbyak/ir/transform/live_range_split.cpp deleted file mode 100644 index 5dd275837..000000000 --- a/legacy/core/src/compiler/jit/xbyak/ir/transform/live_range_split.cpp +++ /dev/null @@ -1,255 +0,0 @@ -/******************************************************************************* - * Copyright 2023-2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include "live_range_split.hpp" - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace xbyak { - -struct split_analysis_data_t { - // - bool cross_call_ = false; - // - std::unordered_set used_vars_; - std::unordered_set modified_vars_; - - static split_analysis_data_t &get(const expr_c &v) { - return v->temp_data().get(); - } - - static bool is_set(const expr_c &v) { - return v->get_temp_data().isa(); - } -}; - -class split_analysis_viewer_t : public ir_viewer_t { -public: - using ir_viewer_t::dispatch; - using ir_viewer_t::view; - - std::vector cur_loop_iter_; - std::vector> cur_loop_scopes_; - std::unordered_set initialized_var_; - std::unordered_map defined_var_map_; - - bool var_cross_call(const expr_c &v) { - auto iter = defined_var_map_.find(v); - return (iter != defined_var_map_.end()) && iter->second; - } - - void filter_curr_scope(std::unordered_set &vars) { - for (auto it = vars.begin(); it != vars.end();) { - bool a = defined_var_map_.find(*it) == defined_var_map_.end(); - bool b = initialized_var_.find(*it) == initialized_var_.end(); - if (a || b) { - it = vars.erase(it); - } else { - ++it; - } - } - } - - void filter_inner_scope(std::unordered_set &inner, - const std::unordered_set &curr) { - for (auto it = inner.begin(); it != inner.end();) { - if (curr.find(*it) != curr.end()) { - it = inner.erase(it); - } else { - ++it; - } - } - } - - void merge_outer_scope(std::unordered_set &outer, - const std::unordered_set &curr) { - outer.insert(curr.begin(), curr.end()); - } - - void view(call_c v) override { - ir_viewer_t::view(v); - for (auto &kv : defined_var_map_) { - kv.second = true; - } - if (!cur_loop_iter_.empty()) { - split_analysis_data_t::get(cur_loop_iter_.back()).cross_call_ = true; - } - } - - void view(var_c v) override { - ir_viewer_t::view(v); - if (is_x86_simd(v->dtype_)) { - if (!cur_loop_iter_.empty() && var_cross_call(v)) { - auto &data = split_analysis_data_t::get(cur_loop_iter_.back()); - data.used_vars_.insert(v); - } - } - } - - void view(define_c v) override { - ir_viewer_t::view(v); - if (is_x86_simd(v->var_->dtype_)) { - defined_var_map_[v->var_] = false; - if (cur_loop_iter_.empty() && v->init_.defined()) { - initialized_var_.insert(v->var_); - } - } - } - - void view(assign_c v) override { - ir_viewer_t::view(v); - if (is_x86_simd(v->var_->dtype_)) { - if (!cur_loop_iter_.empty() && var_cross_call(v->var_)) { - auto &data = split_analysis_data_t::get(cur_loop_iter_.back()); - data.modified_vars_.insert(v->var_); - } - if (cur_loop_iter_.empty()) { - initialized_var_.insert(v->var_); - } - } - } - - void view(for_loop_c v) override { - // recored this as inner loop of outer - if (!cur_loop_scopes_.empty()) { - cur_loop_scopes_.back().emplace_back(v->var_); - } - cur_loop_scopes_.emplace_back(std::vector()); - cur_loop_iter_.emplace_back(v->var_); - // build analysis data - v->var_->temp_data() = split_analysis_data_t(); - auto &data = split_analysis_data_t::get(v->var_); - - // dispatch into loop - ir_viewer_t::view(v); - - // if loop contains call, no spilt for inner loop vars - if (split_analysis_data_t::get(v->var_).cross_call_) { - data.used_vars_.clear(); - data.modified_vars_.clear(); - for (const auto &loop_var : cur_loop_iter_) { - split_analysis_data_t::get(loop_var).cross_call_ = true; - } - } - cur_loop_iter_.pop_back(); - - // filter vars to spilt when defined outside loop - // propagate spilt vars to most outer loop when possible - filter_curr_scope(data.used_vars_); - filter_curr_scope(data.modified_vars_); - if (!cur_loop_iter_.empty()) { - auto &outer = split_analysis_data_t::get(cur_loop_iter_.back()); - merge_outer_scope(outer.used_vars_, data.used_vars_); - merge_outer_scope(outer.modified_vars_, data.modified_vars_); - } - for (auto &inner_loop : cur_loop_scopes_.back()) { - auto &inner = split_analysis_data_t::get(inner_loop); - filter_inner_scope(inner.used_vars_, data.used_vars_); - filter_inner_scope(inner.modified_vars_, data.modified_vars_); - } - cur_loop_scopes_.pop_back(); - } -}; - -// var a -// a = ... -// call(...) -// for(...) { -// a = ... -// } -// -// spilt live range of a: -// -// var a -// a = ... -// call(...) -// var a_1 = a -// for(...) { -// a_1 = ... -// } - -class live_range_splitter_impl_t : public ir_visitor_t { -public: - using ir_visitor_t::dispatch; - using ir_visitor_t::visit; - - size_t var_counter_ = 0; - std::unordered_map var_replace_map_; - - expr_c visit(var_c v) override { - auto iter = var_replace_map_.find(v); - return (iter == var_replace_map_.end()) ? v : iter->second; - } - - stmt_c visit(for_loop_c v) override { - auto &data = split_analysis_data_t::get(v->var_); - if (data.used_vars_.empty()) { - return ir_visitor_t::visit(std::move(v)); - } else { - std::vector seq; - // insert define before loop - for (const auto &x : data.used_vars_) { - auto new_v = x->remake().static_as(); - new_v->name_ += std::string("_spilt_") + std::to_string(var_counter_++); - var_replace_map_[x] = new_v; - seq.emplace_back( - builder::make_var_tensor_def_unattached(new_v, linkage::local, x)); - } - // dispatch loop - seq.emplace_back(ir_visitor_t::visit(std::move(v)).remove_const()); - // insert assign after loop - for (const auto &x : data.modified_vars_) { - seq.emplace_back( - builder::make_assign_unattached(x, var_replace_map_[x])); - } - // remove replacment vars - for (const auto &x : data.used_vars_) { - var_replace_map_.erase(x); - } - - return make_stmt(std::move(seq)); - } - } -}; - -func_c live_range_splitter_t::operator()(func_c v) { - split_analysis_viewer_t analyzer; - analyzer.dispatch(v); - - live_range_splitter_impl_t live_range_splitter; - return live_range_splitter.dispatch(std::move(v)); -} - -} // namespace xbyak -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/jit/xbyak/ir/transform/live_range_split.hpp b/legacy/core/src/compiler/jit/xbyak/ir/transform/live_range_split.hpp deleted file mode 100644 index 94519ff00..000000000 --- a/legacy/core/src/compiler/jit/xbyak/ir/transform/live_range_split.hpp +++ /dev/null @@ -1,45 +0,0 @@ -/******************************************************************************* - * Copyright 2023-2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_JIT_XBYAK_IR_TRANSFORM_LIVE_RANGE_SPLIT_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_JIT_XBYAK_IR_TRANSFORM_LIVE_RANGE_SPLIT_HPP - -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace xbyak { - -/* * - * Spilt live range of vars to smaller live ranges according to calls and loops - * */ -class live_range_splitter_t : public function_pass_t { -public: - live_range_splitter_t() = default; - func_c operator()(func_c v) override; - -private: -}; - -} // namespace xbyak -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/jit/xbyak/ir/transform/low_level_legalizer.cpp b/legacy/core/src/compiler/jit/xbyak/ir/transform/low_level_legalizer.cpp deleted file mode 100644 index 0c59bd6e0..000000000 --- a/legacy/core/src/compiler/jit/xbyak/ir/transform/low_level_legalizer.cpp +++ /dev/null @@ -1,187 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include -#include -#include -#include - -#include -#include - -#include "low_level_legalizer.hpp" - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace xbyak { - -class low_level_legalizer_impl_t : public ir_visitor_t { -public: - using ir_visitor_t::dispatch; - using ir_visitor_t::visit; - - low_level_legalizer_impl_t(const runtime::target_machine_t &target_machine) - : max_f32_lanes_(target_machine.get_device_flags().get_max_vector_lanes( - sc_data_etype::F32)) {} - - expr_c visit(tensor_c v) override { - // avoid dispatch into for loop index dependent tensor - return v; - } - - expr_c visit(cast_c v) override { - auto ret = ir_visitor_t::visit(std::move(v)); - assert(ret.isa()); - auto vv = ret.static_as(); - return transform_f32_cast(vv->dtype_, vv->in_); - } - - stmt_c visit(assign_c v) override { - auto vv = ir_visitor_t::visit(std::move(v)).static_as(); - auto &var = vv->var_; - auto &value = vv->value_; - if (value.isa()) { - // Must cast tensor to dst dtype before assign - return make_stmt(var.remove_const(), - builder::make_cast(var->dtype_, value)); - } - return vv; - } - - expr_c visit(intrin_call_c v) override { - auto vv = ir_visitor_t::visit(std::move(v)).static_as(); - auto dst_dtype = vv->dtype_; - auto src_dtype = vv->args_[0]->dtype_; - // TODO(longsheng): reduce no need to cast to f32 - auto can_cast_to_f32 = !dst_dtype.is_etype(sc_data_etype::F32) && - (src_dtype.lanes_ <= max_f32_lanes_); - switch (vv->type_) { - case intrin_type::abs: { - // float and bf16 - if (dst_dtype.is_etype(sc_data_etype::F32)) { - return transform_fabs(vv->args_[0], UINT64_C(0x7FFFFFFF)); - } else if (dst_dtype.is_etype(sc_data_etype::BF16) || - dst_dtype.is_etype(sc_data_etype::F16)) { - return transform_fabs(vv->args_[0], UINT64_C(0x7FFF)); - } - } break; - case intrin_type::rsqrt: { - if (dst_dtype == datatypes::f32) { - return builder::make_constant(1.f) / builder::make_sqrt(vv->args_[0]); - } - } break; - case intrin_type::reduce_add: { - if (can_cast_to_f32) { - return transform_reduce(vv->args_[0], dst_dtype, - &builder::make_reduce_add); - } - } break; - case intrin_type::reduce_mul: { - if (can_cast_to_f32) { - return transform_reduce(vv->args_[0], dst_dtype, - &builder::make_reduce_mul); - } - } break; - case intrin_type::reduce_min: { - if (can_cast_to_f32) { - return transform_reduce(vv->args_[0], dst_dtype, - &builder::make_reduce_min); - } - } break; - case intrin_type::reduce_max: { - if (can_cast_to_f32) { - return transform_reduce(vv->args_[0], dst_dtype, - &builder::make_reduce_max); - } - } break; - case intrin_type::gather: { - if (vv->dtype_.lanes_ == 1) { - return builder::make_indexing(vv->args_[0], {vv->args_[1]}); - } - } break; - default: - break; - } - return vv; - } - - using make_unary_f = expr (*)(const expr_c &); - using make_binary_f = expr (*)(const expr_c &, const expr_c &); - - expr_c transform_fabs(const expr &src, const union_val &val) { - return builder::make_int_and(src, - builder::make_constant({val}, src->dtype_)); - } - - expr_c transform_reduce(const expr &src, sc_data_type_t dst_dtype, - make_unary_f make_reduce) { - auto type_to_f32 = sc_data_type_t::f32(src->dtype_.lanes_); - return transform_f32_cast( - dst_dtype, // - make_reduce(transform_f32_cast(type_to_f32, src))); - } - - expr_c transform_f32_cast(sc_data_type_t dst_dtype, const expr &src) { - const auto src_dtype = src->dtype_; - const auto is_f32_int8_cast = [](const sc_data_type_t &type_dst, - const sc_data_type_t &type_src) { - return (type_dst.type_code_ == sc_data_etype::U8 || - type_dst.type_code_ == sc_data_etype::S8) && - type_src.type_code_ == sc_data_etype::F32; - }; - const auto is_f16_scalar_cast = [](const sc_data_type_t &type_dst, - const sc_data_type_t &type_src) { - return (type_dst.type_code_ == sc_data_etype::F16) && - (utils::is_one_of(type_src.type_code_, sc_data_etype::U16, - sc_data_etype::S8, sc_data_etype::U8)) && - type_dst.lanes_ == 1; - }; - const auto convert_s32_transform = [](const sc_data_type_t &dst_dtype, - const sc_data_type_t &src_dtype, - const expr &src) { - return builder::make_cast( - dst_dtype, - builder::make_cast(sc_data_type_t::s32(src_dtype.lanes_), src)); - }; - if (is_f32_int8_cast(dst_dtype, src_dtype) || - is_f32_int8_cast(src_dtype, dst_dtype)) { - // int8 to f32 must cast to s32 first - return convert_s32_transform(dst_dtype, src_dtype, src); - } else if (is_f16_scalar_cast(dst_dtype, src_dtype) || - is_f16_scalar_cast(src_dtype, dst_dtype)) { - // other dtype to f16 must cast to s32 first - return convert_s32_transform(dst_dtype, src_dtype, src); - } else { - return builder::make_cast(dst_dtype, src); - } - } - -private: - const uint16_t max_f32_lanes_; -}; - -func_c low_level_legalizer_t::operator()(func_c v) { - low_level_legalizer_impl_t low_level_legalizer(target_machine_); - return low_level_legalizer.dispatch(std::move(v)); -} - -} // namespace xbyak -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/jit/xbyak/ir/transform/low_level_legalizer.hpp b/legacy/core/src/compiler/jit/xbyak/ir/transform/low_level_legalizer.hpp deleted file mode 100644 index 5d1baab6f..000000000 --- a/legacy/core/src/compiler/jit/xbyak/ir/transform/low_level_legalizer.hpp +++ /dev/null @@ -1,49 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_JIT_XBYAK_IR_TRANSFORM_LOW_LEVEL_LEGALIZER_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_JIT_XBYAK_IR_TRANSFORM_LOW_LEVEL_LEGALIZER_HPP - -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace xbyak { - -/* * - * Low level legalizer for xbyak backend, make sure data type and data order is - * compatible with low level requirement. - * */ -class low_level_legalizer_t : public function_pass_t { -public: - low_level_legalizer_t(const runtime::target_machine_t &target_machine) - : target_machine_(target_machine) {} - func_c operator()(func_c v) override; - -private: - const runtime::target_machine_t &target_machine_; -}; - -} // namespace xbyak -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/jit/xbyak/ir/transform/module_var_resolver.cpp b/legacy/core/src/compiler/jit/xbyak/ir/transform/module_var_resolver.cpp deleted file mode 100644 index a7bf3c425..000000000 --- a/legacy/core/src/compiler/jit/xbyak/ir/transform/module_var_resolver.cpp +++ /dev/null @@ -1,107 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include -#include - -#include -#include -#include -#include -#include - -#include "module_var_resolver.hpp" - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace xbyak { - -class module_var_resolver_t_impl_t : public ir_visitor_t { -public: - using ir_visitor_t::dispatch; - using ir_visitor_t::visit; - - std::unordered_map module_vars_; - expr_c module_data_; - - func_c dispatch(func_c v) override { - if (v->params_.size() > 1 && v->params_[1].isa()) { - auto module_tensor = v->params_[1].static_as(); - if (module_tensor->name_ == "__module_data") { - module_data_ = module_tensor; - } - } - return ir_visitor_t::dispatch(std::move(v)); - } - - expr_c dispatch(expr_c v) override { - if (module_vars_.find(v) != module_vars_.end()) { - auto new_expr = module_vars_[v]; - return copy_attr(*v, std::move(new_expr)); - } else { - return ir_visitor_t::dispatch(std::move(v)); - } - } - - stmt_c visit(define_c v) override { - if (v->var_.isa()) { - auto thevar = v->var_.static_as(); - if (module_data_.defined() && thevar->attr_ && - thevar->attr_->has_key(attr_keys::module_global_offset)) { - // TODO(XXX): module_var to indexing, maybe define just before - // use? - auto &offset = thevar->attr_->get_any(attr_keys::module_global_offset); - expr module_idx_ptr; - if (auto absptr = offset.get_or_null()) { - module_idx_ptr = - make_expr(reinterpret_cast(*absptr), - datatypes::s8.get_pointerof()); - } else { - // Get var_ptr = &module_data[offset] - auto index = builder::make_constant(offset.get()); - module_idx_ptr = builder::tensor_ptr(module_data_, {index}); - } - // var_ptr->elem_dtype_ is not module_data_->elem_dtype_ - // var_ptr->elem_dtype_ should be var->dtype_ - auto var_ptr = - builder::make_tensor(thevar->name_ + "_ptr", {1}, thevar->dtype_); - // Replace future module_var use with var_ptr[0] - auto module_var = builder::make_indexing( - var_ptr, builder::make_constant(UINT64_C(0))); - module_vars_[thevar] = module_var; - // Will replace define with module_var's tensor define - auto var_ptr_define = - make_stmt(var_ptr, linkage::local, module_idx_ptr); - return std::move(var_ptr_define); - } - } - return ir_visitor_t::visit(std::move(v)); - } -}; - -func_c module_var_resolver_t::operator()(func_c v) { - module_var_resolver_t_impl_t module_var_resolver; - - return module_var_resolver.dispatch(std::move(v)); -} - -} // namespace xbyak -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/jit/xbyak/ir/transform/module_var_resolver.hpp b/legacy/core/src/compiler/jit/xbyak/ir/transform/module_var_resolver.hpp deleted file mode 100644 index d568d8082..000000000 --- a/legacy/core/src/compiler/jit/xbyak/ir/transform/module_var_resolver.hpp +++ /dev/null @@ -1,46 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_JIT_XBYAK_IR_TRANSFORM_MODULE_VAR_RESOLVER_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_JIT_XBYAK_IR_TRANSFORM_MODULE_VAR_RESOLVER_HPP - -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace xbyak { - -/* * - * Substitute the golbal module var indicated by `module_global_offset` as - * indexing of 1-element-sized local tensor of `module_data` address offset. - * */ -class module_var_resolver_t : public function_pass_t { -public: - module_var_resolver_t() = default; - func_c operator()(func_c v) override; - -private: -}; - -} // namespace xbyak -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/jit/xbyak/ir/transform/register_allocation.cpp b/legacy/core/src/compiler/jit/xbyak/ir/transform/register_allocation.cpp deleted file mode 100644 index 3dfc9cbcc..000000000 --- a/legacy/core/src/compiler/jit/xbyak/ir/transform/register_allocation.cpp +++ /dev/null @@ -1,886 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "call_transform.hpp" -#include "register_allocation.hpp" - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace xbyak { -using namespace xbyak::x86_64; - -SC_MODULE(xbyakjit.register_allocation) - -// atomic accumulator for temp var name -std::atomic temp_index_(0); - -void prepare_virtual_reg(reg_allocator_t *allocator, virtual_reg_t *virt_reg, - const Xbyak::Reg &phy_reg, sc_data_type_t dtype, - bool is_avx512) { - if (phy_reg.isNone()) { - virt_reg->set_type( - get_virt_reg_type(dtype, is_avx512, virt_reg->force_fp_vex_)); - virt_reg->set_unassigned(); - virt_reg->add_weight(virt_reg->extra_weight()); - } else { - auto &slots_map = allocator->slots_map(); - virt_reg->set_type( - get_virt_reg_type(dtype, is_avx512, virt_reg->force_fp_vex_)); - virt_reg->set_designated(slots_map.get_reg_index(phy_reg)); - } - if (!virt_reg->live_range_.empty()) { - allocator->enqueue(virt_reg); - } -} - -/* * - * enclose_set_t: check if any stmt index in a set enclosed by a live range. - * */ -class enclose_set_t { -public: - enclose_set_t() = default; - void insert(stmt_index_t index) { index_map_.insert(index); } - void erase(stmt_index_t index) { index_map_.erase(index); } - bool enclosed_by(const live_range_t &range) { - auto iter = index_map_.lower_bound(range.start_); - if (iter != index_map_.begin()) { - iter--; - } - while (iter != index_map_.end()) { - if (range.end_ <= *iter) { - break; - } - if (range.enclose(*iter)) { - return true; - } - iter++; - } - return false; - } - -private: - std::set index_map_; -}; - -/* * - * Pre-allocation pass: analyze call index for reg allocation, assign hints. - * */ -class call_analysis_t : public xbyak_visitor_t { -public: - using xbyak_visitor_t::dispatch; - using xbyak_visitor_t::visit; - - call_analysis_t(reg_allocator_t *allocator, enclose_set_t &call_index_set) - : allocator_(allocator), call_index_set_(call_index_set) {} - - func_c operator()(func_c v) { return dispatch(std::move(v)); } - - func_c dispatch(func_c v) override { - func_t func = std::const_pointer_cast(v); - func_iface_ = cached_func_abi_interface(func); - set_func_args_hint(v->params_); - return xbyak_visitor_t::dispatch(std::move(v)); - } - - stmt_c visit(returns_c v) override { - if (v->value_.defined()) { - set_func_return_hint(v->value_, func_iface_); - } - return xbyak_visitor_t::visit(std::move(v)); - } - - stmt_c visit(assign_c v) override { - if (v->value_.isa()) { - auto call_v = v->value_.static_as(); - auto func_abi = cached_call_abi_interface(call_v); - set_func_return_hint(v->var_, func_abi); - call_index_set_.insert(GET_STMT_INDEX(v)); - } - return xbyak_visitor_t::visit(std::move(v)); - } - - stmt_c visit(define_c v) override { - if (v->init_.defined() && v->init_.isa()) { - call_index_set_.insert(GET_STMT_INDEX(v)); - } - return xbyak_visitor_t::visit(std::move(v)); - } - - stmt_c visit(evaluate_c v) override { - if (v->value_.isa()) { - call_index_set_.insert(GET_STMT_INDEX(v)); - } - return xbyak_visitor_t::visit(std::move(v)); - } - - expr_c visit(call_c v) override { - auto &args = v->args_; - auto &callee_iface = *cached_call_abi_interface(v); - auto &slots_map = allocator_->slots_map(); - for (size_t i = 0; i < args.size(); ++i) { - auto ¶m = args[i]; - auto &virt_para = GET_VIRTUAL_REG(param); - auto &phy_para = GET_PHYSICAL_REG(param); - auto &initial_loc = callee_iface.param_locs_[i]; - if (initial_loc.get_type() == abi_value_location::tag_type::REGISTER) { - phy_para = initial_loc.get_register(); - } else { - virt_para.set_spilled(); - } - } - return xbyak_visitor_t::visit(std::move(v)); - } - -private: - reg_allocator_t *allocator_; - enclose_set_t &call_index_set_; - abi_function_interface::ptr func_iface_; - - void set_func_args_hint(const std::vector ¶ms) { - auto &slots_map = allocator_->slots_map(); - for (size_t i = 0; i < params.size(); ++i) { - auto ¶m = params[i]; - auto &virt_reg = GET_VIRTUAL_REG(param); - auto &initial_loc = func_iface_->param_locs_[i]; - virt_reg.set_type(get_virt_reg_type(param->dtype_, false)); - if (initial_loc.get_type() == abi_value_location::tag_type::REGISTER) { - Xbyak::Reg src_reg = initial_loc.get_register(); - COMPILE_ASSERT(src_reg.isREG(64) || src_reg.isXMM(), - "Unhandled register kind: " << src_reg.toString()); - virt_reg.set_hint(virt_reg_hint::strong, - slots_map.get_reg_index(src_reg)); - if (i < 2) { - // arg is stream or modudle_data - virt_reg.spill_weight_ = spill_weight_const::initial; - } - } - } - } - - void set_func_return_hint(const expr &v, - const abi_function_interface::ptr &func_abi) { - auto &slots_map = allocator_->slots_map(); - auto &virt_reg = GET_VIRTUAL_REG(v); - auto ret_reg = func_abi->return_val_loc_.get_register(); - virt_reg.set_hint(virt_reg_hint::weak, slots_map.get_reg_index(ret_reg)); - } -}; - -/* * - * Pre-allocation pass: - * prepare physical regs and enqueue all virtual regs to be assigned. - * */ -class pre_allocation_t : public xbyak_visitor_t { -public: - using xbyak_visitor_t::dispatch; - using xbyak_visitor_t::visit; - - pre_allocation_t(reg_allocator_t *allocator, - std::map &virt_reg_map, - const runtime::cpu_flags_t &cpu_flags) - : allocator_(allocator), virt_reg_map_(virt_reg_map), - cpu_flags_(cpu_flags) {} - - func_c operator()(func_c v) { - call_analysis_t call_analysis(allocator_, call_index_set_); - return dispatch(call_analysis(std::move(v))); - } - - expr_c dispatch(expr_c v) override { - // If need reg allocation - if (v.isa() || v.isa()) { - auto &live_range = GET_LIVE_RANGE(v); - auto &virt_reg = GET_VIRTUAL_REG(v); - auto &phy_reg = GET_PHYSICAL_REG(v); - virt_reg_map_[&virt_reg] = v; - if (virt_reg.stat_ == virt_reg_stat::disabled) { - if (call_index_set_.enclosed_by(live_range)) { - virt_reg.set_preserved(); - } - prepare_virtual_reg(allocator_, &virt_reg, phy_reg, v->dtype_, - cpu_flags_.fAVX512F); - } - return v; - } - return xbyak_visitor_t::dispatch(std::move(v)); - } - - stmt_c visit(stmts_c v) override { - if (TRANSFORMED_CALL(v)) { - if (v->size() == 0) { - GET_STMT_DATA(v).optimized_out_ = true; - return v; - } - } - auto ret = xbyak_visitor_t::visit(std::move(v)); - return ret; - } - - stmt_c visit(define_c v) override { - if (GET_LIVE_RANGE(v->var_).empty()) { - GET_STMT_DATA(v).optimized_out_ = true; - return v; - } - if (v->var_.isa() && !v->init_.defined()) { - auto &virt_reg = GET_VIRTUAL_REG(v->var_); - virt_reg.set_buffered(); - allocator_->spilled_virt_regs().insert(&virt_reg); - } - return xbyak_visitor_t::visit(std::move(v)); - } - - stmt_c visit(assign_c v) override { - auto &live_range = GET_LIVE_RANGE(v->var_); - if (v->var_.isa() && live_range.empty()) { - GET_STMT_DATA(v).optimized_out_ = true; - return v; - } - return xbyak_visitor_t::visit(std::move(v)); - } - - expr_c visit(indexing_c v) override { - GET_VIRTUAL_REG(v).set_spilled(); - return xbyak_visitor_t::visit(std::move(v)); - } - - expr_c visit(func_addr_c v) override { - GET_VIRTUAL_REG(v).set_spilled(); - return xbyak_visitor_t::visit(std::move(v)); - } - - expr_c visit(constant_c v) override { - auto &virt_reg = GET_VIRTUAL_REG(v); - virt_reg.set_type(get_virt_reg_type(v->dtype_, false)); - if (is_x86_simd(v->dtype_) || FORCE_SIMD_ENCODE(v)) { - virt_reg.set_spilled(); - } - return v; - } - -private: - reg_allocator_t *allocator_; - std::map &virt_reg_map_; - const runtime::cpu_flags_t &cpu_flags_; - enclose_set_t call_index_set_; -}; - -/* * - * Check address mode conflicts for each operation, resolve conflicts by - * creating small live intervels and inserting load/store when nedded. - * */ -class spill_resolver_t : public xbyak_visitor_t { -public: - using xbyak_visitor_t::dispatch; - using xbyak_visitor_t::visit; - - spill_resolver_t(const live_range_t &spill_range, - std::vector &virtual_regs, - const runtime::cpu_flags_t &cpu_flags) - : spill_range_(spill_range), virtual_regs_(virtual_regs), - cpu_flags_(cpu_flags) {} - - func_c operator()(func_c v) { - return xbyak_visitor_t::dispatch(std::move(v)); - } - - stmt_c dispatch(stmt_c v) override { - auto old_stmt_data = GET_STMT_DATA(v); - cur_index_ = old_stmt_data.index_; - // if current index out of spill_range, skip resolve - if (spill_range_.defined_ && - (old_stmt_data.index_ < spill_range_.start_ || - old_stmt_data.init_index_ > spill_range_.end_)) { - return v; - } - // if current index in spill_range, try resolve - auto ret = xbyak_visitor_t::dispatch(std::move(v)); - if (!ret->temp_data().isa()) { - ret->temp_data() = old_stmt_data; - } - return ret; - } - - stmt_c visit(stmts_c v) override { - std::vector new_seq; - for (auto &s : v->seq_) { - // dispatch stmt - auto ss = dispatch(s).remove_const(); - // remove optimized_out - if (GET_STMT_DATA(ss).optimized_out_) { - continue; - } - // insert before - for (auto &&ts : insert_before_) { - new_seq.push_back(std::move(ts)); - } - insert_before_.clear(); - // insert middle - new_seq.emplace_back(ss); - // insert after - for (auto &&ts : insert_after_) { - new_seq.push_back(std::move(ts)); - } - insert_after_.clear(); - } - return copy_attr(*v, make_stmt(std::move(new_seq))); - } - - stmt_c visit(assign_c v) override { - resolve_dst_ = resolve_dst::none; - dst_is_mem_ = is_spilled(v->var_); - auto ret = - xbyak_visitor_t::visit(std::move(v)).remove_const().static_as(); - if (dst_is_mem_) { - if (resolve_dst_ == resolve_dst::store) { - return insert_store(std::move(ret), cur_index_); - } else if (resolve_dst_ == resolve_dst::load_store) { - return insert_load_store(std::move(ret), cur_index_); - } else if (is_spilled(ret->value_)) { - if (ret->var_.isa()) { - return insert_store(std::move(ret), cur_index_); - } else { - ret->value_ = insert_load(std::move(ret->value_), cur_index_); - return ret; - } - } else if (const_exceed_32bit(ret->value_)) { - ret->value_ = insert_load(std::move(ret->value_), cur_index_); - return ret; - } - } - return ret; - } - - // avoid dispatch into index dependent tensor - expr_c visit(tensor_c v) override { return v; } - - stmt_c visit(for_loop_c v) override { - // if var spilled, spilled begin/end/step must load - return resolve_spill(std::move(v)); - } - - stmt_c visit(define_c v) override { - // var = tensor_ptr, spilled var must be store from reg - return resolve_spill(std::move(v)); - } - - expr_c visit(cast_c v) override { - // var = cast(in), spilled var must be store from reg - return resolve_spill(std::move(v)); - } - - expr_c visit(tensorptr_c v) override { - // var = &ptr[idx], spilled var must be store from reg - return resolve_spill(std::move(v)); - } - - expr_c visit(indexing_c v) override { - // ptr[idx], spilled ptr and idx need to be load to reg - return resolve_spill(std::move(v)); - } - - expr_c visit(xbyak_intrin_c v) override { - // var = xbyak_intrin(args...), check intrin_format - return resolve_spill(std::move(v)); - } - -protected: - stmt_c resolve_spill(for_loop_c v) { - auto stmt_data = v->get_temp_data(); - auto vv = xbyak_visitor_t::visit(std::move(v)) - .remove_const() - .static_as(); - vv->temp_data() = stmt_data; - bool loop_var_spilled = is_spilled(vv->var_); - if (loop_var_spilled && is_spilled(vv->iter_begin_)) { - auto begin_index = GET_STMT_INIT_INDEX(vv); - vv->attr().set(attr_keys::load_loop_begin, - loop_var_load(vv->iter_begin_, begin_index)); - } - if (loop_var_spilled && is_spilled(vv->iter_end_)) { - auto cond_index = GET_STMT_INIT_INDEX(vv->body_) + 1; - vv->attr().set(attr_keys::load_loop_end, - loop_var_load(vv->iter_end_, cond_index)); - } - if (loop_var_spilled && is_spilled(vv->step_)) { - auto step_index = GET_STMT_INDEX(vv->body_); - vv->attr().set(attr_keys::load_loop_step, - loop_var_load(vv->step_, step_index)); - } - return vv; - } - - stmt_c resolve_spill(define_c v) { - auto vv = - xbyak_visitor_t::visit(std::move(v)).remove_const().static_as(); - if (is_spilled(vv->var_) && vv->init_.defined()) { - if (vv->var_.isa() && - (vv->init_.isa() || vv->init_.isa())) { - // load tensor_ptr using lea needs dst be reg - vv->init_ = insert_load(std::move(vv->init_), cur_index_); - } else if (const_exceed_32bit(vv->init_)) { - // load constant to mem location cannot exceed 32bit - vv->init_ = insert_load(std::move(vv->init_), cur_index_); - } else if (is_spilled(vv->init_)) { - // load mem to mem need intermediate reg - vv->init_ = insert_load(std::move(vv->init_), cur_index_); - } - } - return vv; - } - - expr_c resolve_spill(cast_c v) { - // if operand_mem_sum({dst, in}) > 1 - // in allow mem, dst must be reg - resolve_dst_ = resolve_dst::store; - return v; - } - - expr_c resolve_spill(tensorptr_c v) { - resolve_dst_ = resolve_dst::store; - return xbyak_visitor_t::visit(std::move(v)); - } - - expr_c resolve_spill(indexing_c x) { - auto v = xbyak_visitor_t::visit(std::move(x)) - .remove_const() - .static_as(); - auto ptr = v->ptr_; - auto idx = v->idx_.back(); - if (is_spilled(ptr)) { - v->ptr_ = insert_load(std::move(ptr), cur_index_); - } - if (is_spilled(idx)) { - v->idx_ = {insert_load(std::move(idx), cur_index_)}; - } - return v; - } - - expr_c resolve_spill(xbyak_intrin_c x) { - auto v = xbyak_visitor_t::visit(std::move(x)) - .remove_const() - .static_as(); - // resolve args load - auto resolve = [&]() { - size_t i = 0; - for (auto &arg : v->args_) { - if (is_spilled(arg)) { - break; - } - i++; - } - v->args_[i] = insert_load(std::move(v->args_[i]), cur_index_); - return v; - }; - // cond_mask for intrin must be reg - auto &mask = v->modifier_.cond_mask_; - if (mask.defined() && is_spilled(mask)) { - mask = insert_load(std::move(mask), cur_index_); - } - // resolve differnet format - switch (v->format_) { - case xbyak_intrin_format::undefined: { - // no need to resolve - resolve_dst_ = resolve_dst::none; - } break; - case xbyak_intrin_format::directed_all_reg: { - // if operand_mem_sum({src, dst}) > 0 - // src must load to reg, dst must be reg - resolve_dst_ = resolve_dst::store; - if (spilled_args_sum(v->args_) > 0) { - return resolve(); - } - } break; - case xbyak_intrin_format::directed_end_mem: { - // if operand_mem_sum({src, dst}) > 1 - // src allow 1 mem at the end of args, excluding imm - // dst must be reg - resolve_dst_ = resolve_dst::store; - assert(v->args_.size() > 0); - auto n = v->args_.size() - 1; - auto imm_last = n > 0 && is_imm(v->args_.back()); - if (cpu_flags_.fAVX512F && imm_last) { - n = n - 1; - } - if (spilled_args_sum(v->args_, n) > 0) { - return resolve(); - } - } break; - case xbyak_intrin_format::directed_dst_mem: { - // if operand_mem_sum({src, dst}) > 1 - // when dst is mem, src must all load to reg - // when dst is reg, src allow 1 mem - resolve_dst_ = resolve_dst::none; - if (dst_is_mem_ && spilled_args_sum(v->args_) > 0) { - return resolve(); - } else if (spilled_args_sum(v->args_) > 1) { - return resolve(); - } - } break; - case xbyak_intrin_format::directed_dst_reg: { - // if operand_mem_sum({src, dst}) > 1 - // src allow 1 mem, dst must be reg - resolve_dst_ = resolve_dst::store; - if (spilled_args_sum(v->args_) > 1) { - return resolve(); - } - } break; - case xbyak_intrin_format::compound_dst_mem: { - // if operand_mem_sum({src, dst}) > 1 - // when dst is mem, src must all load to reg - // when dst is reg, src allow 1 mem - resolve_dst_ = resolve_dst::none; - if (dst_is_mem_ && spilled_args_sum(v->args_) > 0) { - return resolve(); - } else if (spilled_args_sum(v->args_) > 1) { - return resolve(); - } - } break; - case xbyak_intrin_format::compound_dst_reg: { - // if operand_mem_sum({src, dst}) > 1 - // src allow 1 mem, dst must be reg - resolve_dst_ = resolve_dst::load_store; - if (spilled_args_sum(v->args_) > 1) { - return resolve(); - } - } break; - case xbyak_intrin_format::compound_end_mem: { - // if operand_mem_sum({src, dst}) > 1 - // src allow 1 mem at the end of args, excluding imm - // dst must be reg - resolve_dst_ = resolve_dst::load_store; - assert(v->args_.size() > 0); - auto n = v->args_.size() - 1; - auto imm_last = n > 0 && is_imm(v->args_.back()); - if (cpu_flags_.fAVX512F && imm_last) { - n = n - 1; - } - if (spilled_args_sum(v->args_, n) > 0) { - return resolve(); - } - } break; - } - // Special cases - switch (static_cast(v->type_)) { - case xbyak_intrin_type::call_arg: { - auto &arg = v->args_.back(); - auto &virt_reg = GET_VIRTUAL_REG(arg); - if (dst_is_mem_ && virt_reg.buffered()) { - // local stack tensor as func arg passed to stack - auto node = builder::make_cast(sc_data_type_t::generic(), arg); - v->args_ = {insert_load(std::move(node), cur_index_)}; - return v; - } else if (dst_is_mem_ && const_exceed_32bit(arg)) { - // int exceeds 32bit as func arg passed to stack - v->args_ = {insert_load(std::move(arg), cur_index_)}; - return v; - } - } break; - case xbyak_intrin_type::mask_mov: { - if (dst_is_mem_ && v->modifier_.zero_mask_) { - // store to a var in memory with zero-masking - resolve_dst_ = resolve_dst::store; - } else if (dst_is_mem_ && spilled_args_sum(v->args_) > 0) { - // dst might be indexing, do not change ptr/idx liveness - resolve_dst_ = resolve_dst::none; - return resolve(); - } - } break; - default: - break; - } - return v; - } - -private: - // index for load/store insert - stmt_index_t get_index_load(stmt_index_t index) { return index - 1; } - stmt_index_t get_index_store(stmt_index_t index) { return index + 1; } - - stmt loop_var_load(expr &old_expr, stmt_index_t index) { - assert(old_expr.isa()); - auto index_load = get_index_load(index); - auto old_var = old_expr.static_as(); - auto new_var = - new_temp_var(old_var, "load_" + old_var->name_, index_load, index); - // replace old loop var with new var - old_expr = new_var; - // return load assign stmt - return new_temp_assign(new_var, old_var, index_load); - } - - expr insert_load(expr old_expr, stmt_index_t index) { - auto index_load = get_index_load(index); - auto get_new_expr = [&]() { - if (old_expr.isa()) { - auto old_ptr = old_expr.static_as(); - auto new_ptr = - new_temp_tensor(old_ptr, "load_tensor_", index_load, index); - return new_ptr; - } else { - auto old_var = old_expr.static_as(); - auto new_var = new_temp_var(old_var, "load_var_", index_load, index); - return new_var; - } - }; - auto new_expr = get_new_expr(); - auto new_load = new_temp_assign(new_expr, std::move(old_expr), index_load); - insert_before_.emplace_back(new_load); - return new_expr; - } - - stmt insert_store(assign vv, stmt_index_t index) { - auto index_store = get_index_store(index); - auto &var_range = GET_LIVE_RANGE(vv->var_); - if (var_range.end_ < index_store) { - SC_MODULE_WARN << "POTENTIAL ERROR, dead store to var: " << vv; - var_range.update(index_store); - } - if (vv->var_.isa()) { - auto old_ptr = vv->var_.static_as(); - auto new_ptr = - new_temp_tensor(old_ptr, "store_tensor_", index, index_store); - auto new_store = new_temp_assign(old_ptr, new_ptr, index_store); - insert_after_.emplace_back(new_store); - vv->var_ = new_ptr; - } else { - auto old_var = vv->var_.static_as(); - auto new_var = new_temp_var(old_var, "store_var_", index, index_store); - auto new_store = new_temp_assign(old_var, new_var, index_store); - insert_after_.emplace_back(new_store); - vv->var_ = new_var; - } - return vv; - } - - stmt insert_load_store(assign vv, stmt_index_t index) { - auto index_load = get_index_load(index); - auto index_store = get_index_store(index); - auto &var_range = GET_LIVE_RANGE(vv->var_); - if (var_range.end_ < index_store) { - SC_MODULE_WARN << "POTENTIAL ERROR, dead store to var: " << vv; - var_range.update(index_store); - } - if (vv->var_.isa()) { - auto old_ptr = vv->var_.static_as(); - auto new_ptr = - new_temp_tensor(old_ptr, "store_tensor_", index_load, index_store); - auto new_load = new_temp_assign(new_ptr, old_ptr, index_load); - insert_before_.emplace_back(new_load); - auto new_store = new_temp_assign(old_ptr, new_ptr, index_store); - insert_after_.emplace_back(new_store); - vv->var_ = new_ptr; - } else { - auto old_var = vv->var_.static_as(); - auto new_var = - new_temp_var(old_var, "store_var_", index_load, index_store); - auto new_load = new_temp_assign(new_var, old_var, index_load); - insert_before_.emplace_back(new_load); - auto new_store = new_temp_assign(old_var, new_var, index_store); - insert_after_.emplace_back(new_store); - vv->var_ = new_var; - } - return vv; - } - - expr new_temp_var(const expr &old_var, const std::string &prefix, - stmt_index_t start, stmt_index_t end) { - // new var - auto new_var = builder::make_var(old_var->dtype_, - prefix + std::to_string(temp_index_++)); - // set xbyak_expr_data_t - new_var->temp_data() = xbyak_expr_data_t(); - // set virt_reg - auto &new_virt_reg = GET_VIRTUAL_REG(new_var); - new_virt_reg.type_ = - get_virt_reg_type(new_var->dtype_, cpu_flags_.fAVX512F, true); - new_virt_reg.spill_weight_ = spill_weight_const::infinity; - new_virt_reg.live_range_ = live_range_t(start, end); - // add to new virtual_regs - new_virt_reg.set_unassigned(); - virtual_regs_.push_back(&new_virt_reg); - return new_var; - } - - expr new_temp_tensor(const tensor &old_tsr, const std::string &prefix, - stmt_index_t start, stmt_index_t end) { - // new var - auto new_tensor = builder::make_tensor( - prefix + old_tsr->name_, old_tsr->dims_, old_tsr->elem_dtype_); - // set xbyak_expr_data_t - new_tensor->temp_data() = xbyak_expr_data_t(); - // set virt_reg - auto &new_virt_reg = GET_VIRTUAL_REG(new_tensor); - new_virt_reg.type_ = - get_virt_reg_type(new_tensor->dtype_, cpu_flags_.fAVX512F); - new_virt_reg.spill_weight_ = spill_weight_const::infinity; - new_virt_reg.live_range_ = live_range_t(start, end); - // add to new virtual_regs - new_virt_reg.set_unassigned(); - virtual_regs_.push_back(&new_virt_reg); - return new_tensor; - } - - stmt new_temp_assign(expr var, expr value, stmt_index_t index) { - // new var - auto new_assign = - make_stmt(std::move(var), std::move(value)); - // set xbyak_stmt_data_t - new_assign->temp_data() = xbyak_stmt_data_t(loop_depth()); - GET_STMT_INDEX(new_assign) = index; - GET_STMT_INIT_INDEX(new_assign) = index; - return new_assign; - } - - int spilled_args_sum(const std::vector &args) { - int sum = 0; - for (auto &v : args) { - sum += is_spilled(v) ? 1 : 0; - } - return sum; - } - - int spilled_args_sum(const std::vector &args, size_t n) { - int sum = 0; - for (size_t i = 0; i < n; i++) { - sum += is_spilled(args[i]) ? 1 : 0; - } - return sum; - } - - bool is_imm(const expr &v) { - return v.isa() && GET_VIRTUAL_REG(v).disabled(); - } - - bool is_spilled(const expr &v) { - auto &virt_reg = GET_VIRTUAL_REG(v); - return GET_VIRTUAL_REG(v).spilled(); - } - - std::vector insert_before_; - std::vector insert_after_; - - const live_range_t &spill_range_; - std::vector &virtual_regs_; - const runtime::cpu_flags_t &cpu_flags_; - - enum class resolve_dst { - none, - store, - load_store, - } resolve_dst_ = resolve_dst::none; - - bool dst_is_mem_ = false; - - stmt_index_t cur_index_ = 0; -}; - -/* * - * The actual allocator that run allocation pass for each func_c - * */ -class register_allocation_impl_t : public reg_allocator_t { -public: - register_allocation_impl_t(const x86_64::target_profile_t &profile) - : reg_allocator_t(profile), - cpu_flags_(profile.target_machine_.cpu_flags_) {} - - // Fuction pass for register_allocation_impl - func_c operator()(func_c v) { - func_ = pre_allocation(std::move(v)); - resolve_spill(live_range_t()); - run_allocator(); - set_global_spilled(); - set_register_usage(); - return std::move(func_); - } - - // Enqueue all virtual regs and prepare for spill insertion - func_c pre_allocation(func_c v) { - pre_allocation_t pre_allocation(this, virt_reg_map_, cpu_flags_); - return pre_allocation(std::move(v)); - } - - // Check intrin format and create load/store - void resolve_spill_impl(const live_range_t &spill_range, - std::vector &virtual_regs) override { - spill_resolver_t spill_resolver(spill_range, virtual_regs, cpu_flags_); - func_ = spill_resolver(std::move(func_)); - } - - void set_global_spilled() { - func_t func = std::const_pointer_cast(func_); - func->attr().set(attr_keys::global_spilled, spilled_expr_vec()); - } - - void set_register_usage() { - func_t func = std::const_pointer_cast(func_); - func->attr().set(attr_keys::register_usage, slots_array().utilized_slots()); - } - - std::vector spilled_expr_vec() { - std::vector ret_vec; - auto &spilled = spilled_virt_regs(); - ret_vec.reserve(spilled.size()); - for (auto &virt_reg : spilled) { - ret_vec.push_back(virt_reg_map_[virt_reg]); - } - spilled.clear(); - return ret_vec; - } - -private: - func_c func_; - std::map virt_reg_map_; - const runtime::cpu_flags_t &cpu_flags_; -}; - -func_c register_allocation_t::operator()(func_c v) { - if (v->name_.find("_should_inline_") != std::string::npos) { - return v; - } - register_allocation_impl_t reg_allocator(profile_); - return reg_allocator(std::move(v)); -} - -register_allocation_t::register_allocation_t( - const x86_64::target_profile_t &profile) - : profile_(profile) {} - -} // namespace xbyak -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl diff --git a/legacy/core/src/compiler/jit/xbyak/ir/transform/register_allocation.hpp b/legacy/core/src/compiler/jit/xbyak/ir/transform/register_allocation.hpp deleted file mode 100644 index c27ef2645..000000000 --- a/legacy/core/src/compiler/jit/xbyak/ir/transform/register_allocation.hpp +++ /dev/null @@ -1,81 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#ifndef GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_JIT_XBYAK_IR_TRANSFORM_REGISTER_ALLOCATION_HPP -#define GRAPH_BACKEND_GRAPH_COMPILER_CORE_SRC_COMPILER_JIT_XBYAK_IR_TRANSFORM_REGISTER_ALLOCATION_HPP - -#include -#include - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace xbyak { - -namespace attr_keys { -// attr for func, contains all global spilled exprs inside func -// data type: std::vector -constexpr const char *global_spilled = "global_spilled"; -// attr for func, contains all utilized registers inside func -// data type: std::set -constexpr const char *register_usage = "register_usage"; -// attr for for_loop, contains load assign for spilled iter_begin_ -// data type: stmt -constexpr const char *load_loop_begin = "load_loop_begin"; -// attr for for_loop, contains load assign for spilled iter_end_ -// data type: stmt -constexpr const char *load_loop_end = "load_loop_end"; -// attr for for_loop, contains load assign for spilled step_ -// data type: stmt -constexpr const char *load_loop_step = "load_loop_step"; -} // namespace attr_keys - -/* * - * Register allocation uses virtual_slot to represent physcial register, each - * expr have its own virtual_reg which contains its reg_type and live_range. - * Pre-allocation pass put all virtual_regs in a priority_queue. - * For each virtual_reg, allocator use virtual_slots_map to aquire corresponding - * virtual_slot index range in a virtual_slots_array for assignment and try to - * find a available slot that does not interfer with other virtual_regs. - * Virtual_slot uses non-overlapping balanced intervel tree to store live_range - * and check interference, if no virtual_slot is available, virtual_regs is - * spilled and spill will be resolved using address mode checking. - * - * <------------------Live Range------------------> - * _______________________________________________ - * Slot 0|__[--virt_reg_0--)___[-virt_reg_2-)____________| - * Slot 1|________[------virt_reg_1------)_______________| - * Slot 2|_________________[-------virt_reg_3-------)____| - * Slot 3|_______________________________________________| - * Slot ... - * */ -class register_allocation_t : public function_pass_t { -public: - register_allocation_t(const x86_64::target_profile_t &profile); - func_c operator()(func_c v) override; - -private: - const x86_64::target_profile_t &profile_; -}; - -} // namespace xbyak -} // namespace gc -} // namespace graph -} // namespace impl -} // namespace dnnl - -#endif diff --git a/legacy/core/src/compiler/jit/xbyak/ir/transform/x86_intrinsics_lowering.cpp b/legacy/core/src/compiler/jit/xbyak/ir/transform/x86_intrinsics_lowering.cpp deleted file mode 100644 index 3c7b17420..000000000 --- a/legacy/core/src/compiler/jit/xbyak/ir/transform/x86_intrinsics_lowering.cpp +++ /dev/null @@ -1,1510 +0,0 @@ -/******************************************************************************* - * Copyright 2022-2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "util/utils.hpp" -#include "x86_intrinsics_lowering.hpp" - -namespace dnnl { -namespace impl { -namespace graph { -namespace gc { -namespace xbyak { - -class x86_intrinsics_lowering_impl_t : public xbyak_visitor_t { -public: - using xbyak_visitor_t::dispatch; - using xbyak_visitor_t::visit; - - x86_intrinsics_lowering_impl_t( - const runtime::target_machine_t &target_machine) - : cpu_flags_(target_machine.cpu_flags_) { - COMPILE_ASSERT(cpu_flags_.fAVX2, "Support AVX2"); - } - - std::vector transform_seq_; - - expr_c dispatch(expr_c v) override { return v; } - - stmt_c visit(stmts_c v) override { - std::vector new_seq; - for (auto &s : v->seq_) { - auto ss = xbyak_visitor_t::dispatch(s); - if (!transform_seq_.empty()) { - for (auto &&ts : transform_seq_) { - new_seq.push_back(std::move(ts)); - } - transform_seq_.clear(); - } else { - new_seq.emplace_back(ss.remove_const()); - } - } - return copy_attr(*v, make_stmt(std::move(new_seq))); - } - - stmt_c visit(assign_c v) override { - x86_intrinsics_transform(v->var_, v->value_); - return std::move(v); - } - - stmt_c visit(define_c v) override { - if (v->var_.isa() && v->init_.defined()) { - add_defination(v->var_, v->linkage_); - x86_intrinsics_transform(v->var_, v->init_); - } else if (v->var_.isa() && - v->init_.cast() - .filter([](const intrin_call &v) { - return v->type_ == intrin_type::reinterpret && - v->args_.at(0)->dtype_ == datatypes::index; - }) - .has_value()) { - // handle tensor A[...] = reintepret(...) - auto intri = v->init_.static_as(); - add_defination(v->var_, v->linkage_, intri->args_[0]); - } - return std::move(v); - } - - stmt_c visit(evaluate_c v) override { - if (v->value_.isa()) { - x86_intrinsics_transform(expr(), v->value_); - } - return std::move(v); - } - - void x86_intrinsics_transform(const expr &dst, const expr &val) { - switch (val->node_type_) { - case sc_expr_type::add: { - auto bin = val.static_as(); - transform(dst, {bin->l_, bin->r_}, - dst->dtype_, // - transform_3a_to_2a(xbyak_intrin_type::add), - transform_intrin(xbyak_intrin_type::add)); - } break; - case sc_expr_type::sub: { - auto bin = val.static_as(); - transform(dst, {bin->l_, bin->r_}, - dst->dtype_, // - transform_3a_to_2a(xbyak_intrin_type::sub), - transform_intrin(xbyak_intrin_type::sub)); - } break; - case sc_expr_type::mul: { - auto bin = val.static_as(); - transform(dst, {bin->l_, bin->r_}, - dst->dtype_, // - transform_x86_mul(), transform_intrin(xbyak_intrin_type::mul)); - } break; - case sc_expr_type::div: { - auto bin = val.static_as(); - transform(dst, {bin->l_, bin->r_}, - dst->dtype_, // - transform_x86_mod_div(xbyak_intrin_type::div), - transform_avx_div()); - } break; - case sc_expr_type::mod: { - auto bin = val.static_as(); - transform(dst, {bin->l_, bin->r_}, - dst->dtype_, // - transform_x86_mod_div(xbyak_intrin_type::mod), - transform_disabled("mod")); - } break; - case sc_expr_type::logic_and: { - auto log = val.static_as(); - transform(dst, {log->l_, log->r_}, - dst->dtype_, // - transform_3a_to_2a(xbyak_intrin_type::bit_and), - transform_disabled("logic_and")); - } break; - case sc_expr_type::logic_or: { - auto log = val.static_as(); - transform(dst, {log->l_, log->r_}, - dst->dtype_, // - transform_3a_to_2a(xbyak_intrin_type::bit_or), - transform_disabled("logic_or")); - } break; - case sc_expr_type::logic_not: { - auto log = val.static_as(); - transform(dst, {log->in_, builder::make_constant(UINT64_C(1))}, - dst->dtype_, // - transform_3a_to_2a(xbyak_intrin_type::bit_xor), - transform_disabled("logic_not")); - } break; - case sc_expr_type::select: { - auto sel = val.static_as"), llvm::cl::init("-"), - llvm::cl::cat(runnerCategory)}; - llvm::cl::opt mainFuncName{ - "e", - llvm::cl::desc("The function to be executed. If not specified, the " - "first matching function in the module to be used."), - llvm::cl::value_desc("function name"), llvm::cl::cat(runnerCategory)}; - llvm::cl::opt skipPipeline{ - "skip-pipeline", - llvm::cl::desc("Skip the GPU pipeline. It's expected, that the input is " - "already lowered with 'gc-op --gc-gpu-pipeline'."), - llvm::cl::init(false), llvm::cl::cat(runnerCategory)}; - llvm::cl::list sharedLibs{ - "shared-libs", - llvm::cl::desc("Comma separated library paths to link dynamically."), - llvm::cl::MiscFlags::CommaSeparated, llvm::cl::desc(""), - llvm::cl::cat(runnerCategory)}; - llvm::cl::opt printIr{ - "print-ir", - llvm::cl::desc("Print the resulting IR before the execution."), - llvm::cl::init(false), llvm::cl::cat(runnerCategory)}; - llvm::cl::opt dumpSpirv{ - "dump-spirv", llvm::cl::desc("Dump spirv for generated kernels."), - llvm::cl::init(false), llvm::cl::cat(runnerCategory)}; - llvm::cl::opt objDumpFile{ - "obj-dump-file", - llvm::cl::desc("Dump the compiled object to the specified file."), - llvm::cl::value_desc("file path"), llvm::cl::cat(runnerCategory)}; -}; -} // namespace - -void findFunc(Options &opts, ModuleOp mod) { - bool (*matcher)(ArrayRef, ModuleOp &); - - if (opts.skipPipeline) { - matcher = [](ArrayRef args, ModuleOp &mod) { - if (args.size() != 3) - return false; - auto ctx = mod.getContext(); - auto ptrType = LLVM::LLVMPointerType::get(ctx); - return args[0] == ptrType && args[1] == ptrType && - args[2] == IntegerType::get(ctx, 64); - }; - } else { - matcher = [](ArrayRef args, ModuleOp &) { return args.empty(); }; - } - - if (opts.mainFuncName.empty()) { - auto setFuncName = [&](auto funcOp) { - if (funcOp && !funcOp.isExternal() && funcOp.isPublic() && - matcher(funcOp.getArgumentTypes(), mod)) { - opts.mainFuncName = funcOp.getName().str(); - return true; - } - return false; - }; - - for (auto &op : mod.getBody()->getOperations()) { - if (setFuncName(dyn_cast(op)) || - setFuncName(dyn_cast(op))) { - return; - } - } - gcReportErr("No matching function found."); - } - - ArrayRef args; - if (auto llvmFunc = mod.lookupSymbol(opts.mainFuncName)) { - args = llvmFunc.getArgumentTypes(); - } else if (auto func = mod.lookupSymbol(opts.mainFuncName)) { - args = func.getArgumentTypes(); - } else { - gcReportErr("The function '", opts.mainFuncName.c_str(), "' not found."); - } - - if (!matcher(args, mod)) { - if (opts.skipPipeline) { - gcReportErr("The function '", opts.mainFuncName.c_str(), - "' signature does not match (!llvm.ptr, !llvm.ptr, i64)."); - } - gcReportErr("The function '", opts.mainFuncName.c_str(), - "' must have no arguments."); - } -} - -int main(int argc, char **argv) { - Options opts; - llvm::cl::ParseCommandLineOptions(argc, argv, "GraphCompiler GPU runner\n"); - - std::string errMsg; - auto file = openInputFile(opts.inputFilename, &errMsg); - if (!file) { - gcReportErr("Failed to read input IR: ", errMsg.c_str()); - } - - auto srcMgr = std::make_shared(); - srcMgr->AddNewSourceBuffer(std::move(file), SMLoc()); - MLIRContext mlirCtx{gc::initCompilerAndGetDialects()}; - auto mlirMod = parseSourceFile(srcMgr, {&mlirCtx}); - findFunc(opts, *mlirMod); - - gc::gpu::OclModuleBuilderOpts builderOpts; - SmallVector sharedLibs(opts.sharedLibs.begin(), - opts.sharedLibs.end()); - builderOpts.funcName = opts.mainFuncName; - builderOpts.printIr = opts.printIr; - builderOpts.spirvDump = opts.dumpSpirv; - builderOpts.enableObjectDump = !opts.objDumpFile.getValue().empty(); - builderOpts.sharedLibPaths = sharedLibs; - builderOpts.pipeline = - opts.skipPipeline ? [](OpPassManager &) {} : [](OpPassManager &pm) { - gc::GPUPipelineOptions pipelineOpts; - pipelineOpts.isUsmArgs = false; - pipelineOpts.callFinish = true; -#ifdef GC_USE_IMEX - populateIMEXPipeline(pm, pipelineOpts); -#else - populateGPUPipeline(pm, pipelineOpts); -#endif - }; - - gc::gpu::OclModuleBuilder builder{mlirMod, builderOpts}; - auto runtime = gcGetOrReport(gc::gpu::OclRuntime::get()); - auto oclMod = gcGetOrReport(builder.build(runtime)); - assert(oclMod->isStatic); - - if (!opts.objDumpFile.getValue().empty()) { - gcLogD("Dumping the compiled object to ", opts.objDumpFile.getValue()); - oclMod->dumpToObjectFile(opts.objDumpFile.getValue()); - } - - auto queue = gcGetOrReport(runtime.createQueue()); - gc::gpu::OclContext ctx{runtime, queue}; - gc::gpu::StaticExecutor<0> exec{oclMod}; - gcLogD("Executing function ", opts.mainFuncName.c_str(), "()"); - exec(ctx); - gcGetOrReport(ctx.finish()); - gcGetOrReport(runtime.releaseQueue(queue)); - return 0; -} diff --git a/src/gc-opt/CMakeLists.txt b/src/gc-opt/CMakeLists.txt index 2cf8e41a4..0b28dce26 100644 --- a/src/gc-opt/CMakeLists.txt +++ b/src/gc-opt/CMakeLists.txt @@ -1,78 +1,27 @@ -################################################################################ +# ############################################################################## # Copyright (C) 2025 Intel Corporation # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at # # http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions -# and limitations under the License. -# SPDX-License-Identifier: Apache-2.0 -################################################################################ +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. SPDX-License-Identifier: Apache-2.0 +# ############################################################################## if(NOT GC_ENABLE_OPT) - message(STATUS "gc-opt is not enabled.") + message(STATUS "gc-opt is disabled") return() endif() -if(GC_DEV_LINK_LLVM_DYLIB) - set(MLIR_LINK_COMPONENTS - MLIR - ) - get_property(dialect_libs GLOBAL PROPERTY GC_DIALECT_LIBS) - get_property(conversion_libs GLOBAL PROPERTY GC_PASS_LIBS) -else() - set(MLIR_LINK_COMPONENTS - MLIROptLib - MLIRBuiltinToLLVMIRTranslation - MLIRLLVMDialect - MLIRLLVMToLLVMIRTranslation - MLIRToLLVMIRTranslationRegistration - ) - get_property(dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS) - get_property(conversion_libs GLOBAL PROPERTY MLIR_CONVERSION_LIBS) +gc_add_mlir_tool(gc-opt GcOpt.cpp) +target_link_libraries(gc-opt PRIVATE ${PROJECT_NAME}) +if(NOT GC_DYLINK) + target_link_libraries(gc-opt PRIVATE MLIROptLib MLIRLinalgTransformOps + MLIRTransformDialectTransforms) endif() - -add_llvm_executable(gc-opt gc-opt.cpp) -llvm_update_compile_flags(gc-opt) -mlir_check_all_link_libraries(gc-opt) - -get_property(extension_libs GLOBAL PROPERTY MLIR_EXTENSION_LIBS) - -target_link_libraries(gc-opt PUBLIC GcInterface) -target_link_libraries(gc-opt PRIVATE - ${dialect_libs} - ${extension_libs} - ${conversion_libs} - ${MLIR_LINK_COMPONENTS} - GcPasses - GcAnalysis -) - -target_link_libraries(gc-opt PRIVATE GcGpuPasses) - -if(GC_ENABLE_IMEX) - include(imex) - get_property(IMEX_INCLUDES GLOBAL PROPERTY IMEX_INCLUDES) - target_include_directories(gc-opt PRIVATE ${IMEX_INCLUDES}) - target_link_libraries(gc-opt PRIVATE - IMEXGPUXDialect - IMEXXeTileDialect - IMEXRegionDialect - IMEXRegionTransforms - IMEXTransforms - IMEXGPUToGPUX - IMEXGPUToSPIRV - IMEXGPUXToLLVM - IMEXXeGPUToVC - IMEXXeTileToXeGPU - IMEXUtil - ) -endif() - -set_property(GLOBAL APPEND PROPERTY GC_TOOLS gc-opt) diff --git a/src/gc-opt/GcOpt.cpp b/src/gc-opt/GcOpt.cpp new file mode 100644 index 000000000..12c806748 --- /dev/null +++ b/src/gc-opt/GcOpt.cpp @@ -0,0 +1,36 @@ + +/* + * Copyright (C) 2025 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "gc/Transforms/Passes.h" + +#include "mlir/Dialect/Linalg/TransformOps/DialectExtension.h" +#include "mlir/Dialect/Transform/IR/TransformDialect.h" +#include "mlir/Dialect/Transform/Transforms/Passes.h" +#include "mlir/Tools/mlir-opt/MlirOptMain.h" + +int main(int argc, char *argv[]) { + mlir::DialectRegistry ®istry = mlir::gc::getDialectRegistry(); + registry.insert(); + mlir::linalg::registerTransformDialectExtension(registry); + mlir::transform::registerTransformPasses(); + + mlir::gc::registerGPUPipeline(); + return mlir::asMainReturnCode(mlir::MlirOptMain( + argc, argv, "Graph Compiler modular optimizer driver\n", registry)); +} diff --git a/src/gc-opt/gc-opt.cpp b/src/gc-opt/gc-opt.cpp deleted file mode 100644 index 2de956e01..000000000 --- a/src/gc-opt/gc-opt.cpp +++ /dev/null @@ -1,98 +0,0 @@ - -/* - * Copyright (C) 2025 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "gc/Dialect/CPURuntime/Transforms/CPURuntimePasses.h" -#include "gc/Dialect/LLVMIR/XeVMDialect.h" -#include "gc/Dialect/Linalgx/LinalgxDialect.h" -#include "gc/Dialect/Microkernel/MicrokernelDialect.h" -#ifdef GC_HAS_ONEDNN_DIALECT -#include "gc/Dialect/OneDNNGraph/OneDNNGraphDialect.h" -#endif -#include "gc/Conversion/Passes.h" -#include "mlir/Target/LLVMIR/Dialect/All.h" - -#include "gc/Target/LLVM/XeVM/Target.h" -#include "gc/Target/LLVMIR/Dialect/XeVM/XeVMToLLVMIRTranslation.h" -#include "gc/Transforms/Microkernel/MicrokernelPasses.h" -#include "gc/Transforms/Passes.h" -#include "mlir/InitAllDialects.h" -#include "mlir/InitAllExtensions.h" -#include "mlir/InitAllPasses.h" -#include "mlir/Target/LLVMIR/Dialect/All.h" -#include "mlir/Tools/mlir-opt/MlirOptMain.h" - -#ifdef GC_USE_IMEX -#include -#include -#endif - -namespace mlir::gc { -void registerCPUPipeline(); -#ifdef GC_USE_GPU -void registerGPUPipeline(); -#endif - -#ifdef GC_USE_IMEX -void registerIMEXPipeline(); -#endif - -} // namespace mlir::gc - -int main(int argc, char *argv[]) { -#ifdef GC_USE_IMEX - imex::registerTransformsPasses(); - // Conversion passes - imex::registerConvertGPUToGPUX(); - imex::registerConvertGPUXToLLVM(); - imex::registerConvertGPUXToSPIRV(); - imex::registerConvertXeGPUToVC(); - imex::registerConvertXeTileToXeGPU(); - mlir::gc::registerIMEXPipeline(); -#endif - mlir::registerAllPasses(); - mlir::gc::registerCPUPipeline(); - mlir::gc::registerGraphCompilerPasses(); - mlir::registerGCConversionPasses(); - mlir::cpuruntime::registerCPURuntimePasses(); - mlir::microkernel::registerMicrokernelPasses(); - - mlir::DialectRegistry registry; -#ifdef GC_HAS_ONEDNN_DIALECT - registry.insert(); -#endif - registry.insert(); - registry.insert(); - registry.insert(); - mlir::registerAllDialects(registry); -#ifdef GC_USE_IMEX - registry.insert<::imex::xetile::XeTileDialect, ::imex::gpux::GPUXDialect>(); -#endif - registry.insert(); - mlir::registerConvertXeVMToLLVMInterface(registry); - mlir::registerXeVMDialectTranslation(registry); - mlir::xevm::registerXeVMTargetInterfaceExternalModels(registry); -#ifdef GC_USE_GPU - mlir::gc::registerGPUPipeline(); -#endif - mlir::cpuruntime::registerConvertCPURuntimeToLLVMInterface(registry); - mlir::registerAllExtensions(registry); // TODO: cleanup - mlir::registerAllToLLVMIRTranslations(registry); - return mlir::asMainReturnCode(mlir::MlirOptMain( - argc, argv, "Graph Compiler modular optimizer driver\n", registry)); -} diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 4baaa28de..1facecd90 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,12 +1,6 @@ -if (NOT GC_ENABLE_TEST) - message(STATUS "The tests are not enabled.") - return() -endif () +if(NOT GC_ENABLE_TEST) + message(STATUS "Tests are disabled") + return() +endif() -include(gtest) -add_subdirectory(dnnl) add_subdirectory(mlir) - -if(GC_BENCH_ENABLE) - add_subdirectory(benchgc) -endif() \ No newline at end of file diff --git a/test/README.md b/test/README.md deleted file mode 100644 index e6a697285..000000000 --- a/test/README.md +++ /dev/null @@ -1,23 +0,0 @@ -# GC Tests - -Tests structured the way MLIR part is similar to what one finds in the upstream (`mlir/unittest` for C++ tests and `mlir/test` for lits). These can be invoked by `gc-check` cmake target. - -Backend specific tests are separate, gtest-based. - -``` -tests - \ - dnnl - \ - dnnl-specific tests - mlir <-------- file structure kept same as upstream - \ - tests - \ - mlir lit tests - lit.cfg.py - lit.site.cfg.py.in - unittest - \ - mlir unittests -``` diff --git a/test/benchgc/.gitignore b/test/benchgc/.gitignore deleted file mode 100644 index 0fcd2be1e..000000000 --- a/test/benchgc/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -dist/ -src/benchgc.egg-info/ -build -benchgc.egg-info/ -__pycache__ diff --git a/test/benchgc/CMakeLists.txt b/test/benchgc/CMakeLists.txt deleted file mode 100644 index ff00d27b7..000000000 --- a/test/benchgc/CMakeLists.txt +++ /dev/null @@ -1,43 +0,0 @@ -################################################################################ -# Copyright (C) 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions -# and limitations under the License. -# SPDX-License-Identifier: Apache-2.0 -################################################################################ - -if(NOT GC_BENCH_ENABLE) - message(STATUS "Benchgc is not enabled") - return() -endif() - -configure_file(setup.py ${CMAKE_BINARY_DIR}/test/benchgc/setup.py COPYONLY) - -file(GLOB PYTHON_SCRIPTS "*.py") -foreach(PY_SCRIPT ${PYTHON_SCRIPTS}) - configure_file(${PY_SCRIPT} ${CMAKE_BINARY_DIR} COPYONLY) -endforeach() - -find_package(Python 3.10 REQUIRED COMPONENTS Interpreter NumPy REQUIRED) -add_custom_target(benchgc - COMMAND ${Python_EXECUTABLE} setup.py bdist_wheel - WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/test/benchgc/" - DEPENDS GcPythonModules) - -add_subdirectory("src/benchgc") -add_subdirectory("src/benchgc/arg") -add_subdirectory("src/benchgc/mlir") -add_subdirectory("src/benchgc/linalg") -add_subdirectory("src/benchgc/tensor") -add_subdirectory("src/benchgc/arith") -add_subdirectory("src/benchgc/pattern") -add_subdirectory("src/benchgc/math") diff --git a/test/benchgc/README.md b/test/benchgc/README.md deleted file mode 100644 index 239105c82..000000000 --- a/test/benchgc/README.md +++ /dev/null @@ -1,437 +0,0 @@ -# benchgc - benchmark tool for graph compiler - -## Description - -Benchgc is a tool used to verify the correctness and performance of graph compiler. Benchgc accepts MLIR files as test cases and prepares test data for them. For correctness verification, Benchgc will use PyTorch as a reference for comparison. - -## Prerequisite -* python >= 3.10 -* torch >= 2.2 -* Enable mlir python binding, Refer to [`python/README.md`](../../python/README.md) for detail -* Set the envs - * OMP_NUM_THREADS [int] : the `num_threads` for dlti attr, default = 1 - -## Build -There are two ways for using benchgc - -* Build `.whl` and install benchgc -``` -# Please execute at the top level of the project - -mkdir build && cd build -cmake .. -DMLIR_DIR=$MLIR_PATH -DGC_TEST_ENABLE=ON -DGC_ENABLE_BINDINGS_PYTHON=ON -DGC_BENCH_ENABLE=ON -make -j benchgc -python -m pip install test/benchgc/dist/benchgc-*.whl - -``` - -* Run benchgc from source code - -``` -# Please execute at the top level of the project - -mkdir build && cd build -cmake .. -DMLIR_DIR=$MLIR_PATH -DGC_TEST_ENABLE=ON -DGC_ENABLE_BINDINGS_PYTHON=ON -DGC_BENCH_ENABLE=ON -make -j GcPythonModules -export PYTHONPATH=$(pwd)/python_packages/gc_mlir_core/:$(pwd)/../test/benchgc/src/ -``` - -## Synopsis -``` -python -m benchgc [OPTIONS] --mode [MODE] --driver [DRIVER] --case [CASE] -``` -## Common Options -### --mode [str] -* C : correctness testing (by default) -* P : performance testing - -### --driver [str] -* linalg: test the single op in linalg dialect -* mlir: upload a mlir file and run -* pattern: predefined pattern test such as mlp - -### --case [str] -* if driver=mlir, please provide a mlir file here to test -* if driver=pattern, please provide the pre-defined pattern name, such as mlp here -* if driver is a dialect name, please provide the detail op name to start a single op test - -### --entry [str] -* default : "entry" -* the entry name of the kernel of input mlir or generated mlir - -### --seed [int] -* set the seed to generate the test data and reprodce the test - -### --verbose [int] -* set the verbose level, default : 0 -* 0 : NO_VERBOSE -* 1 : MODULE_VERBOSE, print the module will be executed -* 2 : ARG_VERBOSE, + print arg information -* 3 : COMPARE_VERBOSE, + print threshold for comparison -* 4 : ERROR_OUTPUT_VERBOSE, + print all error data points if failed -* 5 : OUTPUT_VERBOSE, + print all result including passed tensor -* 6 : INPUT_VERBOSE, + print input torch tensors - -### --ir_printing (action=store_true) -* Print the ir during the pass-pipeline - -### --md index:SHAPExTYPE -* Describe the shape and data type for argument -* Not available when driver=mlir -* index means the order of argument, including both inputs and outs -* use prefix `0x` (e.g. `0xbf16`) to represent 0d memref or tensor input -* use data type directly (e.g.`f32`) to represent a normal scalar - -``` -# %arg0 -> index = 0 -# tensor<2x2x2xf32> -> index = 1 - -module { - func.func @entry(%arg0: f32) -> tensor<2x2x2xf32> attributes {llvm.emit_c_interface} { - %0 = tensor.empty() : tensor<2x2x2xf32> - %1 = linalg.fill ins(%arg0 : f32) outs(%0 : tensor<2x2x2xf32>) -> tensor<2x2x2xf32> - return %1 : tensor<2x2x2xf32> - } -} -``` - -### --fill index:fill_type:[:fill_parameter]* -* If not set, benchgc will assign a default method for the argument - -| description | fill_type | fill_parameter | -|-------------|-----------|-----------| -| Zero | Z | | -| Normal | N | mean, std | -| Poisson | P | lambda | -| Binomial | B | n, p | -| Uniform | U | a, b | -| Integer | I | a, b | -| Pytorch tensor dump | F | dump filename | -| Benchdnn driver | D | driver_name[:driver filling parameter]* | - -### --cpu_cache_sizes, --max_vector_width -* BenchGC will automatically obtain target info and add the DLTI attr to the IR -* In some cases, if the system info obtained by BenchGC is not accurate, you can specify the relevant attributes for BenchGC through these options. -* --cpu_cache_sizes: cpu cache sizes in bytes, format: L1:L2:L3, example: `--cpu_cache_sizes 49152:2097152:110100480` -* --max_vector_width: the maximum width of vector registers available in a CPU, example `--max_vector_width ` - -#### Benchdnn driver filling - -| driver_name | driver filling parameter | -|-------------|--------------------------| -| binary | src0/src1:src0 dtype:src1 dtype:dst dtype | -| conv | src/wei:src dtype:wei dtype:dst dtype:amplifier | -| eltwise | algorithm: alpha: beta (please check https://oneapi-src.github.io/oneDNN/dev_guide_eltwise.html) | -| matmul | src/wei:src dtype:wei dtype:dst dtype:amplifier | -| pool | not required | - -### --cmp index:cmp_type:[:cmp_parameter]* -* If not set, benchgc will assign a default method for the argument - -| description | cmp_type | cmp_parameter | -|-------------|-----------|-----------| -| P2P check | P | threshold, zero_percent(mistrust check) | -| Norm check | N | threshold | -| Benchdnn driver | D | driver_name:dtype:case | - -## Bench Options -### --bench_kind [str] -* py : use the MLIR Python API to invoke the kernel and use Python to calculate the time cost -* wrapper : modify MLIR by wrapping the kernel into a new method and calling the `nanoTime()` method before and after calling the kernel. Finally, calculate the difference as the time cost - -### --warm_up [int] -* warm-up times of the execution - -### --repeat [int] -* repeat times of the execution - -## Pattern Options -Each pattern has its own unique options. -### mlp -* `--batch_size`: the input -* `--hidden_size_list`: hidden_sizes of mlp, example: 32x16x64 -* `--has_bias`: if the matmul op has bias, example: 1x0 -* `--act_type`: choices=["noop", "relu"] -* `--dtype`: choices=["bf16", "f32"] - -## Example -### Correctness testing example -``` -# single add op test -# using the same data filling / compare strategy as the benchdnn primitive driver if not set -python3 -m benchgc --verbose 6 --driver linalg --case add --md 0:4x5xf32 --md 1:4x5xf32 --md 2:4x5xf32 - -arg0 shape: [4, 5] dtype: f32 fill_type: D fill_param: ['binary', 'src0', 'f32', 'f32', 'f32'] cmp_type: D cmp_param: ['binary', 'f32', 'add'] -arg1 shape: [4, 5] dtype: f32 fill_type: D fill_param: ['binary', 'src1', 'f32', 'f32', 'f32'] cmp_type: D cmp_param: ['binary', 'f32', 'add'] -arg2 shape: [4, 5] dtype: f32 fill_type: Z fill_param: [] cmp_type: D cmp_param: ['binary', 'f32', 'add'] -module { - func.func @entry(%arg0: tensor<4x5xf32>, %arg1: tensor<4x5xf32>) -> tensor<4x5xf32> attributes {llvm.emit_c_interface} { - %cst = arith.constant 0.000000e+00 : f32 - %0 = tensor.empty() : tensor<4x5xf32> - %1 = linalg.fill ins(%cst : f32) outs(%0 : tensor<4x5xf32>) -> tensor<4x5xf32> - %2 = linalg.add ins(%arg0, %arg1 : tensor<4x5xf32>, tensor<4x5xf32>) outs(%1 : tensor<4x5xf32>) -> tensor<4x5xf32> - return %2 : tensor<4x5xf32> - } -} - -fill arg0: -tensor([[ -5.0000, 10.0000, 3.7500, -2.5000, -8.7500], - [ 6.2500, 0.0000, -6.2500, 8.7500, 2.5000], - [ -3.7500, -10.0000, 5.0000, -1.2500, -7.5000], - [ 7.5000, 1.2500, -5.0000, 10.0000, 3.7500]]) -fill arg1: -tensor([[ 1.2500, -5.0000, 10.0000, 3.7500, -2.5000], - [ -8.7500, 6.2500, 1.0000, -6.2500, 8.7500], - [ 2.5000, -3.7500, -10.0000, 5.0000, -1.2500], - [ -7.5000, 7.5000, 1.2500, -5.0000, 10.0000]]) -fill arg2: -tensor([[0., 0., 0., 0., 0.], - [0., 0., 0., 0., 0.], - [0., 0., 0., 0., 0.], - [0., 0., 0., 0., 0.]]) -p2p check: threshold: 0.0000001 - (0, 0): ref: -5.0000000 res: -5.0000000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (0, 1): ref: 10.0000000 res: 10.0000000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (0, 2): ref: 3.7500000 res: 3.7500000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (0, 3): ref: -2.5000000 res: -2.5000000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (0, 4): ref: -8.7500000 res: -8.7500000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (1, 0): ref: 6.2500000 res: 6.2500000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (1, 1): ref: 0.0000000 res: 0.0000000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (1, 2): ref: -6.2500000 res: -6.2500000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (1, 3): ref: 8.7500000 res: 8.7500000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (1, 4): ref: 2.5000000 res: 2.5000000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (2, 0): ref: -3.7500000 res: -3.7500000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (2, 1): ref: -10.0000000 res: -10.0000000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (2, 2): ref: 5.0000000 res: 5.0000000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (2, 3): ref: -1.2500000 res: -1.2500000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (2, 4): ref: -7.5000000 res: -7.5000000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (3, 0): ref: 7.5000000 res: 7.5000000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (3, 1): ref: 1.2500000 res: 1.2500000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (3, 2): ref: -5.0000000 res: -5.0000000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (3, 3): ref: 10.0000000 res: 10.0000000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (3, 4): ref: 3.7500000 res: 3.7500000 abs_diff: 0.0000000 rel_diff: 0.0000000 -p2p check: threshold: 0.0000001 - (0, 0): ref: 1.2500000 res: 1.2500000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (0, 1): ref: -5.0000000 res: -5.0000000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (0, 2): ref: 10.0000000 res: 10.0000000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (0, 3): ref: 3.7500000 res: 3.7500000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (0, 4): ref: -2.5000000 res: -2.5000000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (1, 0): ref: -8.7500000 res: -8.7500000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (1, 1): ref: 6.2500000 res: 6.2500000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (1, 2): ref: 1.0000000 res: 1.0000000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (1, 3): ref: -6.2500000 res: -6.2500000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (1, 4): ref: 8.7500000 res: 8.7500000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (2, 0): ref: 2.5000000 res: 2.5000000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (2, 1): ref: -3.7500000 res: -3.7500000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (2, 2): ref: -10.0000000 res: -10.0000000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (2, 3): ref: 5.0000000 res: 5.0000000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (2, 4): ref: -1.2500000 res: -1.2500000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (3, 0): ref: -7.5000000 res: -7.5000000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (3, 1): ref: 7.5000000 res: 7.5000000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (3, 2): ref: 1.2500000 res: 1.2500000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (3, 3): ref: -5.0000000 res: -5.0000000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (3, 4): ref: 10.0000000 res: 10.0000000 abs_diff: 0.0000000 rel_diff: 0.0000000 -p2p check: threshold: 0.0000001 - (0, 0): ref: -3.7500000 res: -3.7500000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (0, 1): ref: 5.0000000 res: 5.0000000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (0, 2): ref: 13.7500000 res: 13.7500000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (0, 3): ref: 1.2500000 res: 1.2500000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (0, 4): ref: -11.2500000 res: -11.2500000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (1, 0): ref: -2.5000000 res: -2.5000000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (1, 1): ref: 6.2500000 res: 6.2500000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (1, 2): ref: -5.2500000 res: -5.2500000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (1, 3): ref: 2.5000000 res: 2.5000000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (1, 4): ref: 11.2500000 res: 11.2500000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (2, 0): ref: -1.2500000 res: -1.2500000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (2, 1): ref: -13.7500000 res: -13.7500000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (2, 2): ref: -5.0000000 res: -5.0000000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (2, 3): ref: 3.7500000 res: 3.7500000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (2, 4): ref: -8.7500000 res: -8.7500000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (3, 0): ref: 0.0000000 res: 0.0000000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (3, 1): ref: 8.7500000 res: 8.7500000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (3, 2): ref: -3.7500000 res: -3.7500000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (3, 3): ref: 5.0000000 res: 5.0000000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (3, 4): ref: 13.7500000 res: 13.7500000 abs_diff: 0.0000000 rel_diff: 0.0000000 -PASSED: linalg.add -``` - -``` -# set the arg0 filling follows a distribution N(0, 5) -# set the arg1 filling follows a uniform integer filling [-3, 3] -# use P2P compare strategy on arg2 with threshold = 0 & mistrust rate = 100.0% -# zero threshold will fail the case here - -python3 -m benchgc --verbose 6 --driver linalg --case matmul_transpose_b --md 0:2x5xf32 --md 1:2x5xf32 --md 2:2x2xf32 --fill 0:N:0:5 --fill 1:I:-3:3 --cmp 2:P:0:100 -arg0 shape: [2, 5] dtype: f32 fill_type: N fill_param: ['0', '5'] cmp_type: D cmp_param: ['matmul', 'f32', 'matmul_transpose_b'] -arg1 shape: [2, 5] dtype: f32 fill_type: I fill_param: ['-3', '3'] cmp_type: D cmp_param: ['matmul', 'f32', 'matmul_transpose_b'] -arg2 shape: [2, 2] dtype: f32 fill_type: Z fill_param: [] cmp_type: P cmp_param: ['0', '100'] -module { - func.func @entry(%arg0: tensor<2x5xf32>, %arg1: tensor<2x5xf32>) -> tensor<2x2xf32> attributes {llvm.emit_c_interface} { - %cst = arith.constant 0.000000e+00 : f32 - %0 = tensor.empty() : tensor<2x2xf32> - %1 = linalg.fill ins(%cst : f32) outs(%0 : tensor<2x2xf32>) -> tensor<2x2xf32> - %2 = linalg.matmul_transpose_b {cast = #linalg.type_fn} ins(%arg0, %arg1 : tensor<2x5xf32>, tensor<2x5xf32>) outs(%1 : tensor<2x2xf32>) -> tensor<2x2xf32> - return %2 : tensor<2x2xf32> - } -} - -fill arg0: -tensor([[ 7.7050, -1.4671, -10.8939, 2.8422, -5.4226], - [ -6.9930, 2.0167, 4.1901, -3.5963, -2.0167]]) -fill arg1: -tensor([[-3., 0., 1., 0., 0.], - [ 3., -3., 2., -3., 0.]]) -fill arg2: -tensor([[0., 0.], - [0., 0.]]) -p2p check: threshold: 0.0000010 - (0, 0): ref: 7.7049804 res: 7.7049804 abs_diff: 0.0000000 rel_diff: 0.0000000 - (0, 1): ref: -1.4671445 res: -1.4671445 abs_diff: 0.0000000 rel_diff: 0.0000000 - (0, 2): ref: -10.8939466 res: -10.8939466 abs_diff: 0.0000000 rel_diff: 0.0000000 - (0, 3): ref: 2.8421564 res: 2.8421564 abs_diff: 0.0000000 rel_diff: 0.0000000 - (0, 4): ref: -5.4226117 res: -5.4226117 abs_diff: 0.0000000 rel_diff: 0.0000000 - (1, 0): ref: -6.9929771 res: -6.9929771 abs_diff: 0.0000000 rel_diff: 0.0000000 - (1, 1): ref: 2.0167341 res: 2.0167341 abs_diff: 0.0000000 rel_diff: 0.0000000 - (1, 2): ref: 4.1901317 res: 4.1901317 abs_diff: 0.0000000 rel_diff: 0.0000000 - (1, 3): ref: -3.5962880 res: -3.5962880 abs_diff: 0.0000000 rel_diff: 0.0000000 - (1, 4): ref: -2.0167177 res: -2.0167177 abs_diff: 0.0000000 rel_diff: 0.0000000 -p2p check: threshold: 0.0000010 - (0, 0): ref: -3.0000000 res: -3.0000000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (0, 1): ref: 0.0000000 res: 0.0000000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (0, 2): ref: 1.0000000 res: 1.0000000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (0, 3): ref: 0.0000000 res: 0.0000000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (0, 4): ref: 0.0000000 res: 0.0000000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (1, 0): ref: 3.0000000 res: 3.0000000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (1, 1): ref: -3.0000000 res: -3.0000000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (1, 2): ref: 2.0000000 res: 2.0000000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (1, 3): ref: -3.0000000 res: -3.0000000 abs_diff: 0.0000000 rel_diff: 0.0000000 - (1, 4): ref: 0.0000000 res: 0.0000000 abs_diff: 0.0000000 rel_diff: 0.0000000 -p2p check: threshold: 0.0000000 - (0, 0): ref: -34.0088882 res: -34.0088882 abs_diff: 0.0000000 rel_diff: 0.0000000 - (0, 1): ref: -2.7979884 res: -2.7979879 abs_diff: 0.0000005 rel_diff: 0.0000002 - (1, 0): ref: 25.1690636 res: 25.1690636 abs_diff: 0.0000000 rel_diff: 0.0000000 - (1, 1): ref: -7.8600063 res: -7.8600044 abs_diff: 0.0000019 rel_diff: 0.0000002 -FAIL: linalg.matmul_transpose_b -``` - -### Perf testing example -* single op example -``` -python3 -m benchgc --verbose 1 --mode P --driver linalg --case add --md 0:4x5xf32 --md 1:4x5xf32 --md 2:4x5xf32 - -module { - func.func @entry(%arg0: tensor<4x5xf32>, %arg1: tensor<4x5xf32>) -> tensor<4x5xf32> attributes {llvm.emit_c_interface} { - %cst = arith.constant 0.000000e+00 : f32 - %0 = tensor.empty() : tensor<4x5xf32> - %1 = linalg.fill ins(%cst : f32) outs(%0 : tensor<4x5xf32>) -> tensor<4x5xf32> - %2 = linalg.add ins(%arg0, %arg1 : tensor<4x5xf32>, tensor<4x5xf32>) outs(%1 : tensor<4x5xf32>) -> tensor<4x5xf32> - return %2 : tensor<4x5xf32> - } -} - -===========bench result=========== -{ - "args": { - "mode": "P", - "driver": "linalg", - "case": "add", - "md": [ - "0:4x5xf32", - "1:4x5xf32", - "2:4x5xf32" - ], - "fill": [], - "cmp": [], - "seed": 0, - "verbose": 1, - "entry": "entry", - "ir_printing": false, - "cast": "cast_signed", - "dimension": null, - "dimensions": null, - "dilations": null, - "strides": null, - "bench_kind": "py", - "warm_up": 100, - "repeat": 100 - }, - "compile_cost(ms)": 37.72595152258873, - "execute_cost(ms)": 0.00022314488887786865 -} -``` - -* mlir example -``` -python3 -m benchgc --mode P --verbose 1 --driver mlir --case=./test.mlir --bench_kind wrapper --warm_up 50 --repeat 200 -\module { - func.func @entry(%arg0: tensor<512x128xf32>) -> tensor<512x128xf32> attributes {llvm.emit_c_interface} { - %cst = arith.constant 0.000000e+00 : f32 - %0 = tensor.empty() : tensor<512x128xf32> - %1 = linalg.fill ins(%cst : f32) outs(%0 : tensor<512x128xf32>) -> tensor<512x128xf32> - %2 = linalg.abs ins(%arg0 : tensor<512x128xf32>) outs(%1 : tensor<512x128xf32>) -> tensor<512x128xf32> - return %2 : tensor<512x128xf32> - } -} - -===========bench result=========== -{ - "args": { - "mode": "P", - "driver": "mlir", - "case": "/home/xurui/gc_v2/test.mlir", - "md": [], - "fill": [], - "cmp": [], - "seed": 0, - "verbose": 1, - "entry": "entry", - "ir_printing": false, - "bench_kind": "wrapper", - "warm_up": 50, - "repeat": 200 - }, - "compile_cost(ms)": 70.6995539367199, - "execute_cost(ms)": 0.029325044999999984 -} -``` -* mlp example -``` -python3 -m benchgc --verbose 1 --mode P --driver pattern --case mlp --batch_size=32 --hidden_size_list=32x16x64 --has_bias=0x0 --act_type=noop --dtype=f32 - -module { - func.func @entry(%arg0: tensor<32x32xf32>, %arg1: tensor<32x16xf32>, %arg2: tensor<16x64xf32>) -> tensor<32x64xf32> attributes {llvm.emit_c_interface} { - %0 = tensor.empty() : tensor<32x16xf32> - %1 = linalg.matmul {cast = #linalg.type_fn} ins(%arg0, %arg1 : tensor<32x32xf32>, tensor<32x16xf32>) outs(%0 : tensor<32x16xf32>) -> tensor<32x16xf32> - %2 = tensor.empty() : tensor<32x64xf32> - %3 = linalg.matmul {cast = #linalg.type_fn} ins(%1, %arg2 : tensor<32x16xf32>, tensor<16x64xf32>) outs(%2 : tensor<32x64xf32>) -> tensor<32x64xf32> - return %3 : tensor<32x64xf32> - } -} - -===========bench result=========== -{ - "args": { - "mode": "P", - "driver": "pattern", - "case": "mlp", - "md": [], - "fill": [], - "cmp": [], - "seed": 0, - "verbose": 1, - "entry": "entry", - "ir_printing": false, - "bench_kind": "py", - "warm_up": 100, - "repeat": 100, - "batch_size": 32, - "hidden_size_list": "32x16x64", - "has_bias": "0x0", - "act_type": "noop", - "dtype": "f32" - }, - "compile_cost(ms)": 109.86808314919472, - "execute_cost(ms)": 0.02944003790616989 -} - -``` \ No newline at end of file diff --git a/test/benchgc/cases/generic.mlir b/test/benchgc/cases/generic.mlir deleted file mode 100644 index 3da555777..000000000 --- a/test/benchgc/cases/generic.mlir +++ /dev/null @@ -1,15 +0,0 @@ -#map = affine_map<(d0, d1, d2) -> (d0, d2)> -#map1 = affine_map<(d0, d1, d2) -> (d2, d1)> -#map2 = affine_map<(d0, d1, d2) -> (d0, d1)> -module { - func.func @entry(%arg0: tensor<3x2xf32>, %arg1: tensor<2x3xf32>) -> tensor<3x3xf32> attributes {llvm.emit_c_interface} { - %0 = tensor.empty() : tensor<3x3xf32> - %1 = linalg.generic {indexing_maps = [#map, #map1, #map2], iterator_types = ["parallel", "parallel", "reduction"]} ins(%arg0, %arg1 : tensor<3x2xf32>, tensor<2x3xf32>) outs(%0 : tensor<3x3xf32>) { - ^bb0(%in: f32, %in_0: f32, %out: f32): - %2 = arith.mulf %in, %in_0 : f32 - %3 = arith.addf %out, %2 : f32 - linalg.yield %3 : f32 - } -> tensor<3x3xf32> - return %1 : tensor<3x3xf32> - } -} \ No newline at end of file diff --git a/test/benchgc/cases/llama2.mlir b/test/benchgc/cases/llama2.mlir deleted file mode 100644 index 1f557b3e6..000000000 --- a/test/benchgc/cases/llama2.mlir +++ /dev/null @@ -1,113 +0,0 @@ -module { - func.func @entry(%arg0: tensor<1x32x4096xbf16>, %arg1: tensor<4096x4096xbf16>, %arg2: tensor<1x32x4096xbf16>, %arg3: tensor<1xf32>, %arg4: tensor<4096xbf16>, %arg5: tensor<11008x4096xbf16>, %arg6: tensor<11008x4096xbf16>, %arg7: tensor<4096x11008xbf16>, %arg8: tensor<1xf32>, %arg9: tensor<4096xbf16>) -> (tensor<1x32x4096xbf16>, tensor<1x32x4096xbf16>) attributes {llvm.emit_c_interface} { - %collapsed = tensor.collapse_shape %arg0 [[0, 1], [2]] : tensor<1x32x4096xbf16> into tensor<32x4096xbf16> - %cst = arith.constant 0.000000e+00 : bf16 - %0 = tensor.empty() : tensor<32x4096xbf16> - %1 = linalg.fill ins(%cst : bf16) outs(%0 : tensor<32x4096xbf16>) -> tensor<32x4096xbf16> - %2 = linalg.matmul_transpose_b ins(%collapsed, %arg1 : tensor<32x4096xbf16>, tensor<4096x4096xbf16>) outs(%1 : tensor<32x4096xbf16>) -> tensor<32x4096xbf16> - %expanded = tensor.expand_shape %2 [[0, 1], [2]] output_shape [1, 32, 4096] : tensor<32x4096xbf16> into tensor<1x32x4096xbf16> - %3 = tensor.empty() : tensor<1x32x4096xbf16> - %4 = linalg.add ins(%arg2, %expanded : tensor<1x32x4096xbf16>, tensor<1x32x4096xbf16>) outs(%3 : tensor<1x32x4096xbf16>) -> tensor<1x32x4096xbf16> - %5 = tensor.empty() : tensor<1x32x4096xf32> - %6 = linalg.copy ins(%4 : tensor<1x32x4096xbf16>) outs(%5 : tensor<1x32x4096xf32>) -> tensor<1x32x4096xf32> - %cst_0 = arith.constant dense<2.000000e+00> : tensor<1x32x4096xf32> - %7 = tensor.empty() : tensor<1x32x4096xf32> - %8 = linalg.powf ins(%6, %cst_0 : tensor<1x32x4096xf32>, tensor<1x32x4096xf32>) outs(%7 : tensor<1x32x4096xf32>) -> tensor<1x32x4096xf32> - %cst_1 = arith.constant 0.000000e+00 : f32 - %9 = tensor.empty() : tensor<1x32xf32> - %10 = linalg.fill ins(%cst_1 : f32) outs(%9 : tensor<1x32xf32>) -> tensor<1x32xf32> - %reduced = linalg.reduce ins(%8 : tensor<1x32x4096xf32>) outs(%10 : tensor<1x32xf32>) dimensions = [2] - (%in: f32, %init: f32) { - %67 = arith.addf %in, %init : f32 - linalg.yield %67 : f32 - } - %cst_2 = arith.constant dense<4.096000e+03> : tensor<1x32xf32> - %11 = tensor.empty() : tensor<1x32xf32> - %12 = linalg.div ins(%reduced, %cst_2 : tensor<1x32xf32>, tensor<1x32xf32>) outs(%11 : tensor<1x32xf32>) -> tensor<1x32xf32> - %expanded_3 = tensor.expand_shape %12 [[0], [1, 2]] output_shape [1, 32, 1] : tensor<1x32xf32> into tensor<1x32x1xf32> - %13 = tensor.empty() : tensor<1x32x1xf32> - %broadcasted = linalg.broadcast ins(%arg8 : tensor<1xf32>) outs(%13 : tensor<1x32x1xf32>) dimensions = [0, 1] - %14 = tensor.empty() : tensor<1x32x1xf32> - %15 = linalg.add ins(%expanded_3, %broadcasted : tensor<1x32x1xf32>, tensor<1x32x1xf32>) outs(%14 : tensor<1x32x1xf32>) -> tensor<1x32x1xf32> - %cst_4 = arith.constant dense<-5.000000e-01> : tensor<1x32x1xf32> - %16 = tensor.empty() : tensor<1x32x1xf32> - %17 = linalg.powf ins(%15, %cst_4 : tensor<1x32x1xf32>, tensor<1x32x1xf32>) outs(%16 : tensor<1x32x1xf32>) -> tensor<1x32x1xf32> - %collapsed_5 = tensor.collapse_shape %17 [[0], [1, 2]] : tensor<1x32x1xf32> into tensor<1x32xf32> - %18 = tensor.empty() : tensor<1x32x4096xf32> - %broadcasted_6 = linalg.broadcast ins(%collapsed_5 : tensor<1x32xf32>) outs(%18 : tensor<1x32x4096xf32>) dimensions = [2] - %19 = tensor.empty() : tensor<1x32x4096xf32> - %20 = linalg.mul ins(%6, %broadcasted_6 : tensor<1x32x4096xf32>, tensor<1x32x4096xf32>) outs(%19 : tensor<1x32x4096xf32>) -> tensor<1x32x4096xf32> - %21 = tensor.empty() : tensor<1x32x4096xbf16> - %22 = linalg.copy ins(%20 : tensor<1x32x4096xf32>) outs(%21 : tensor<1x32x4096xbf16>) -> tensor<1x32x4096xbf16> - %23 = tensor.empty() : tensor<1x32x4096xbf16> - %broadcasted_7 = linalg.broadcast ins(%arg4 : tensor<4096xbf16>) outs(%23 : tensor<1x32x4096xbf16>) dimensions = [0, 1] - %24 = tensor.empty() : tensor<1x32x4096xbf16> - %25 = linalg.mul ins(%broadcasted_7, %22 : tensor<1x32x4096xbf16>, tensor<1x32x4096xbf16>) outs(%24 : tensor<1x32x4096xbf16>) -> tensor<1x32x4096xbf16> - %collapsed_8 = tensor.collapse_shape %25 [[0, 1], [2]] : tensor<1x32x4096xbf16> into tensor<32x4096xbf16> - %cst_9 = arith.constant 0.000000e+00 : bf16 - %26 = tensor.empty() : tensor<32x11008xbf16> - %27 = linalg.fill ins(%cst_9 : bf16) outs(%26 : tensor<32x11008xbf16>) -> tensor<32x11008xbf16> - %28 = linalg.matmul_transpose_b ins(%collapsed_8, %arg5 : tensor<32x4096xbf16>, tensor<11008x4096xbf16>) outs(%27 : tensor<32x11008xbf16>) -> tensor<32x11008xbf16> - %expanded_10 = tensor.expand_shape %28 [[0, 1], [2]] output_shape [1, 32, 11008] : tensor<32x11008xbf16> into tensor<1x32x11008xbf16> - %29 = tensor.empty() : tensor<1x32x11008xbf16> - %cst_11 = arith.constant dense<1.000000e+00> : tensor<1x32x11008xbf16> - %30 = linalg.negf ins(%expanded_10 : tensor<1x32x11008xbf16>) outs(%29 : tensor<1x32x11008xbf16>) -> tensor<1x32x11008xbf16> - %31 = linalg.exp ins(%30 : tensor<1x32x11008xbf16>) outs(%29 : tensor<1x32x11008xbf16>) -> tensor<1x32x11008xbf16> - %32 = linalg.add ins(%cst_11, %31 : tensor<1x32x11008xbf16>, tensor<1x32x11008xbf16>) outs(%29 : tensor<1x32x11008xbf16>) -> tensor<1x32x11008xbf16> - %33 = linalg.div ins(%cst_11, %32 : tensor<1x32x11008xbf16>, tensor<1x32x11008xbf16>) outs(%29 : tensor<1x32x11008xbf16>) -> tensor<1x32x11008xbf16> - %34 = tensor.empty() : tensor<1x32x11008xbf16> - %35 = linalg.mul ins(%33, %expanded_10 : tensor<1x32x11008xbf16>, tensor<1x32x11008xbf16>) outs(%34 : tensor<1x32x11008xbf16>) -> tensor<1x32x11008xbf16> - %collapsed_12 = tensor.collapse_shape %25 [[0, 1], [2]] : tensor<1x32x4096xbf16> into tensor<32x4096xbf16> - %cst_13 = arith.constant 0.000000e+00 : bf16 - %36 = tensor.empty() : tensor<32x11008xbf16> - %37 = linalg.fill ins(%cst_13 : bf16) outs(%36 : tensor<32x11008xbf16>) -> tensor<32x11008xbf16> - %38 = linalg.matmul_transpose_b ins(%collapsed_12, %arg6 : tensor<32x4096xbf16>, tensor<11008x4096xbf16>) outs(%37 : tensor<32x11008xbf16>) -> tensor<32x11008xbf16> - %expanded_14 = tensor.expand_shape %38 [[0, 1], [2]] output_shape [1, 32, 11008] : tensor<32x11008xbf16> into tensor<1x32x11008xbf16> - %39 = tensor.empty() : tensor<1x32x11008xbf16> - %40 = linalg.mul ins(%35, %expanded_14 : tensor<1x32x11008xbf16>, tensor<1x32x11008xbf16>) outs(%39 : tensor<1x32x11008xbf16>) -> tensor<1x32x11008xbf16> - %collapsed_15 = tensor.collapse_shape %40 [[0, 1], [2]] : tensor<1x32x11008xbf16> into tensor<32x11008xbf16> - %cst_16 = arith.constant 0.000000e+00 : bf16 - %41 = tensor.empty() : tensor<32x4096xbf16> - %42 = linalg.fill ins(%cst_16 : bf16) outs(%41 : tensor<32x4096xbf16>) -> tensor<32x4096xbf16> - %43 = linalg.matmul_transpose_b ins(%collapsed_15, %arg7 : tensor<32x11008xbf16>, tensor<4096x11008xbf16>) outs(%42 : tensor<32x4096xbf16>) -> tensor<32x4096xbf16> - %expanded_17 = tensor.expand_shape %43 [[0, 1], [2]] output_shape [1, 32, 4096] : tensor<32x4096xbf16> into tensor<1x32x4096xbf16> - %44 = tensor.empty() : tensor<1x32x4096xbf16> - %45 = linalg.add ins(%4, %expanded_17 : tensor<1x32x4096xbf16>, tensor<1x32x4096xbf16>) outs(%44 : tensor<1x32x4096xbf16>) -> tensor<1x32x4096xbf16> - %46 = tensor.empty() : tensor<1x32x4096xf32> - %47 = linalg.copy ins(%45 : tensor<1x32x4096xbf16>) outs(%46 : tensor<1x32x4096xf32>) -> tensor<1x32x4096xf32> - %cst_18 = arith.constant dense<2.000000e+00> : tensor<1x32x4096xf32> - %48 = tensor.empty() : tensor<1x32x4096xf32> - %49 = linalg.powf ins(%47, %cst_18 : tensor<1x32x4096xf32>, tensor<1x32x4096xf32>) outs(%48 : tensor<1x32x4096xf32>) -> tensor<1x32x4096xf32> - %cst_19 = arith.constant 0.000000e+00 : f32 - %50 = tensor.empty() : tensor<1x32xf32> - %51 = linalg.fill ins(%cst_19 : f32) outs(%50 : tensor<1x32xf32>) -> tensor<1x32xf32> - %reduced_20 = linalg.reduce ins(%49 : tensor<1x32x4096xf32>) outs(%51 : tensor<1x32xf32>) dimensions = [2] - (%in: f32, %init: f32) { - %67 = arith.addf %in, %init : f32 - linalg.yield %67 : f32 - } - %cst_21 = arith.constant dense<4.096000e+03> : tensor<1x32xf32> - %52 = tensor.empty() : tensor<1x32xf32> - %53 = linalg.div ins(%reduced_20, %cst_21 : tensor<1x32xf32>, tensor<1x32xf32>) outs(%52 : tensor<1x32xf32>) -> tensor<1x32xf32> - %expanded_22 = tensor.expand_shape %53 [[0], [1, 2]] output_shape [1, 32, 1] : tensor<1x32xf32> into tensor<1x32x1xf32> - %54 = tensor.empty() : tensor<1x32x1xf32> - %broadcasted_23 = linalg.broadcast ins(%arg8 : tensor<1xf32>) outs(%54 : tensor<1x32x1xf32>) dimensions = [0, 1] - %55 = tensor.empty() : tensor<1x32x1xf32> - %56 = linalg.add ins(%expanded_22, %broadcasted_23 : tensor<1x32x1xf32>, tensor<1x32x1xf32>) outs(%55 : tensor<1x32x1xf32>) -> tensor<1x32x1xf32> - %cst_24 = arith.constant dense<-5.000000e-01> : tensor<1x32x1xf32> - %57 = tensor.empty() : tensor<1x32x1xf32> - %58 = linalg.powf ins(%56, %cst_24 : tensor<1x32x1xf32>, tensor<1x32x1xf32>) outs(%57 : tensor<1x32x1xf32>) -> tensor<1x32x1xf32> - %collapsed_25 = tensor.collapse_shape %58 [[0], [1, 2]] : tensor<1x32x1xf32> into tensor<1x32xf32> - %59 = tensor.empty() : tensor<1x32x4096xf32> - %broadcasted_26 = linalg.broadcast ins(%collapsed_25 : tensor<1x32xf32>) outs(%59 : tensor<1x32x4096xf32>) dimensions = [2] - %60 = tensor.empty() : tensor<1x32x4096xf32> - %61 = linalg.mul ins(%47, %broadcasted_26 : tensor<1x32x4096xf32>, tensor<1x32x4096xf32>) outs(%60 : tensor<1x32x4096xf32>) -> tensor<1x32x4096xf32> - %62 = tensor.empty() : tensor<1x32x4096xbf16> - %63 = linalg.copy ins(%61 : tensor<1x32x4096xf32>) outs(%62 : tensor<1x32x4096xbf16>) -> tensor<1x32x4096xbf16> - %64 = tensor.empty() : tensor<1x32x4096xbf16> - %broadcasted_27 = linalg.broadcast ins(%arg9 : tensor<4096xbf16>) outs(%64 : tensor<1x32x4096xbf16>) dimensions = [0, 1] - %65 = tensor.empty() : tensor<1x32x4096xbf16> - %66 = linalg.mul ins(%broadcasted_27, %63 : tensor<1x32x4096xbf16>, tensor<1x32x4096xbf16>) outs(%65 : tensor<1x32x4096xbf16>) -> tensor<1x32x4096xbf16> - return %66, %45 : tensor<1x32x4096xbf16>, tensor<1x32x4096xbf16> - } -} \ No newline at end of file diff --git a/test/benchgc/setup.py b/test/benchgc/setup.py deleted file mode 100644 index a9b49e6f0..000000000 --- a/test/benchgc/setup.py +++ /dev/null @@ -1,30 +0,0 @@ -################################################################################ -# Copyright 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -import setuptools - -setuptools.setup( - name="benchgc", - description="benchmark tool for graph compiler", - package_dir={ - "benchgc": "src/benchgc", - "gc_mlir": "../../python_packages/gc_mlir_core/gc_mlir", - }, - packages=setuptools.find_packages("src") - + setuptools.find_namespace_packages("../../python_packages/gc_mlir_core"), - package_data={"gc_mlir": ["_mlir_libs/*.so"]}, - install_requires=["torch", "numpy"], -) diff --git a/test/benchgc/src/benchgc/CMakeLists.txt b/test/benchgc/src/benchgc/CMakeLists.txt deleted file mode 100644 index 5700c4f36..000000000 --- a/test/benchgc/src/benchgc/CMakeLists.txt +++ /dev/null @@ -1,22 +0,0 @@ -################################################################################ -# Copyright (C) 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions -# and limitations under the License. -# SPDX-License-Identifier: Apache-2.0 -################################################################################ - - -file(GLOB PYTHON_SCRIPTS "*.py") -foreach(PY_SCRIPT ${PYTHON_SCRIPTS}) - configure_file(${PY_SCRIPT} ${CMAKE_BINARY_DIR}/test/benchgc/src/benchgc/ COPYONLY) -endforeach() diff --git a/test/benchgc/src/benchgc/__init__.py b/test/benchgc/src/benchgc/__init__.py deleted file mode 100644 index 3b87b7051..000000000 --- a/test/benchgc/src/benchgc/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -################################################################################ -# Copyright 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -import pathlib -import sys - -sys.path.append(pathlib.Path(__file__).parent.resolve().__str__()) diff --git a/test/benchgc/src/benchgc/__main__.py b/test/benchgc/src/benchgc/__main__.py deleted file mode 100644 index 379078a9d..000000000 --- a/test/benchgc/src/benchgc/__main__.py +++ /dev/null @@ -1,407 +0,0 @@ -################################################################################ -# Copyright 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - - -import argparse -import json -import sys -from typing import Dict, List - -import benchgc.mlir.util -import benchgc.util -import runner -import torch -from benchgc.arg import ( - compare_tensor, - fill_tensor, - set_default_compare, - set_default_fill, -) -from benchgc.arg.arg import Arg -from benchgc.bench import mlir_wrapper_bench, py_timeit_bench -from benchgc.mlir.arg import get_mlir_args -from benchgc.pattern import get_pattern_clz -from gc_mlir import ir -from gc_mlir.graph_compiler import GraphCompiler - - -def add_common_options(parser: argparse.ArgumentParser): - """common options for benchgc""" - parser.add_argument( - "--mode", - required=False, - help="specify the test mode, C for correctness testing, P for performance testing", - choices=["C", "P"], - default="C", - type=str, - ) - parser.add_argument( - "--driver", - required=False, - help="specify the test driver", - choices=["linalg", "tensor", "mlir", "pattern"], - type=str, - ) - parser.add_argument( - "--case", - required=False, - help="test which operation in the specified driver", - type=str, - ) - - parser.add_argument( - "--md", - required=False, - help="format: #ARG:SHAPExTYPE", - type=str, - default=[], - action="append", - ) - parser.add_argument( - "--fill", - required=False, - help="format: #ARG:type:parameter", - type=str, - default=[], - action="append", - ) - parser.add_argument( - "--cmp", - required=False, - help="format: #ARG:type:parameter", - type=str, - default=[], - action="append", - ) - - parser.add_argument( - "--seed", - required=False, - default=0, - type=int, - help="a seed value to generate data filling", - ) - parser.add_argument( - "--verbose", - type=int, - default=benchgc.util.NO_VERBOSE, - help="verbose level", - choices=[ - benchgc.util.NO_VERBOSE, - benchgc.util.MODULE_VERBOSE, - benchgc.util.ARG_VERBOSE, - benchgc.util.COMPARE_VERBOSE, - benchgc.util.ERROR_OUTPUT_VERBOSE, - benchgc.util.OUTPUT_VERBOSE, - benchgc.util.INPUT_VERBOSE, - ], - ) - - parser.add_argument( - "--entry", - required=False, - default="entry", - help="the entry func name of a mlir", - type=str, - ) - - parser.add_argument( - "--ir_printing", - action="store_true", - help="if we need print the ir during the pass-pipeline", - ) - - parser.add_argument( - "--cpu_cache_sizes", - required=False, - help="set the cpu cache sizes, format: L1:L2:L3", - type=str, - ) - - parser.add_argument( - "--max_vector_width", - required=False, - help="set the cpu max_vector_width", - type=int, - ) - - if parser.parse_known_args()[0].driver == "linalg": - parser.add_argument( - "--cast", - required=False, - default="cast_signed", - help="define attribute supported by linalg op such as matmul_transpose_b", - choices=["cast_signed", "cast_unsigned"], - type=str, - ) - - # single dimension index - # linalg.softmax - parser.add_argument( - "--dimension", - required=False, - default=None, - help="define the dimension attribute in linalg op", - type=int, - ) - - # multiple dimensions array - # linalg.broadcast / linalg.reduce - parser.add_argument( - "--dimensions", - required=False, - default=None, - action="append", - help="define the dimensions attribute in linalg op", - type=int, - ) - - parser.add_argument( - "--dilations", - required=False, - default=None, - action="append", - help="define the dilations attribute in linalg op", - type=int, - ) - - parser.add_argument( - "--strides", - required=False, - default=None, - action="append", - help="define the strides attribute in linalg op", - type=int, - ) - - parser.add_argument( - "--permutation", - required=False, - default=None, - action="append", - help="define the permutation attribute in linalg op", - type=int, - ) - -def add_bench_options(parser: argparse.ArgumentParser): - """add options for bench mode""" - if parser.parse_known_args()[0].mode == "P": - parser.add_argument( - "--bench_kind", type=str, choices=["py", "wrapper"], default="py" - ) - parser.add_argument("--warm_up", type=int, default=100) - parser.add_argument("--repeat", type=int, default=100) - - - -def add_pattern_options(parser: argparse.ArgumentParser): - """add options for each pattern""" - if parser.parse_known_args()[0].driver == "pattern": - pattern_name = parser.parse_known_args()[0].case - get_pattern_clz(pattern_name).add_args(parser) - - -def get_module_and_args(flags: argparse.Namespace): - args: List[Arg] = [] - if flags.driver in ["mlir", "pattern"]: - # we need to find all args by reading the entry function - with ir.Context() as ctx: - if flags.driver == "mlir": - with open(flags.case, "r") as mlir_file: - module = ir.Module.parse(mlir_file.read()) - elif flags.driver == "pattern": - pattern_clz = get_pattern_clz(flags.case) - module = pattern_clz(ctx, flags).ir_module - else: - raise Exception("unexpected error") - - entry = benchgc.mlir.util.get_kernel_func_from_module(module, flags.entry) - idx: int = 0 - # FIXME: only support RankTensorType now - for i in entry.type.inputs: - args.append(Arg(idx)) - args[-1].dtype = str(i.element_type) - args[-1].shape = list(i.shape) - args[-1].set_scalar() - idx += 1 - - for o in entry.type.results: - args.append(Arg(idx)) - args[-1].dtype = str(o.element_type) - args[-1].shape = list(o.shape) - args[-1].set_scalar() - idx += 1 - - elif flags.driver in ["linalg"]: - # all arg shape/dt should be provided in single op test - for i in range(len(flags.md)): - args.append(Arg(i)) - - for md in flags.md: - colon = md.find(":") - if colon == -1: - raise Exception("Wrong md format: %s", md) - idx = int(md[:colon]) - args[idx].set_md(md[colon + 1 :]) - - from .linalg import mlir_op - - if flags.case.startswith("reduce."): - mlir_func = mlir_op["reduce"] - else: - mlir_func = mlir_op[flags.case] - module = mlir_func(flags, args) - else: - raise Exception(f"unsupported driver {flags.driver}") - - for fill in flags.fill: - colon = fill.find(":") - if colon == -1: - raise Exception("Wrong fill format: %s", fill) - idx = int(fill[:colon]) - args[idx].set_fill(fill[colon + 1 :]) - - for cmp in flags.cmp: - colon = cmp.find(":") - if colon == -1: - raise Exception("Wrong cmp format: %s", cmp) - idx = int(cmp[:colon]) - args[idx].set_cmp(cmp[colon + 1 :]) - - entry = benchgc.mlir.util.get_kernel_func_from_module(module, flags.entry) - - with module.context: - entry.attributes["llvm.emit_c_interface"] = ir.UnitAttr.get() - - for i, arg in enumerate(args): - # use zero filling if the arg is return value - set_default_fill(flags, arg, args, i >= len(entry.type.inputs)) - set_default_compare(flags, arg, args, i >= len(entry.type.inputs)) - - for arg in args: - arg.print_verbose(flags.verbose) - - benchgc.mlir.util.attach_dlti(flags, module) - - if flags.verbose >= benchgc.util.MODULE_VERBOSE: - print(module) - return module, args - - -def correctness_testing(flags: argparse.Namespace, module: ir.Module, args: List[Arg]): - ref_args: List[torch.Tensor] = [] - gc_args: List[torch.Tensor | int] = [] - ref_tensors: Dict[str, torch.Tensor] = {} - gc_tensors: Dict[str, torch.Tensor] = {} - - for i in range(len(args)): - tensor = fill_tensor(flags, args[i], i) - gc_tensors["%arg" + str(i)] = tensor - ref_tensors["%arg" + str(i)] = tensor.clone() - ref_args.append(ref_tensors["%arg" + str(i)]) - if args[i].scalar: - gc_args.append(tensor.data_ptr()) - else: - gc_args.append(tensor) - - entry = benchgc.mlir.util.get_kernel_func_from_module(module, flags.entry) - # ref_out contains return value of the entry - ref_out = runner.ref_run(entry, ref_tensors) - - # we need to swap the result into the args if some arg is the return value - for i in range(len(ref_out)): - ref_args[0 - i - 1] = ref_out[0 - i - 1] - - mlir_args = get_mlir_args(gc_args) - passes = "any(gc-cpu-pipeline)" - - with module.context as ctx: - if flags.ir_printing: - ctx.enable_multithreading(False) - compiler = GraphCompiler(passes) - engine = compiler.compile_and_jit(module, flags.ir_printing) - engine.invoke(flags.entry, *mlir_args) - - fail, mistrust = False, False - for i in range(len(args)): - # gc_arg contains address for scalar value - # we need to find result by arg name - res = compare_tensor( - args[i], ref_args[i], gc_tensors["%arg" + str(i)], flags.verbose - ) - fail = fail or (not res[0]) - if res[1] is not None: - mistrust = mistrust | res[1] - if fail: - print(f"FAIL: {flags.driver}.{flags.case}") - sys.exit(1) - elif mistrust: - print(f"MISTRUST: {flags.driver}.{flags.case}") - else: - print(f"PASSED: {flags.driver}.{flags.case}") - - -def performance_testing(flags: argparse.Namespace, module: ir.Module, args: List[Arg]): - gc_args: List[torch.Tensor | int] = [] - gc_tensors: Dict[str, torch.Tensor] = {} - for i in range(len(args)): - tensor = fill_tensor(flags, args[i], i) - gc_tensors["%arg" + str(i)] = tensor - if args[i].scalar: - gc_args.append(tensor.data_ptr()) - else: - gc_args.append(tensor) - - mlir_args = get_mlir_args(gc_args) - with module.context as ctx, ir.Location.unknown(): - if flags.ir_printing: - ctx.enable_multithreading(False) - bench_kind = py_timeit_bench if flags.bench_kind == "py" else mlir_wrapper_bench - execute_cost, compile_cost = bench_kind( - module, - flags.entry, - "any(gc-cpu-pipeline)", - mlir_args, - flags.ir_printing, - flags.repeat, - flags.warm_up, - ) - print("===========bench result===========") - json_res = json.dumps( - { - "args": vars(flags), - "compile_cost(ms)": compile_cost, - "execute_cost(ms)": execute_cost, - }, - indent=4, - ) - print(json_res) - - -if __name__ == "__main__": - arg_parser = argparse.ArgumentParser(prog="benchmark tool for graph compiler") - add_common_options(arg_parser) - add_bench_options(arg_parser) - add_pattern_options(arg_parser) - flags = arg_parser.parse_args() - benchgc.util.set_seed(flags.seed) - ir_module, module_args = get_module_and_args(flags) - if flags.mode == "C": - correctness_testing(flags, ir_module, module_args) - elif flags.mode == "P": - performance_testing(flags, ir_module, module_args) - else: - pass diff --git a/test/benchgc/src/benchgc/arg/CMakeLists.txt b/test/benchgc/src/benchgc/arg/CMakeLists.txt deleted file mode 100644 index 614e306da..000000000 --- a/test/benchgc/src/benchgc/arg/CMakeLists.txt +++ /dev/null @@ -1,22 +0,0 @@ -################################################################################ -# Copyright (C) 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions -# and limitations under the License. -# SPDX-License-Identifier: Apache-2.0 -################################################################################ - - -file(GLOB PYTHON_SCRIPTS "*.py") -foreach(PY_SCRIPT ${PYTHON_SCRIPTS}) - configure_file(${PY_SCRIPT} ${CMAKE_BINARY_DIR}/test/benchgc/src/benchgc/arg/ COPYONLY) -endforeach() diff --git a/test/benchgc/src/benchgc/arg/__init__.py b/test/benchgc/src/benchgc/arg/__init__.py deleted file mode 100644 index 0cd1dcba4..000000000 --- a/test/benchgc/src/benchgc/arg/__init__.py +++ /dev/null @@ -1,172 +0,0 @@ -################################################################################ -# Copyright 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -import argparse -from typing import List, Tuple - -import benchgc.arg.binary as binary -import benchgc.arg.compare -import benchgc.arg.conv as conv -import benchgc.arg.eltwise as eltwise -import benchgc.arg.matmul as matmul -import benchgc.arg.pool as pool -import benchgc.arg.reduce as reduce -import benchgc.arg.softmax as softmax -import benchgc.util -import torch -from benchgc.arg.arg import Arg -from benchgc.pattern import get_pattern_clz - -onednn_module = { - "binary": binary, - "eltwise": eltwise, - "matmul": matmul, - "softmax": softmax, - "conv": conv, - "pool": pool, - "reduce": reduce, -} - - -def set_default_fill( - flags: argparse.Namespace, arg: Arg, arglist: List[Arg], is_return: bool -): - if arg.fill_type != "-": - return - - if is_return: - arg.fill_type = "Z" - arg.fill_param = [] - return - - for _, module in onednn_module.items(): - if flags.driver + "." + flags.case in module.op: - module.default_fill(flags, arg, arglist) - return - elif flags.driver == "pattern": - get_pattern_clz(flags.case).default_fill(flags, arg, arglist) - return - # use N(0, 1) as default - arg.fill_type = "N" - arg.fill_param = ["0", "1"] - - -def set_default_compare( - flags: argparse.Namespace, arg: Arg, arglist: List[Arg], is_return: bool -): - if arg.cmp_type != "-": - return - - if is_return: - for _, module in onednn_module.items(): - if flags.driver + "." + flags.case in module.op: - module.default_compare(flags, arg, arglist) - return - elif flags.driver == "pattern": - get_pattern_clz(flags.case).default_compare(flags, arg, arglist) - return - - dtype: torch.dtype = benchgc.util.get_dtype(arg.dtype) - arg.cmp_type = "P" - if dtype.is_floating_point: - arg.cmp_param = [str(1e-05)] - else: - arg.cmp_param = ["0"] - if is_return: - arg.cmp_param.append("70.0") - else: - arg.cmp_param.append("100.0") - - -def fill_tensor(flags: argparse.Namespace, arg: Arg, idx: int) -> torch.Tensor: - if arg.dtype == "" or arg.fill_type == "": - raise Exception("arg%d filling: dtype/fill_type is not set" % idx) - - # set the seed for the filling - benchgc.util.torch_seed(1, idx) - if arg.fill_type == "N" and len(arg.fill_param) == 2: - # Normal distribution - mean = float(arg.fill_param[0]) - std = float(arg.fill_param[1]) - tensor = torch.normal(mean=mean, std=std, size=arg.shape) - - elif arg.fill_type == "P" and len(arg.fill_param) == 1: - # Poisson distribution - _lambda = float(arg.fill_param[0]) - lambda_tensor = torch.full(arg.shape, _lambda) - tensor = torch.poisson(lambda_tensor) - elif arg.fill_type == "B" and len(arg.fill_param) == 2: - # Binomial distribution - n = int(arg.fill_param[0]) - p = float(arg.fill_param[1]) - bdist = torch.distributions.binomial.Binomial(total_count=n, probs=p) - tensor = bdist.sample(torch.Size(arg.shape)) - elif arg.fill_type == "U" and len(arg.fill_param) == 2: - # Uniform distribution - a = float(arg.fill_param[0]) - b = float(arg.fill_param[1]) - tensor = torch.distributions.uniform.Uniform(a, b).sample(torch.Size(arg.shape)) - elif arg.fill_type == "I" and len(arg.fill_param) == 2: - # integer range - a = int(arg.fill_param[0]) - b = int(arg.fill_param[1]) - tensor = torch.randint(a, b + 1, torch.Size(arg.shape)) - elif arg.fill_type == "F" and len(arg.fill_param) == 1: - # read from pytorch tensor dump file - filename = arg.fill_param[0] - tensor = torch.load(f=filename) - if not isinstance(tensor, torch.Tensor): - raise Exception(f"torch object from file {filename} is not a tensor object") - if tensor.shape != torch.Size(arg.shape): - raise Exception(f"tensor object from file {filename} does not match shape") - if tensor.dtype != benchgc.util.get_dtype(arg.dtype): - raise Exception(f"tensor object from file {filename} does not match dtype") - elif arg.fill_type == "D" and len(arg.fill_param) > 0: - # Driver fill - driver: str = arg.fill_param[0] - driver_module = onednn_module[driver] - tensor = driver_module.fill( - arg.shape, benchgc.util.get_dtype(arg.dtype), arg.fill_param[1:] - ) - elif arg.fill_type == "Z": - tensor = torch.zeros(size=arg.shape) - else: - raise Exception("invalid fill type or fill parameter") - - tensor = tensor.to(benchgc.util.get_dtype(arg.dtype)) - if flags.verbose >= benchgc.util.INPUT_VERBOSE: - print("fill arg%d: " % idx) - print(tensor) - return tensor - - -def compare_tensor( - arg: Arg, ref: torch.Tensor, res: torch.Tensor, verbose: int -) -> Tuple[bool, bool | None]: - - if arg.cmp_type == "P": # p2p check - threshold = float(arg.cmp_param[0]) - zero_percent = float(arg.cmp_param[1]) - return benchgc.arg.compare.p2p(threshold, zero_percent, ref, res, verbose) - if arg.cmp_type == "N": # norm check - threshold = float(arg.cmp_param[0]) - return benchgc.arg.compare.norm(threshold, ref, res, verbose) - elif arg.cmp_type == "D" and len(arg.cmp_param) > 0: # driver check - driver: str = arg.cmp_param[0] - driver_module = onednn_module[driver] - return driver_module.compare(arg.cmp_param[1:], ref, res, verbose) - else: - raise Exception("invalid compare type or compare parameter") diff --git a/test/benchgc/src/benchgc/arg/arg.py b/test/benchgc/src/benchgc/arg/arg.py deleted file mode 100644 index 3bf232a93..000000000 --- a/test/benchgc/src/benchgc/arg/arg.py +++ /dev/null @@ -1,54 +0,0 @@ -################################################################################ -# Copyright 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -from typing import List - -import benchgc.mlir.arg -import benchgc.util - - -class Arg(benchgc.mlir.arg.MLIRArg): - fill_type: str - fill_param: List[str] - - cmp_type: str - cmp_param: List[str] - - index: int - - def __init__(self, index: int): - self.dtype = "" - self.fill_type = "-" - self.fill_param = [] - self.cmp_type = "-" - self.cmp_param = [] - self.index = index - - def print_verbose(self, verbose: int): - if verbose >= benchgc.util.ARG_VERBOSE: - print( - f"arg{self.index} shape: {self.shape} dtype: {self.dtype} fill_type: {self.fill_type} fill_param: {self.fill_param} cmp_type: {self.cmp_type} cmp_param: {self.cmp_param}" - ) - - def set_fill(self, fill: str): - splited: List[str] = fill.split(":") - self.fill_type = splited[0] - self.fill_param = splited[1:] - - def set_cmp(self, cmp: str): - splited: List[str] = cmp.split(":") - self.cmp_type = splited[0] - self.cmp_param = splited[1:] diff --git a/test/benchgc/src/benchgc/arg/binary.py b/test/benchgc/src/benchgc/arg/binary.py deleted file mode 100644 index 6e2cb0c30..000000000 --- a/test/benchgc/src/benchgc/arg/binary.py +++ /dev/null @@ -1,98 +0,0 @@ -################################################################################ -# Copyright 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -import argparse -from typing import Dict, List, Set, Tuple - -import benchgc.util -import torch -from benchgc.arg.arg import Arg -from benchgc.arg.compare import p2p - -# op should use this filling - -op: Set[str] = set( - ["linalg.add", "linalg.div", "linalg.mul", "linalg.max", "linalg.min", "linalg.sub"] -) - -# params format: [src0 | src1, src0 dt, src1 dt, dst dt] - - -def default_fill( - flags: argparse.Namespace, - arg: Arg, - arglist: List[Arg], -): - if arg.index > 1: - raise Exception("binary fill: dst filling is not allowed") - arg.fill_type = "D" - arg.fill_param = [ - "binary", - "src0" if arg.index == 0 else "src1", - arglist[0].dtype, - arglist[1].dtype, - arglist[2].dtype, - ] - - -def fill(shape: List[int], dtype: torch.dtype, params: List[str]) -> torch.Tensor: - name, _, _, _ = params - - accept_name: Dict[str, int] = {"src0": 1, "src1": 2} - if name in accept_name: - arg: int = accept_name[name] - else: - raise Exception("unknown arg name %s", name) - - range_: int = 16 - f_min = 0 if dtype == torch.uint8 else -range_ // 2 - - idx: torch.Tensor = torch.arange( - benchgc.util.nelem(shape), dtype=torch.int - ).reshape(shape) - values: torch.Tensor = (f_min + (12 * idx + 5 * arg + 16) % (range_ + 1)) * 1.25 - if arg == 2: - values = torch.where(values == 0.0, 1, values) - return values.to(dtype=dtype) - - -# compare param: dtype, case - - -def default_compare( - flags: argparse.Namespace, - arg: Arg, - arglist: List[Arg], -): - arg.cmp_type = "D" - arg.cmp_param = ["binary", arg.dtype, flags.case] - - -def compare( - param: List[str], ref: torch.Tensor, res: torch.Tensor, verbose: int -) -> Tuple[bool, bool | None]: - dtype = benchgc.util.get_dtype(param[0]) - - ref = ref.to(torch.float) - res = res.to(torch.float) - - if param[1] in ["div", "div_unsigned"]: - abs_diff = (ref.to(torch.float) - res.to(torch.float)).abs() - init_check = abs_diff < benchgc.util.get_eps(dtype) - else: - init_check = None - - return p2p(benchgc.util.get_eps(dtype), 30.0, ref, res, verbose, init_check) diff --git a/test/benchgc/src/benchgc/arg/compare.py b/test/benchgc/src/benchgc/arg/compare.py deleted file mode 100644 index 2e4c31e85..000000000 --- a/test/benchgc/src/benchgc/arg/compare.py +++ /dev/null @@ -1,137 +0,0 @@ -################################################################################ -# Copyright 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -from typing import Callable, List, Tuple - -import benchgc.util -import numpy -import torch - - -def iterate_tensor(tensor: torch.Tensor, fn: Callable[[Tuple[int, ...]], None]): - if tensor.ndim == 0: - fn(tuple()) - return - index: List[int] = [0] * tensor.ndim - - def dfs(depth: int): - if depth == tensor.ndim: - fn(tuple(index)) - else: - for i in range(tensor.shape[depth]): - index[depth] = i - dfs(depth + 1) - - dfs(0) - - -def norm( - threshold: float, ref: torch.Tensor, res: torch.Tensor, verbose: int -) -> Tuple[bool, bool | None]: - - f32_ref = ref.to(torch.float32) - f32_res = res.to(torch.float32) - if f32_ref.nelement() == 0: - return (True, None) - - diff_square_sum = torch.square(torch.subtract(f32_ref, f32_res)).sum() - square_sum = torch.square(f32_ref).sum() - - l2_diff_norm = torch.sqrt(diff_square_sum / square_sum).item() - if verbose >= benchgc.util.COMPARE_VERBOSE: - print(f"norm check: {l2_diff_norm:.10f} / threshold: {threshold:.10f}") - - return (l2_diff_norm < threshold, None) - - -def p2p( - threshold: float, - zero_percent: float, - ref: torch.Tensor, - res: torch.Tensor, - verbose: int, - init_check: torch.Tensor | None = None, -) -> Tuple[bool, bool | None]: - - if verbose >= benchgc.util.COMPARE_VERBOSE: - print(f"p2p check: threshold: {threshold:.7f}") - f32_ref = ref.to(torch.float32) - f32_res = res.to(torch.float32) - - if init_check is None: - check = torch.tensor(False) - else: - check = init_check - - check = check.bitwise_or(torch.bitwise_and(f32_ref.isnan(), f32_res.isnan())) - check = check.bitwise_or(torch.bitwise_and(f32_ref.isneginf(), f32_res.isneginf())) - check = check.bitwise_or(torch.bitwise_and(f32_ref.isposinf(), f32_res.isposinf())) - - # choose diff/rel_diff based on value - abs_diff = (f32_ref - f32_res).abs() - rel_diff = abs_diff / torch.where( - f32_ref.abs() > numpy.finfo(numpy.float32).smallest_subnormal, - f32_ref.abs(), - 1, - ) - # pick a diff for comparison - diff = torch.where(f32_ref.abs() > 1e-5, rel_diff, abs_diff) - - check = check.bitwise_or(diff <= threshold) - - if verbose >= benchgc.util.OUTPUT_VERBOSE: - iterate_tensor( - check, - lambda idx: print( - "%20s: ref: %12.7f res: %12.7f abs_diff: %12.7f rel_diff: %12.7f" - % ( - idx, - f32_ref[idx].item(), - f32_res[idx].item(), - abs_diff[idx].item(), - rel_diff[idx].item(), - ) - ), - ) - if check.all(): - # check mistrusted - zero = res.nelement() - res.count_nonzero().item() - if res.nelement() < 10: - mistrust = False - else: - mistrust = zero * 100.0 / res.nelement() > zero_percent - return (True, mistrust) - else: - if ( - verbose < benchgc.util.OUTPUT_VERBOSE - ): # skip verbose print if full output tensor is alrady printed - fail = torch.argwhere(torch.where(check, 0, 1)) - if verbose < benchgc.util.ERROR_OUTPUT_VERBOSE: - # only print top 10 failed data points if verbose level does not satisfied - fail = fail[:10] - for idx in fail: - index: Tuple[int, ...] = tuple(idx.tolist()) - print( - "%20s: ref: %12.7f res: %12.7f abs_diff: %12.7f rel_diff: %12.7f" - % ( - index, - f32_ref[index].item(), - f32_res[index].item(), - abs_diff[index].item(), - rel_diff[index].item(), - ) - ) - return (False, None) diff --git a/test/benchgc/src/benchgc/arg/conv.py b/test/benchgc/src/benchgc/arg/conv.py deleted file mode 100644 index ba46f201c..000000000 --- a/test/benchgc/src/benchgc/arg/conv.py +++ /dev/null @@ -1,189 +0,0 @@ -################################################################################ -# Copyright 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -import argparse -from typing import Dict, List, Set, Tuple - -import benchgc.util -import torch -from benchgc.arg.arg import Arg -from benchgc.arg.compare import p2p - -op: Set[str] = set( - [ - "linalg.conv_1d_ncw_fcw", - "linalg.conv_1d_nwc_wcf", - "linalg.conv_1d", - "linalg.conv_2d_nchw_fchw", - "linalg.conv_2d_ngchw_fgchw", - "linalg.conv_2d_ngchw_gfchw", - "linalg.conv_2d_nhwc_fhwc", - "linalg.conv_2d_nhwc_hwcf", - "linalg.conv_2d", - "linalg.conv_3d_ncdhw_fcdhw", - "linalg.conv_3d_ndhwc_dhwcf", - "linalg.conv_3d", - "linalg.depthwise_conv_1d_ncw_cw", - "linalg.depthwise_conv_1d_nwc_wc", - "linalg.depthwise_conv_1d_nwc_wcm", - "linalg.depthwise_conv_2d_nchw_chw", - "linalg.depthwise_conv_2d_nhwc_hwc", - "linalg.depthwise_conv_2d_nhwc_hwcm", - "linalg.depthwise_conv_3d_ncdhw_cdhw", - "linalg.depthwise_conv_3d_ndhwc_dhwc", - "linalg.depthwise_conv_3d_ndhwc_dhwcm", - ] -) - -# params format: [src | wei, src dt, wei dt, dst dt, amp] - - -def default_fill( - flags: argparse.Namespace, - arg: Arg, - arglist: List[Arg], -): - if arg.index > 2: - raise Exception("conv fill: dst filling is not allowed") - arg.fill_type = "D" - arg.fill_param = [ - "conv", - "src" if arg.index == 0 else "wei", - arglist[0].dtype, - arglist[1].dtype, - arglist[2].dtype, - ] - - # find the amplifier of the conv - wei = arglist[1] - nelem = wei.nelem() - if flags.driver == "linalg": - if flags.case in [ - "conv_1d_ncw_fcw", - "conv_2d_nchw_fchw", - "conv_2d_ngchw_fgchw", - "conv_2d_nhwc_fhwc", - "conv_3d_ncdhw_fcdhw", - ]: - arg.fill_param.append(str(nelem // wei.shape[0])) - elif flags.case in ["conv_2d_ngchw_gfchw"]: - arg.fill_param.append(str(nelem // wei.shape[1])) - elif flags.case in [ - "conv_1d_nwc_wcf", - "conv_2d_nhwc_hwcf", - "conv_3d_ndhwc_dhwcf", - "depthwise_conv_1d_nwc_wcm", - "depthwise_conv_2d_nhwc_hwcm", - "depthwise_conv_3d_ndhwc_dhwcm", - ]: - arg.fill_param.append(str(nelem // wei.shape[-1])) - elif flags.case in [ - "conv_1d", - "conv_2d", - "conv_3d", - "depthwise_conv_1d_ncw_cw", - "depthwise_conv_1d_nwc_wc", - "depthwise_conv_2d_nchw_chw", - "depthwise_conv_2d_nhwc_hwc", - "depthwise_conv_3d_ncdhw_cdhw", - "depthwise_conv_3d_ndhwc_dhwc", - ]: - arg.fill_param.append(str(nelem)) - - -def fill(shape: List[int], dtype: torch.dtype, params: List[str]) -> torch.Tensor: - name, src_dt, wei_dt, dst_dt, amp = params - - arg_rng: List[Dict[torch.dtype, Tuple[int, int]]] = [ - { - torch.float32: (-32, 32), - torch.bfloat16: (-4, 4), - torch.float16: (-4, 4), - }, # src - { - torch.float32: (-32, 32), - torch.bfloat16: (-8, 8), - torch.float16: (-2, 2), - }, # wei - ] - - target = torch.empty(size=shape, dtype=torch.float32) - target = target.view(-1) - - src_dt = benchgc.util.get_dtype(src_dt) - wei_dt = benchgc.util.get_dtype(wei_dt) - - src_min, src_max = arg_rng[0][src_dt] - wei_min, wei_max = arg_rng[1][wei_dt] - max_value = max(abs(src_min), abs(src_max)) * max(abs(wei_min), abs(wei_max)) - safe_digits: int = min( - benchgc.util.get_digits("f32"), benchgc.util.get_digits(dst_dt) - ) - safe_n_acc = (1 << safe_digits) // max_value - - if name == "src": - arg_min, arg_max = arg_rng[0][src_dt] - density = 1.0 - elif name == "wei": - arg_min, arg_max = arg_rng[1][wei_dt] - density = min(safe_n_acc / int(amp), 1.0) - else: - raise Exception("unknown arg name %s", name) - - benchgc.util.torch_seed() - - density_t = torch.full(shape, density, dtype=torch.float32) - bernoulli_t = torch.bernoulli(density_t) - condi = density_t == 1 - is_one_t = torch.where(condi, True, bernoulli_t) - gen_value = torch.randint(arg_min, arg_max + 1, size=shape) - target = is_one_t * gen_value - - # make sure the first element is positive - first_val = target.flatten()[0] - if first_val <= 0.0: - while first_val <= 0.0: - first_val = torch.randint(arg_min, arg_max + 1, size=()).item() - target_f = target.view(-1) - target_f[0] = first_val - target = target_f.view(shape) - - return target.to(dtype=dtype) - - -def default_compare( - flags: argparse.Namespace, - arg: Arg, - arglist: List[Arg], -): - arg.cmp_type = "D" - arg.cmp_param = ["conv", arg.dtype, flags.case] - - -def compare( - param: List[str], ref: torch.Tensor, res: torch.Tensor, verbose: int -) -> Tuple[bool, bool | None]: - dtype = benchgc.util.get_dtype(param[0]) - - ref = ref.to(torch.float) - res = res.to(torch.float) - return p2p( - 0.0, # use a relax threshold if using wino - 70.0 if dtype == torch.uint8 else 85.0, - ref, - res, - verbose, - ) diff --git a/test/benchgc/src/benchgc/arg/eltwise.py b/test/benchgc/src/benchgc/arg/eltwise.py deleted file mode 100644 index bf7dbff54..000000000 --- a/test/benchgc/src/benchgc/arg/eltwise.py +++ /dev/null @@ -1,177 +0,0 @@ -################################################################################ -# Copyright 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -import argparse -from typing import List, Set, Tuple - -import benchgc.util -import torch -from benchgc.arg.arg import Arg -from benchgc.arg.compare import p2p - -# params format: [alg, alpha, beta] - -# op should use this filling - -op: Set[str] = set( - [ - "linalg.abs", - "linalg.negf", - "linalg.exp", - "linalg.ceil", - "linalg.erf", - "linalg.floor", - "linalg.log", - "linalg.round", - "linalg.rsqrt", - "linalg.sqrt", - "linalg.square", - "linalg.tanh", - ] -) - - -def default_fill( - flags: argparse.Namespace, - arg: Arg, - arglist: List[Arg], -): - if arg.index > 0: - raise Exception("eltwise fill: dst filling is not allowed") - arg.fill_param = ["eltwise", flags.case] - if flags.driver == "linalg" and flags.case in [ - "abs", - "exp", - "ceil", - "erf", - "floor", - "log", - "round", - "sqrt", - "square", - "tanh", - ]: - arg.fill_param.extend(["", ""]) - elif flags.driver == "linalg" and flags.case == "negf": - arg.fill_param.extend(["-1", "0"]) - elif flags.driver == "linalg" and flags.case == "rsqrt": - arg.fill_param.extend(["1", "-0.5"]) - arg.fill_type = "D" - - -def fill(shape: List[int], dtype: torch.dtype, params: List[str]) -> torch.Tensor: - alg, alpha, beta = params - nelems = benchgc.util.nelem(shape) - - float_limit: torch.finfo = torch.finfo(torch.float32) - - alpha = 0.0 if alpha == "" else float(alpha) - beta = 0.0 if beta == "" else float(beta) - - coeff = torch.tensor( - [1, -1, 1, -1, 10.0, -10.0, 10.0, -10.0, 10.0, 10.0, 10.0, 1, 1] - ) - bias = torch.tensor([0, 0, 0, 0, 0, 0, 0, 0, 88.0, 22.0, 44.0, alpha, beta]) - rand_int_mask = torch.tensor( - [ - True, - True, - False, - False, - True, - True, - False, - False, - False, - False, - False, - False, - False, - ] - ) - rand_uni_mask = torch.tensor( - [ - False, - False, - True, - True, - False, - False, - True, - True, - True, - True, - True, - False, - False, - ] - ) - - if alg == "log": - # append more value for Log validation - coeff = torch.cat((coeff, torch.tensor([1, 1])), dim=0) - bias = torch.cat( - (bias, torch.tensor([float_limit.max, float_limit.min])), dim=0 - ) - rand_int_mask = torch.cat((rand_int_mask, torch.tensor([False, False])), dim=0) - rand_uni_mask = torch.cat((rand_uni_mask, torch.tensor([False, False])), dim=0) - - repeats: int = (nelems + coeff.nelement() - 1) // coeff.nelement() - - coeff = coeff.repeat(repeats)[:nelems] - bias = bias.repeat(repeats)[:nelems] - - rand_int_mask = rand_int_mask.repeat(repeats)[:nelems] - benchgc.util.torch_seed() - rand_int = torch.where(rand_int_mask, torch.randint(0, 10, [nelems]), 0) - - rand_uni_mask = rand_uni_mask.repeat(repeats)[:nelems] - rand_uni = torch.where(rand_uni_mask, torch.rand(nelems) * 0.09, 0) - - value = ((rand_int + rand_uni) * coeff + bias).to(dtype=dtype) - return value.reshape(shape) - - -# param: dtype, case - - -def default_compare( - flags: argparse.Namespace, - arg: Arg, - arglist: List[Arg], -): - arg.cmp_type = "D" - arg.cmp_param = ["eltwise", arg.dtype, flags.case] - - -def compare( - param: List[str], ref: torch.Tensor, res: torch.Tensor, verbose: int -) -> Tuple[bool, bool | None]: - dtype = benchgc.util.get_dtype(param[0]) - ref = ref.to(torch.float) - res = res.to(torch.float) - - threshold = 4e-6 if dtype == torch.float else benchgc.util.get_eps(dtype) - if dtype == torch.float and param[1] in ["tanh", "log"]: - threshold = 4e-5 - - return p2p( - threshold, - 65.0 if dtype.is_floating_point else 100.0, - ref, - res, - verbose, - ) diff --git a/test/benchgc/src/benchgc/arg/matmul.py b/test/benchgc/src/benchgc/arg/matmul.py deleted file mode 100644 index 76d0ce81e..000000000 --- a/test/benchgc/src/benchgc/arg/matmul.py +++ /dev/null @@ -1,173 +0,0 @@ -################################################################################ -# Copyright 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -import argparse -from typing import Dict, List, Set, Tuple - -import benchgc.util -import torch -from benchgc.arg.arg import Arg -from benchgc.arg.compare import p2p - -# params format: [src | wei, src dt, wei dt, dst dt, amp] -# use other filling type for bias - -op: Set[str] = set( - [ - "linalg.batch_matmul", - "linalg.batch_matmul_transpose_a", - "linalg.batch_matmul_transpose_b", - "linalg.batch_matvec", - "linalg.batch_mmt4d", - "linalg.batch_vecmat", - "linalg.batch_reduce_matmul", - "linalg.dot", - "linalg.matmul", - "linalg.matmul_transpose_a", - "linalg.matmul_transpose_b", - "linalg.matvec", - "linalg.mmt4d", - "linalg.vecmat", - ] -) - - -def default_fill( - flags: argparse.Namespace, - arg: Arg, - arglist: List[Arg], -): - if arg.index > 1: - raise Exception("matmul fill: dst filling is not allowed") - arg.fill_type = "D" - arg.fill_param = [ - "matmul", - "src" if arg.index == 0 else "wei", - arglist[0].dtype, - arglist[1].dtype, - arglist[2].dtype, - ] - - # find the amplifier K of the matmul - if flags.driver == "linalg": - if ( - flags.case == "matmul_transpose_b" - or flags.case == "batch_matmul" - and arg.index == 0 - or flags.case == "batch_matmul_transpose_b" - or flags.case == "batch_matvec" - or flags.case == "batch_vecmat" - and arg.index == 0 - or flags.case == "matmul" - and arg.index == 0 - or flags.case == "matvec" - or flags.case == "vecmat" - and arg.index == 0 - or flags.case == "dot" - ): - arg.fill_param.append(str(arg.shape[-1])) - elif ( - flags.case == "batch_matmul" - and arg.index == 1 - or flags.case == "batch_matmul_transpose_a" - or flags.case == "batch_vecmat" - and arg.index == 1 - or flags.case == "matmul" - and arg.index == 1 - or flags.case == "matmul_transpose_a" - or flags.case == "vecmat" - and arg.index == 1 - ): - arg.fill_param.append(str(arg.shape[-2])) - elif flags.case == "batch_mmt4d" or flags.case == "mmt4d": - arg.fill_param.append(str(arg.shape[-1] * arg.shape[-3])) - # reduce the matmul will amplified by B * K - elif flags.case == "batch_reduce_matmul" and arg.index == 0: - arg.fill_param.append(str(arg.shape[-1] * arg.shape[0])) - elif flags.case == "batch_reduce_matmul" and arg.index == 1: - arg.fill_param.append(str(arg.shape[-2] * arg.shape[0])) - - -def fill(shape: List[int], dtype: torch.dtype, params: List[str]) -> torch.Tensor: - name, src_dt, wei_dt, dst_dt, amp = params - - arg_rng: List[Dict[torch.dtype, Tuple[int, int]]] = [ - { - torch.float32: (-64, 64), - torch.bfloat16: (-4, 4), - torch.float16: (-4, 4), - }, # src - { - torch.float32: (-128, 128), - torch.bfloat16: (-8, 8), - torch.float16: (-2, 2), - }, # wei - ] - - src_dt = benchgc.util.get_dtype(src_dt) - wei_dt = benchgc.util.get_dtype(wei_dt) - - src_min, src_max = arg_rng[0][src_dt] - wei_min, wei_max = arg_rng[1][wei_dt] - max_value = max(abs(src_min), abs(src_max)) * max(abs(wei_min), abs(wei_max)) - safe_digits: int = min( - benchgc.util.get_digits("f32"), benchgc.util.get_digits(dst_dt) - ) - - safe_n_acc = (1 << safe_digits) // max_value - - if name == "src": - arg_min, arg_max = arg_rng[0][src_dt] - density = 1.0 - elif name == "wei": - arg_min, arg_max = arg_rng[1][wei_dt] - density = min(safe_n_acc / int(amp), 1.0) - else: - raise Exception("unknown arg name %s", name) - - benchgc.util.torch_seed(1, 0 if name == "src" else 1) - value = torch.bernoulli(torch.full(shape, density)) * torch.randint( - arg_min, arg_max, shape - ) - while value.flatten()[0] <= 0: - value.flatten()[0] = torch.randint(arg_min, arg_max + 1, size=[1])[0].item() - - return value.to(dtype) - - -def default_compare( - flags: argparse.Namespace, - arg: Arg, - arglist: List[Arg], -): - arg.cmp_type = "D" - arg.cmp_param = ["matmul", arg.dtype, flags.case] - - -def compare( - param: List[str], ref: torch.Tensor, res: torch.Tensor, verbose: int -) -> Tuple[bool, bool | None]: - dtype = benchgc.util.get_dtype(param[0]) - - ref = ref.to(torch.float) - res = res.to(torch.float) - return p2p( - 1e-6 if dtype == torch.float else benchgc.util.get_eps(dtype), - 90.0, - ref, - res, - verbose, - ) diff --git a/test/benchgc/src/benchgc/arg/pool.py b/test/benchgc/src/benchgc/arg/pool.py deleted file mode 100644 index 7179b7f91..000000000 --- a/test/benchgc/src/benchgc/arg/pool.py +++ /dev/null @@ -1,97 +0,0 @@ -################################################################################ -# Copyright 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -import argparse -from typing import List, Set, Tuple - -import benchgc.util -import torch -from benchgc.arg.arg import Arg -from benchgc.arg.compare import p2p - -op: Set[str] = set( - [ - "linalg.pooling_nchw_max", - "linalg.pooling_nchw_sum", - "linalg.pooling_ncw_max", - "linalg.pooling_ncw_sum", - "linalg.pooling_ndhwc_max", - "linalg.pooling_ndhwc_sum", - "linalg.pooling_nhwc_max", - "linalg.pooling_nhwc_sum", - "linalg.pooling_nhwc_min", - "linalg.pooling_nwc_max", - "linalg.pooling_nwc_min", - "linalg.pooling_nwc_sum", - ] -) - - -def default_fill( - flags: argparse.Namespace, - arg: Arg, - arglist: List[Arg], -): - if arg.index > 1: - raise Exception("pool fill: dst filling is not allowed") - arg.fill_type = "D" - arg.fill_param = ["pool"] - - -def fill(shape: List[int], dtype: torch.dtype, params: List[str]) -> torch.Tensor: - arg_rng: Tuple[int, int] = { - torch.float64: (-2048, 2048), - torch.float32: (-2048, 2048), - torch.int32: (-2048, 2048), - torch.bfloat16: (-32, 32), - torch.float16: (-32, 32), - torch.int8: (-128, 127), - torch.uint8: (0, 255), - }[dtype] - - benchgc.util.torch_seed() - target = torch.randint(arg_rng[0], arg_rng[1] + 1, size=[benchgc.util.nelem(shape)]) - # make sure the first element is not negative - if target[0] <= 0.0: - while target[0] <= 0.0: - target[0] = torch.randint(arg_rng[0], arg_rng[1], size=(1,))[0].item() - - return target.reshape(shape).to(dtype=dtype) - - -def default_compare( - flags: argparse.Namespace, - arg: Arg, - arglist: List[Arg], -): - arg.cmp_type = "D" - arg.cmp_param = ["pool", arg.dtype, flags.case] - - -def compare( - param: List[str], ref: torch.Tensor, res: torch.Tensor, verbose: int -) -> Tuple[bool, bool | None]: - dtype = benchgc.util.get_dtype(param[0]) - - ref = ref.to(torch.float) - res = res.to(torch.float) - return p2p( - benchgc.util.get_eps(dtype) * 10.0, - 99.0, - ref, - res, - verbose, - ) diff --git a/test/benchgc/src/benchgc/arg/reduce.py b/test/benchgc/src/benchgc/arg/reduce.py deleted file mode 100644 index 7f4ea8571..000000000 --- a/test/benchgc/src/benchgc/arg/reduce.py +++ /dev/null @@ -1,111 +0,0 @@ -################################################################################ -# Copyright 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -import argparse -from typing import List, Set, Tuple - -import benchgc.arg -import benchgc.util -import torch -from benchgc.arg.arg import Arg -from benchgc.arg.compare import p2p - -op: Set[str] = set( - [ - "linalg.reduce.add", - "linalg.reduce.mul", - "linalg.reduce.max", - "linalg.reduce.min", - "linalg.reduce.l1", - "linalg.reduce.l2_square", - ] -) - - -def default_fill( - flags: argparse.Namespace, - arg: Arg, - arglist: List[Arg], -): - if arg.index > 0: - raise Exception("reduce fill: dst filling is not allowed") - arg.fill_param = [ - "reduce", - flags.case, - arglist[0].dtype, - arglist[1].dtype, - str(arglist[0].nelem() // arglist[1].nelem()), - ] - arg.fill_type = "D" - -def fill(shape: List[int], dtype: torch.dtype, params: List[str]) -> torch.Tensor: - - op, sdtype, ddtype, amp = params - - sdtype = benchgc.util.get_dtype(sdtype) - ddtype = benchgc.util.get_dtype(ddtype) - - safe_to_reduce_elems: int = benchgc.util.get_problem_bounds(op, sdtype)[0] - - neutral_value: float = 1.0 if op == "reduce.mul" else 0.0 - - shift: float = ( - 1.0 - if (op == "reduce.min" and not sdtype.is_signed and not ddtype.is_signed) - else 0.0 - ) - - value_range: int = benchgc.util.get_problem_bounds(op, sdtype)[1] - - is_mul_fp: bool = op == "reduce.mul" and sdtype.is_floating_point - min_range: int = -value_range if is_mul_fp else 1 - - index = torch.arange(benchgc.util.nelem(shape)).reshape(shape) - - benchgc.util.torch_seed() - value = torch.randint(min_range, value_range + 1, size=shape) - if is_mul_fp: - value = torch.pow(2, value) - if sdtype.is_signed: # random choose positive or negative - value = torch.where(torch.BoolTensor(size=shape), value, -value) - - non_neutral_mask = benchgc.util.flip_coin( - index, - torch.full(shape, safe_to_reduce_elems / int(amp), dtype=torch.float32), - ) - if isinstance(non_neutral_mask, torch.Tensor): - value = torch.where(non_neutral_mask, value, neutral_value) - else: - raise Exception("Flip coin failed when generate the reduce data filling") - value = value + shift - return value.to(dtype) - - -def default_compare( - flags: argparse.Namespace, - arg: Arg, - arglist: List[Arg], -): - arg.cmp_type = "D" - arg.cmp_param = ["reduce", arg.dtype, flags.case] - -def compare( - param: List[str], ref: torch.Tensor, res: torch.Tensor, verbose: int -) -> Tuple[bool, bool | None]: - dtype = ref.dtype - ref = ref.to(torch.float) - res = res.to(torch.float) - return p2p(benchgc.util.get_eps(dtype), 30.0, ref, res, verbose) diff --git a/test/benchgc/src/benchgc/arg/softmax.py b/test/benchgc/src/benchgc/arg/softmax.py deleted file mode 100644 index a9731ec0a..000000000 --- a/test/benchgc/src/benchgc/arg/softmax.py +++ /dev/null @@ -1,93 +0,0 @@ -################################################################################ -# Copyright 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -import argparse -import operator -from functools import reduce -from typing import List, Set, Tuple - -import benchgc.util -import torch -from benchgc.arg.arg import Arg -from benchgc.arg.compare import p2p - -op: Set[str] = set(["linalg.softmax"]) - - -# params format: [reduce dimension] - - -def default_fill( - flags: argparse.Namespace, - arg: Arg, - arglist: List[Arg], -): - if arg.index > 0: - raise Exception("softmax fill: dst filling is not allowed") - arg.fill_type = "D" - arg.fill_param = ["softmax", str(flags.dimension)] - - -def fill(shape: List[int], dtype: torch.dtype, params: List[str]) -> torch.Tensor: - dimension: int = int(params[0]) - - outer: int = reduce(operator.mul, shape[:dimension], 1) - inner: int = reduce(operator.mul, shape[dimension + 1 :], 1) - benchgc.util.torch_seed() - sign = torch.randint(0, 1, size=[1, shape[dimension], 1]) * 2 - 1 - value = torch.randint(87, 90, size=[outer, shape[dimension], inner]) - value = torch.where(value == 87, 0, value) - value = value * sign - value = torch.where(value == 0, torch.finfo(dtype).min, value) - return value.reshape(shape).to(dtype) - - -# param: dtype, case, reduce size -def default_compare( - flags: argparse.Namespace, - arg: Arg, - arglist: List[Arg], -): - arg.cmp_type = "D" - arg.cmp_param = [ - "softmax", - arg.dtype, - flags.case, - str(arg.shape[int(flags.dimension)]), - ] - - -def compare( - param: List[str], ref: torch.Tensor, res: torch.Tensor, verbose: int -) -> Tuple[bool, bool | None]: - dtype = benchgc.util.get_dtype(param[0]) - ref = ref.to(torch.float) - res = res.to(torch.float) - - reduce_size = int(param[2]) - nzeros = ( - reduce_size - 1 - if dtype == torch.int8 or dtype == torch.uint8 - else max(0, reduce_size - 8) - ) - - return p2p( - benchgc.util.get_eps(dtype) * (5.0 if dtype == torch.float else 1.0), - 100.0 * nzeros / reduce_size, - ref, - res, - verbose, - ) diff --git a/test/benchgc/src/benchgc/arith/CMakeLists.txt b/test/benchgc/src/benchgc/arith/CMakeLists.txt deleted file mode 100644 index 63d2bfa79..000000000 --- a/test/benchgc/src/benchgc/arith/CMakeLists.txt +++ /dev/null @@ -1,22 +0,0 @@ -################################################################################ -# Copyright (C) 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions -# and limitations under the License. -# SPDX-License-Identifier: Apache-2.0 -################################################################################ - - -file(GLOB PYTHON_SCRIPTS "*.py") -foreach(PY_SCRIPT ${PYTHON_SCRIPTS}) - configure_file(${PY_SCRIPT} ${CMAKE_BINARY_DIR}/test/benchgc/src/benchgc/arith COPYONLY) -endforeach() diff --git a/test/benchgc/src/benchgc/arith/__init__.py b/test/benchgc/src/benchgc/arith/__init__.py deleted file mode 100644 index 42d6dd0aa..000000000 --- a/test/benchgc/src/benchgc/arith/__init__.py +++ /dev/null @@ -1,43 +0,0 @@ -################################################################################ -# Copyright 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -import argparse -import importlib -from typing import Callable, Dict, List, Tuple - -import torch -from benchgc.arg import Arg -from benchgc.mlir.util import MLIRCache -from gc_mlir import ir - -ref_op: Dict[ - str, - Callable[ - [MLIRCache, ir.OpView, Dict[str, torch.Tensor]], - Tuple[torch.Tensor, ...], - ], -] = {} -mlir_op: Dict[str, Callable[[argparse.Namespace, List[Arg], List[Arg]], ir.Module]] = {} - -for dri in ["basic"]: - mod = importlib.import_module(f"benchgc.arith.{dri}") - for key in mod.__dict__: - if key.startswith("ref_"): - op: str = key.removeprefix("ref_") - ref_op[op] = mod.__dict__[key] - if key.startswith("mlir_"): - op: str = key.removeprefix("mlir_") - mlir_op[op] = mod.__dict__[key] diff --git a/test/benchgc/src/benchgc/arith/basic.py b/test/benchgc/src/benchgc/arith/basic.py deleted file mode 100644 index 6c1cb9a53..000000000 --- a/test/benchgc/src/benchgc/arith/basic.py +++ /dev/null @@ -1,106 +0,0 @@ -################################################################################ -# Copyright 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -from typing import Dict, Tuple - -import benchgc.util -import torch -from benchgc.mlir.util import MLIRCache -from gc_mlir import ir - - -def ref_constant( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - value = op.attributes["value"] - if isinstance(value, ir.FloatAttr): - return ( - torch.full(size=tuple(), fill_value=value.__float__(), dtype=torch.float), - ) - elif isinstance(value, ir.DenseFPElementsAttr): - if value.is_splat: - return ( - torch.full( - size=tuple(value.type.shape), - fill_value=value.get_splat_value().value, - dtype=benchgc.util.get_dtype(str(value.get_splat_value().type)), - ), - ) - else: - raise Exception("only support splat value now") - elif isinstance(value, ir.IntegerAttr): - return (torch.full(size=tuple(), fill_value=value.__int__(), dtype=torch.int),) - elif isinstance(value, ir.DenseIntElementsAttr): - if value.is_splat: - return ( - torch.full( - size=tuple(value.type.shape), - fill_value=value.get_splat_value().value, - dtype=benchgc.util.get_dtype(str(value.get_splat_value().type)), - ), - ) - else: - raise Exception("only support splat value now") - else: - raise Exception("Not support constant type %s", type(value)) - - -def ref_mulf( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return (var[cache.opr[0]] * var[cache.opr[1]],) - - -def ref_addf( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return (var[cache.opr[0]] + var[cache.opr[1]],) - - -def ref_maxf( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return (torch.max(var[cache.opr[0]], var[cache.opr[1]]),) - - -def ref_minf( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return (torch.min(var[cache.opr[0]], var[cache.opr[1]]),) - - -def ref_muli( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return (var[cache.opr[0]] * var[cache.opr[1]],) - - -def ref_addi( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return (var[cache.opr[0]] + var[cache.opr[1]],) - - -def ref_maxsi( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return (torch.max(var[cache.opr[0]], var[cache.opr[1]]),) - - -def ref_minsi( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return (torch.min(var[cache.opr[0]], var[cache.opr[1]]),) diff --git a/test/benchgc/src/benchgc/bench.py b/test/benchgc/src/benchgc/bench.py deleted file mode 100644 index 0c8763191..000000000 --- a/test/benchgc/src/benchgc/bench.py +++ /dev/null @@ -1,206 +0,0 @@ -################################################################################ -# Copyright (C) 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions -# and limitations under the License. -# SPDX-License-Identifier: Apache-2.0 -################################################################################ - -import ctypes -import random -import timeit -from typing import Any, List, Tuple - -import numpy as np -from benchgc.mlir.util import ( - emit_benchmark_wrapped_main_func, - emit_nano_time, - get_kernel_func_from_module, -) -from gc_mlir import ir, runtime -from gc_mlir.graph_compiler import GraphCompiler - - -def py_timeit_bench( - ir_module: ir.Module, - entry_name: str, - pipeline: str, - mlir_args: List[Any], - ir_printing: bool = False, - repeat_time: int = 100, - warm_up: int = 20, -) -> Tuple[float, float]: - """benchmark mlir with python timeit.""" - compiler = GraphCompiler(pipeline) - compile_begin = timeit.default_timer() - engine = compiler.compile_and_jit(ir_module, ir_printing=ir_printing) - compile_cost = (timeit.default_timer() - compile_begin) * 1000 - - # Copied from execution_engine.py so that the cost of cast does not affect perf result. - func = engine.lookup(entry_name) - packed_args = (ctypes.c_void_p * len(mlir_args))() - for argNum in range(len(mlir_args)): - packed_args[argNum] = ctypes.cast(mlir_args[argNum], ctypes.c_void_p) - - def run_bench(func, arg): - func(arg) - - timeit.timeit(lambda: run_bench(func, packed_args), number=warm_up) - total_time = timeit.timeit(lambda: run_bench(func, packed_args), number=repeat_time) - execute_cost = total_time * 1000 / repeat_time - return (execute_cost, compile_cost) - - -def mlir_wrapper_bench( - ir_module: ir.Module, - entry_name: str, - pipeline: str, - mlir_args: List[Any], - ir_printing: bool = False, - repeat_time: int = 100, - warm_up: int = 20, -) -> Tuple[float, float]: - """benchmark mlir with a wrapper func.""" - kernel_func = get_kernel_func_from_module(ir_module, entry_name) - wrapper_module = ir_module - with ir.InsertionPoint(wrapper_module.body): - emit_benchmark_wrapped_main_func(kernel_func, emit_nano_time()) - compiler = GraphCompiler(pipeline) - compile_begin = timeit.default_timer() - engine = compiler.compile_and_jit(wrapper_module, ir_printing=ir_printing) - compile_cost = (timeit.default_timer() - compile_begin) * 1000 - - np_timers_ns = np.array([0], dtype=np.int64) - time_arg = ctypes.pointer( - ctypes.pointer(runtime.get_ranked_memref_descriptor(np_timers_ns)) - ) - total_time = 0 - ns_to_ms_scale = 1e-6 - - def run(engine_invoke, bench_func_name, *mlir_args): - engine_invoke(bench_func_name, *mlir_args) - - for i in range(repeat_time + warm_up): - run(engine.invoke, "wrapped_main", time_arg, *mlir_args) - if i >= warm_up: - total_time += int(np_timers_ns[0]) * ns_to_ms_scale - execute_cost = total_time / repeat_time - return (execute_cost, compile_cost) - - -# for test -def fake_bench( - ir_module: ir.Module = None, - entry_name: str = None, - pipeline: str = None, - mlir_args: list = None, - ir_printing=False, - repeat_time=100, - warm_up=20, -) -> Tuple[float, float]: - """genrate fake benchmark result.""" - execute_cost = float(random.randint(1, 100)) - compile_cost = float(random.randint(1, 100)) - return (execute_cost, compile_cost) - - -def batch_py_timeit_bench( - ir_modules: List[ir.Module], - entry_name: str, - pipeline: str, - mlir_args: list, - ir_printing=False, - repeat_time=5, - warm_up=2, -) -> List[Tuple[float, float]]: - """benchmark a batch of mlir with python timeit.""" - compiler = GraphCompiler(pipeline) - funcs = [] - compile_costs = [] - for m in ir_modules: - compile_begin = timeit.default_timer() - engine = compiler.compile_and_jit(m, ir_printing=ir_printing) - compile_cost = (timeit.default_timer() - compile_begin) * 1000 - compile_costs.append(compile_cost) - funcs.append(engine.lookup(entry_name)) - - # Copied from execution_engine.py so that the cost of cast does not affect perf result. - packed_args = (ctypes.c_void_p * len(mlir_args))() - for argNum in range(len(mlir_args)): - packed_args[argNum] = ctypes.cast(mlir_args[argNum], ctypes.c_void_p) - - def run_bench(func, arg): - func(arg) - - for func in funcs: - timeit.timeit(lambda: run_bench(func, packed_args), number=warm_up) - - execute_costs = [] - for func in funcs: - total_time = timeit.timeit( - lambda: run_bench(func, packed_args), number=repeat_time - ) - execute_cost = total_time * 1000 / repeat_time - execute_costs.append(execute_cost) - return list(zip(compile_costs, execute_costs)) - - -def batch_mlir_wrapper_bench( - ir_modules: ir.Module, - entry_name: str, - pipeline: str, - mlir_args: list, - ir_printing=False, - repeat_time=5, - warm_up=2, -) -> Tuple[float, float]: - """benchmark a batch of mlir with wrapper func.""" - compiler = GraphCompiler(pipeline) - - engine_invokes = [] - compile_costs = [] - for m in ir_modules: - kernel_func = get_kernel_func_from_module(m, entry_name) - wrapper_module = m - with ir.InsertionPoint(wrapper_module.body): - emit_benchmark_wrapped_main_func(kernel_func, emit_nano_time()) - compile_begin = timeit.default_timer() - engine = compiler.compile_and_jit(wrapper_module, ir_printing=ir_printing) - compile_cost = (timeit.default_timer() - compile_begin) * 1000 - compile_costs.append(compile_cost) - engine_invokes.append(engine.invoke) - - np_timers_ns = np.array([0], dtype=np.int64) - time_arg = ctypes.pointer( - ctypes.pointer(runtime.get_ranked_memref_descriptor(np_timers_ns)) - ) - total_time = 0 - ns_to_ms_scale = 1e-6 - - def run(engine_invoke, bench_func_name, *mlir_args): - engine_invoke(bench_func_name, *mlir_args) - - for engine_invoke in engine_invokes: - for _ in range(warm_up): - run(engine_invoke, "wrapped_main", time_arg, *mlir_args) - - execute_costs = [] - for engine_invoke in engine_invokes: - total_time = 0 - for _ in range(repeat_time): - run(engine_invoke, "wrapped_main", time_arg, *mlir_args) - total_time += int(np_timers_ns[0]) * ns_to_ms_scale - - execute_cost = total_time / repeat_time - execute_costs.append(execute_cost) - - return list(zip(compile_costs, execute_costs)) diff --git a/test/benchgc/src/benchgc/linalg/CMakeLists.txt b/test/benchgc/src/benchgc/linalg/CMakeLists.txt deleted file mode 100644 index 8daf7848a..000000000 --- a/test/benchgc/src/benchgc/linalg/CMakeLists.txt +++ /dev/null @@ -1,22 +0,0 @@ -################################################################################ -# Copyright (C) 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions -# and limitations under the License. -# SPDX-License-Identifier: Apache-2.0 -################################################################################ - - -file(GLOB PYTHON_SCRIPTS "*.py") -foreach(PY_SCRIPT ${PYTHON_SCRIPTS}) - configure_file(${PY_SCRIPT} ${CMAKE_BINARY_DIR}/test/benchgc/src/benchgc/linalg/ COPYONLY) -endforeach() diff --git a/test/benchgc/src/benchgc/linalg/__init__.py b/test/benchgc/src/benchgc/linalg/__init__.py deleted file mode 100644 index 0b4dd9bfa..000000000 --- a/test/benchgc/src/benchgc/linalg/__init__.py +++ /dev/null @@ -1,53 +0,0 @@ -################################################################################ -# Copyright 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -import argparse -import importlib -from typing import Callable, Dict, List, Tuple - -import torch -from benchgc.arg import Arg -from benchgc.mlir.util import MLIRCache -from gc_mlir import ir - -ref_op: Dict[ - str, - Callable[ - [MLIRCache, ir.OpView, Dict[str, torch.Tensor]], - Tuple[torch.Tensor, ...], - ], -] = {} -mlir_op: Dict[str, Callable[[argparse.Namespace, List[Arg]], ir.Module]] = {} - -for dri in [ - "binary", - "matmul", - "eltwise", - "misc", - "generic", - "softmax", - "conv", - "pool", - "reduce", -]: - mod = importlib.import_module(f"benchgc.linalg.{dri}") - for key in mod.__dict__: - if key.startswith("ref_"): - op: str = key.removeprefix("ref_") - ref_op[op] = mod.__dict__[key] - if key.startswith("mlir_"): - op: str = key.removeprefix("mlir_") - mlir_op[op] = mod.__dict__[key] diff --git a/test/benchgc/src/benchgc/linalg/binary.py b/test/benchgc/src/benchgc/linalg/binary.py deleted file mode 100644 index 66f3a7abe..000000000 --- a/test/benchgc/src/benchgc/linalg/binary.py +++ /dev/null @@ -1,144 +0,0 @@ -################################################################################ -# Copyright 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -import argparse -from typing import Dict, List, Tuple - -import torch -from benchgc.arg import Arg -from benchgc.mlir.module import init_module -from benchgc.mlir.util import MLIRCache -from gc_mlir import ir -from gc_mlir.dialects import linalg - - -def ref_add( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return (torch.add(var[cache.opr[0]], var[cache.opr[1]]),) - - -def mlir_add(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.add(arg0, arg1, outs=[args[2].get_zero_op(ctx)]) - ], - ) - - -def ref_powf( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return (torch.pow(var[cache.opr[0]], var[cache.opr[1]]),) - - -def mlir_powf(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.powf(arg0, arg1, outs=[args[2].get_zero_op(ctx)]) - ], - ) - - -def ref_div( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return (torch.div(var[cache.opr[0]], var[cache.opr[1]]),) - - -def mlir_div(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.div(arg0, arg1, outs=[args[2].get_zero_op(ctx)]) - ], - ) - - -def ref_max( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return (torch.max(var[cache.opr[0]], var[cache.opr[1]]),) - - -def mlir_max(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.max(arg0, arg1, outs=[args[2].get_zero_op(ctx)]) - ], - ) - - -def ref_min( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return (torch.min(var[cache.opr[0]], var[cache.opr[1]]),) - - -def mlir_min(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.min(arg0, arg1, outs=[args[2].get_zero_op(ctx)]) - ], - ) - - -def ref_mul( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return (torch.mul(var[cache.opr[0]], var[cache.opr[1]]),) - - -def mlir_mul(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.mul(arg0, arg1, outs=[args[2].get_zero_op(ctx)]) - ], - ) - - -def ref_sub( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return (torch.sub(var[cache.opr[0]], var[cache.opr[1]]),) - - -def mlir_sub(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.sub(arg0, arg1, outs=[args[2].get_zero_op(ctx)]) - ], - ) diff --git a/test/benchgc/src/benchgc/linalg/conv.py b/test/benchgc/src/benchgc/linalg/conv.py deleted file mode 100644 index f1f8d7f97..000000000 --- a/test/benchgc/src/benchgc/linalg/conv.py +++ /dev/null @@ -1,836 +0,0 @@ -################################################################################ -# Copyright 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -import argparse -from typing import Dict, List, Tuple - -import torch -from benchgc.arg import Arg -from benchgc.mlir.module import init_module -from benchgc.mlir.util import MLIRCache -from gc_mlir import ir -from gc_mlir.dialects import linalg - - -def ref_conv_1d_ncw_fcw( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - strides: ir.DenseIntElementsAttr = op.attributes["strides"] - dilations: ir.DenseIntElementsAttr = op.attributes["dilations"] - return ( - torch.conv1d( - var[cache.opr[0]], - var[cache.opr[1]], - stride=[strides[i] for i in range(len(strides))], - dilation=[dilations[i] for i in range(len(dilations))], - ), - ) - - -def mlir_conv_1d_ncw_fcw(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.conv_1d_ncw_fcw( - arg0, - arg1, - outs=[args[2].get_zero_op(ctx)], - strides=flags.strides, - dilations=flags.dilations, - ) - ], - ) - - -def ref_conv_1d_nwc_wcf( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - strides: ir.DenseIntElementsAttr = op.attributes["strides"] - dilations: ir.DenseIntElementsAttr = op.attributes["dilations"] - - # src: nwc -> ncw - # wei: wcf -> fcw - # dst: nwf -> nfw - - return ( - torch.conv1d( - var[cache.opr[0]].permute([0, 2, 1]), - var[cache.opr[1]].permute([2, 1, 0]), - stride=[strides[i] for i in range(len(strides))], - dilation=[dilations[i] for i in range(len(dilations))], - ) - .permute([0, 2, 1]) - .contiguous(), - ) - - -def mlir_conv_1d_nwc_wcf(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.conv_1d_nwc_wcf( - arg0, - arg1, - outs=[args[2].get_zero_op(ctx)], - strides=flags.strides, - dilations=flags.dilations, - ) - ], - ) - - -def ref_conv_1d_ncw_fcw( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - strides: ir.DenseIntElementsAttr = op.attributes["strides"] - dilations: ir.DenseIntElementsAttr = op.attributes["dilations"] - return ( - torch.conv1d( - var[cache.opr[0]], - var[cache.opr[1]], - stride=[strides[i] for i in range(len(strides))], - dilation=[dilations[i] for i in range(len(dilations))], - ), - ) - - -def mlir_conv_1d_ncw_fcw(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.conv_1d_ncw_fcw( - arg0, - arg1, - outs=[args[2].get_zero_op(ctx)], - strides=flags.strides, - dilations=flags.dilations, - ) - ], - ) - - -def ref_conv_1d( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return ( - torch.conv1d( - var[cache.opr[0]].unsqueeze(0).unsqueeze(0), - var[cache.opr[1]].unsqueeze(0).unsqueeze(0), - ) - .squeeze(0) - .squeeze(0), - ) - - -def mlir_conv_1d(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.conv_1d( - arg0, - arg1, - outs=[args[2].get_zero_op(ctx)], - ) - ], - ) - - -def ref_conv_2d_nchw_fchw( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - strides: ir.DenseIntElementsAttr = op.attributes["strides"] - dilations: ir.DenseIntElementsAttr = op.attributes["dilations"] - return ( - torch.conv2d( - var[cache.opr[0]], - var[cache.opr[1]], - stride=[strides[i] for i in range(len(strides))], - dilation=[dilations[i] for i in range(len(dilations))], - ), - ) - - -def mlir_conv_2d_nchw_fchw(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.conv_2d_nchw_fchw( - arg0, - arg1, - outs=[args[2].get_zero_op(ctx)], - strides=flags.strides, - dilations=flags.dilations, - ) - ], - ) - - -def ref_conv_2d_ngchw_fgchw( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - strides: ir.DenseIntElementsAttr = op.attributes["strides"] - dilations: ir.DenseIntElementsAttr = op.attributes["dilations"] - - src = var[cache.opr[0]] - wei = var[cache.opr[1]] - groups: int = src.shape[1] - - dst = torch.conv2d( - src.reshape( - [src.shape[0], src.shape[1] * src.shape[2], src.shape[3], src.shape[4]] - ), # merge group axis with channel - wei.transpose(0, 1) - .contiguous() - .reshape( - [wei.shape[0] * wei.shape[1], wei.shape[2], wei.shape[3], wei.shape[4]] - ), # merge group axis with output channel - stride=[strides[i] for i in range(len(strides))], - dilation=[dilations[i] for i in range(len(dilations))], - groups=groups, - ) - return ( - dst.reshape( - [dst.shape[0], groups, dst.shape[1] // groups, dst.shape[2], dst.shape[3]] - ), - ) # split group axis from output channel - - -def mlir_conv_2d_ngchw_fgchw(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.conv_2d_ngchw_fgchw( - arg0, - arg1, - outs=[args[2].get_zero_op(ctx)], - strides=flags.strides, - dilations=flags.dilations, - ) - ], - ) - - -def ref_conv_2d_ngchw_gfchw( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - strides: ir.DenseIntElementsAttr = op.attributes["strides"] - dilations: ir.DenseIntElementsAttr = op.attributes["dilations"] - - src = var[cache.opr[0]] - wei = var[cache.opr[1]] - groups: int = src.shape[1] - - dst = torch.conv2d( - src.reshape( - [src.shape[0], src.shape[1] * src.shape[2], src.shape[3], src.shape[4]] - ), # merge group axis with channel - wei.reshape( - [wei.shape[0] * wei.shape[1], wei.shape[2], wei.shape[3], wei.shape[4]] - ), # merge group axis with output channel - stride=[strides[i] for i in range(len(strides))], - dilation=[dilations[i] for i in range(len(dilations))], - groups=groups, - ) - return ( - dst.reshape( - [dst.shape[0], groups, dst.shape[1] // groups, dst.shape[2], dst.shape[3]] - ), - ) # split group axis from output channel - - -def mlir_conv_2d_ngchw_gfchw(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.conv_2d_ngchw_gfchw( - arg0, - arg1, - outs=[args[2].get_zero_op(ctx)], - strides=flags.strides, - dilations=flags.dilations, - ) - ], - ) - - -def ref_conv_2d_nhwc_fhwc( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - strides: ir.DenseIntElementsAttr = op.attributes["strides"] - dilations: ir.DenseIntElementsAttr = op.attributes["dilations"] - return ( - torch.conv2d( - var[cache.opr[0]].permute([0, 3, 1, 2]), - var[cache.opr[1]].permute([0, 3, 1, 2]), - stride=[strides[i] for i in range(len(strides))], - dilation=[dilations[i] for i in range(len(dilations))], - ) - .permute([0, 2, 3, 1]) - .contiguous(), - ) - - -def mlir_conv_2d_nhwc_fhwc(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.conv_2d_nhwc_fhwc( - arg0, - arg1, - outs=[args[2].get_zero_op(ctx)], - strides=flags.strides, - dilations=flags.dilations, - ) - ], - ) - - -def ref_conv_2d_nhwc_hwcf( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - strides: ir.DenseIntElementsAttr = op.attributes["strides"] - dilations: ir.DenseIntElementsAttr = op.attributes["dilations"] - return ( - torch.conv2d( - var[cache.opr[0]].permute([0, 3, 1, 2]), - var[cache.opr[1]].permute([3, 2, 0, 1]), - stride=[strides[i] for i in range(len(strides))], - dilation=[dilations[i] for i in range(len(dilations))], - ) - .permute([0, 2, 3, 1]) - .contiguous(), - ) - - -def mlir_conv_2d_nhwc_hwcf(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.conv_2d_nhwc_hwcf( - arg0, - arg1, - outs=[args[2].get_zero_op(ctx)], - strides=flags.strides, - dilations=flags.dilations, - ) - ], - ) - - -def ref_conv_2d( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return ( - torch.conv2d( - var[cache.opr[0]].unsqueeze(0).unsqueeze(0), - var[cache.opr[1]].unsqueeze(0).unsqueeze(0), - ) - .squeeze(0) - .squeeze(0), - ) - - -def mlir_conv_2d(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.conv_2d( - arg0, - arg1, - outs=[args[2].get_zero_op(ctx)], - ) - ], - ) - - -def ref_conv_3d_ncdhw_fcdhw( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - strides: ir.DenseIntElementsAttr = op.attributes["strides"] - dilations: ir.DenseIntElementsAttr = op.attributes["dilations"] - return ( - torch.conv3d( - var[cache.opr[0]], - var[cache.opr[1]], - stride=[strides[i] for i in range(len(strides))], - dilation=[dilations[i] for i in range(len(dilations))], - ), - ) - - -def mlir_conv_3d_ncdhw_fcdhw(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.conv_3d_ncdhw_fcdhw( - arg0, - arg1, - outs=[args[2].get_zero_op(ctx)], - strides=flags.strides, - dilations=flags.dilations, - ) - ], - ) - - -def ref_conv_3d_ndhwc_dhwcf( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - strides: ir.DenseIntElementsAttr = op.attributes["strides"] - dilations: ir.DenseIntElementsAttr = op.attributes["dilations"] - return ( - torch.conv3d( - var[cache.opr[0]].permute([0, 4, 1, 2, 3]), - var[cache.opr[1]].permute([4, 3, 0, 1, 2]), - stride=[strides[i] for i in range(len(strides))], - dilation=[dilations[i] for i in range(len(dilations))], - ) - .permute([0, 2, 3, 4, 1]) - .contiguous(), - ) - - -def mlir_conv_3d_ndhwc_dhwcf(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.conv_3d_ndhwc_dhwcf( - arg0, - arg1, - outs=[args[2].get_zero_op(ctx)], - strides=flags.strides, - dilations=flags.dilations, - ) - ], - ) - - -def ref_conv_3d( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return ( - torch.conv3d( - var[cache.opr[0]].unsqueeze(0).unsqueeze(0), - var[cache.opr[1]].unsqueeze(0).unsqueeze(0), - ) - .squeeze(0) - .squeeze(0), - ) - - -def mlir_conv_3d(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.conv_3d( - arg0, - arg1, - outs=[args[2].get_zero_op(ctx)], - ) - ], - ) - - -def ref_depthwise_conv_1d_ncw_cw( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - strides: ir.DenseIntElementsAttr = op.attributes["strides"] - dilations: ir.DenseIntElementsAttr = op.attributes["dilations"] - groups: int = var[cache.opr[0]].shape[1] - return ( - torch.conv1d( - var[cache.opr[0]], - var[cache.opr[1]].unsqueeze(1), - stride=[strides[i] for i in range(len(strides))], - dilation=[dilations[i] for i in range(len(dilations))], - groups=groups, - ), - ) - - -def mlir_depthwise_conv_1d_ncw_cw( - flags: argparse.Namespace, args: List[Arg] -) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.depthwise_conv_1d_ncw_cw( - arg0, - arg1, - outs=[args[2].get_zero_op(ctx)], - strides=flags.strides, - dilations=flags.dilations, - ) - ], - ) - - -def ref_depthwise_conv_1d_nwc_wc( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - strides: ir.DenseIntElementsAttr = op.attributes["strides"] - dilations: ir.DenseIntElementsAttr = op.attributes["dilations"] - groups: int = var[cache.opr[0]].shape[-1] - return ( - torch.conv1d( - var[cache.opr[0]].transpose(-1, -2), - var[cache.opr[1]].transpose(-1, -2).unsqueeze(1), - stride=[strides[i] for i in range(len(strides))], - dilation=[dilations[i] for i in range(len(dilations))], - groups=groups, - ) - .transpose(-1, -2) - .contiguous(), - ) - - -def mlir_depthwise_conv_1d_nwc_wc( - flags: argparse.Namespace, args: List[Arg] -) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.depthwise_conv_1d_nwc_wc( - arg0, - arg1, - outs=[args[2].get_zero_op(ctx)], - strides=flags.strides, - dilations=flags.dilations, - ) - ], - ) - - -def ref_depthwise_conv_1d_nwc_wcm( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - strides: ir.DenseIntElementsAttr = op.attributes["strides"] - dilations: ir.DenseIntElementsAttr = op.attributes["dilations"] - src = var[cache.opr[0]] - groups: int = src.shape[-1] - wei = var[cache.opr[1]] - dst = ( - torch.conv1d( - src.transpose(-1, -2), - wei.reshape([wei.shape[0], wei.shape[1] * wei.shape[2]]) - .transpose(-1, -2) - .unsqueeze(1), - stride=[strides[i] for i in range(len(strides))], - dilation=[dilations[i] for i in range(len(dilations))], - groups=groups, - ) - .transpose(-1, -2) - .contiguous() - ) - return (dst.reshape([dst.shape[0], dst.shape[1], wei.shape[1], wei.shape[2]]),) - - -def mlir_depthwise_conv_1d_nwc_wcm( - flags: argparse.Namespace, args: List[Arg] -) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.depthwise_conv_1d_nwc_wcm( - arg0, - arg1, - outs=[args[2].get_zero_op(ctx)], - strides=flags.strides, - dilations=flags.dilations, - ) - ], - ) - - -def ref_depthwise_conv_2d_nchw_chw( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - strides: ir.DenseIntElementsAttr = op.attributes["strides"] - dilations: ir.DenseIntElementsAttr = op.attributes["dilations"] - groups: int = var[cache.opr[0]].shape[1] - return ( - torch.conv2d( - var[cache.opr[0]], - var[cache.opr[1]].unsqueeze(1), - stride=[strides[i] for i in range(len(strides))], - dilation=[dilations[i] for i in range(len(dilations))], - groups=groups, - ), - ) - - -def mlir_depthwise_conv_2d_nchw_chw( - flags: argparse.Namespace, args: List[Arg] -) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.depthwise_conv_2d_nchw_chw( - arg0, - arg1, - outs=[args[2].get_zero_op(ctx)], - strides=flags.strides, - dilations=flags.dilations, - ) - ], - ) - - -def ref_depthwise_conv_2d_nhwc_hwc( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - strides: ir.DenseIntElementsAttr = op.attributes["strides"] - dilations: ir.DenseIntElementsAttr = op.attributes["dilations"] - groups: int = var[cache.opr[0]].shape[-1] - return ( - torch.conv2d( - var[cache.opr[0]].permute([0, 3, 1, 2]), - var[cache.opr[1]].permute([2, 0, 1]).unsqueeze(1), - stride=[strides[i] for i in range(len(strides))], - dilation=[dilations[i] for i in range(len(dilations))], - groups=groups, - ) - .permute([0, 2, 3, 1]) - .contiguous(), - ) - - -def mlir_depthwise_conv_2d_nhwc_hwc( - flags: argparse.Namespace, args: List[Arg] -) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.depthwise_conv_2d_nhwc_hwc( - arg0, - arg1, - outs=[args[2].get_zero_op(ctx)], - strides=flags.strides, - dilations=flags.dilations, - ) - ], - ) - - -def ref_depthwise_conv_2d_nhwc_hwcm( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - strides: ir.DenseIntElementsAttr = op.attributes["strides"] - dilations: ir.DenseIntElementsAttr = op.attributes["dilations"] - groups: int = var[cache.opr[0]].shape[-1] - wei = var[cache.opr[1]] - dst = ( - torch.conv2d( - var[cache.opr[0]].permute([0, 3, 1, 2]), - wei.reshape([wei.shape[0], wei.shape[1], wei.shape[2] * wei.shape[3]]) - .permute([2, 0, 1]) - .unsqueeze(1), - stride=[strides[i] for i in range(len(strides))], - dilation=[dilations[i] for i in range(len(dilations))], - groups=groups, - ) - .permute([0, 2, 3, 1]) - .contiguous() - ) - return ( - dst.reshape( - [dst.shape[0], dst.shape[1], dst.shape[2], wei.shape[-2], wei.shape[-1]] - ), - ) - - -def mlir_depthwise_conv_2d_nhwc_hwcm( - flags: argparse.Namespace, args: List[Arg] -) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.depthwise_conv_2d_nhwc_hwcm( - arg0, - arg1, - outs=[args[2].get_zero_op(ctx)], - strides=flags.strides, - dilations=flags.dilations, - ) - ], - ) - - -def ref_depthwise_conv_3d_ncdhw_cdhw( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - strides: ir.DenseIntElementsAttr = op.attributes["strides"] - dilations: ir.DenseIntElementsAttr = op.attributes["dilations"] - groups: int = var[cache.opr[0]].shape[1] - return ( - torch.conv3d( - var[cache.opr[0]], - var[cache.opr[1]].unsqueeze(1), - stride=[strides[i] for i in range(len(strides))], - dilation=[dilations[i] for i in range(len(dilations))], - groups=groups, - ), - ) - - -def mlir_depthwise_conv_3d_ncdhw_cdhw( - flags: argparse.Namespace, args: List[Arg] -) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.depthwise_conv_3d_ncdhw_cdhw( - arg0, - arg1, - outs=[args[2].get_zero_op(ctx)], - strides=flags.strides, - dilations=flags.dilations, - ) - ], - ) - - -def ref_depthwise_conv_3d_ndhwc_dhwc( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - strides: ir.DenseIntElementsAttr = op.attributes["strides"] - dilations: ir.DenseIntElementsAttr = op.attributes["dilations"] - groups: int = var[cache.opr[0]].shape[-1] - return ( - torch.conv3d( - var[cache.opr[0]].permute([0, 4, 1, 2, 3]), - var[cache.opr[1]].permute([3, 0, 1, 2]).unsqueeze(1), - stride=[strides[i] for i in range(len(strides))], - dilation=[dilations[i] for i in range(len(dilations))], - groups=groups, - ) - .permute([0, 2, 3, 4, 1]) - .contiguous(), - ) - - -def mlir_depthwise_conv_3d_ndhwc_dhwc( - flags: argparse.Namespace, args: List[Arg] -) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.depthwise_conv_3d_ndhwc_dhwc( - arg0, - arg1, - outs=[args[2].get_zero_op(ctx)], - strides=flags.strides, - dilations=flags.dilations, - ) - ], - ) - - -def ref_depthwise_conv_3d_ndhwc_dhwcm( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - strides: ir.DenseIntElementsAttr = op.attributes["strides"] - dilations: ir.DenseIntElementsAttr = op.attributes["dilations"] - groups: int = var[cache.opr[0]].shape[-1] - wei = var[cache.opr[1]] - dst = ( - torch.conv3d( - var[cache.opr[0]].permute([0, 4, 1, 2, 3]), - wei.reshape( - [wei.shape[0], wei.shape[1], wei.shape[2], wei.shape[3] * wei.shape[4]] - ) - .permute([3, 0, 1, 2]) - .unsqueeze(1), - stride=[strides[i] for i in range(len(strides))], - dilation=[dilations[i] for i in range(len(dilations))], - groups=groups, - ) - .permute([0, 2, 3, 4, 1]) - .contiguous() - ) - return ( - dst.reshape( - [ - dst.shape[0], - dst.shape[1], - dst.shape[2], - dst.shape[3], - wei.shape[-2], - wei.shape[-1], - ] - ), - ) - - -def mlir_depthwise_conv_3d_ndhwc_dhwcm( - flags: argparse.Namespace, args: List[Arg] -) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.depthwise_conv_3d_ndhwc_dhwcm( - arg0, - arg1, - outs=[args[2].get_zero_op(ctx)], - strides=flags.strides, - dilations=flags.dilations, - ) - ], - ) diff --git a/test/benchgc/src/benchgc/linalg/eltwise.py b/test/benchgc/src/benchgc/linalg/eltwise.py deleted file mode 100644 index 760fcb0a1..000000000 --- a/test/benchgc/src/benchgc/linalg/eltwise.py +++ /dev/null @@ -1,209 +0,0 @@ -################################################################################ -# Copyright 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -import argparse -from typing import Dict, List, Tuple - -import torch -from benchgc.arg import Arg -from benchgc.mlir.module import init_module -from benchgc.mlir.util import MLIRCache -from gc_mlir import ir -from gc_mlir.dialects import linalg - - -def ref_abs( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return (torch.abs(var[cache.opr[0]]),) - - -def mlir_abs(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0],), - (args[1],), - lambda ctx, arg0: [linalg.abs(arg0, outs=[args[1].get_zero_op(ctx)])], - ) - - -def ref_ceil( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return (torch.ceil(var[cache.opr[0]]),) - - -def mlir_ceil(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0],), - (args[1],), - lambda ctx, arg0: [linalg.ceil(arg0, outs=[args[1].get_zero_op(ctx)])], - ) - - -def ref_floor( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return (torch.floor(var[cache.opr[0]]),) - - -def mlir_floor(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0],), - (args[1],), - lambda ctx, arg0: [linalg.floor(arg0, outs=[args[1].get_zero_op(ctx)])], - ) - - -def ref_erf( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return (torch.erf(var[cache.opr[0]]),) - - -def mlir_erf(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0],), - (args[1],), - lambda ctx, arg0: [linalg.erf(arg0, outs=[args[1].get_zero_op(ctx)])], - ) - - -def mlir_log(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0],), - (args[1],), - lambda ctx, arg0: [linalg.log(arg0, outs=[args[1].get_zero_op(ctx)])], - ) - - -def ref_log( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return (torch.log(var[cache.opr[0]]),) - - -def mlir_negf(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0],), - (args[1],), - lambda ctx, arg0: [linalg.negf(arg0, outs=[args[1].get_zero_op(ctx)])], - ) - - -def ref_negf( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return (torch.neg(var[cache.opr[0]]),) - - -def ref_exp( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return (torch.exp(var[cache.opr[0]]),) - - -def mlir_exp(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0],), - (args[1],), - lambda ctx, arg0: [linalg.negf(arg0, outs=[args[1].get_zero_op(ctx)])], - ) - - -def ref_round( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - # torch.round is following the priciple "round half to even" - # we need another implementation - - v = torch.floor(var[cache.opr[0]]) - return (v + torch.where(var[cache.opr[0]] - v >= 0.5, 1, 0),) - - -def mlir_round(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0],), - (args[1],), - lambda ctx, arg0: [linalg.round(arg0, outs=[args[1].get_zero_op(ctx)])], - ) - - -def ref_rsqrt( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return (torch.rsqrt(var[cache.opr[0]]),) - - -def mlir_rsqrt(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0],), - (args[1],), - lambda ctx, arg0: [linalg.rsqrt(arg0, outs=[args[1].get_zero_op(ctx)])], - ) - - -def ref_sqrt( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return (torch.sqrt(var[cache.opr[0]]),) - - -def mlir_sqrt(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0],), - (args[1],), - lambda ctx, arg0: [linalg.sqrt(arg0, outs=[args[1].get_zero_op(ctx)])], - ) - - -def ref_square( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return (torch.square(var[cache.opr[0]]),) - - -def mlir_square(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0],), - (args[1],), - lambda ctx, arg0: [linalg.square(arg0, outs=[args[1].get_zero_op(ctx)])], - ) - - -def ref_tanh( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return (torch.tanh(var[cache.opr[0]]),) - - -def mlir_tanh(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0],), - (args[1],), - lambda ctx, arg0: [linalg.tanh(arg0, outs=[args[1].get_zero_op(ctx)])], - ) diff --git a/test/benchgc/src/benchgc/linalg/generic.py b/test/benchgc/src/benchgc/linalg/generic.py deleted file mode 100644 index 891ba325e..000000000 --- a/test/benchgc/src/benchgc/linalg/generic.py +++ /dev/null @@ -1,142 +0,0 @@ -################################################################################ -# Copyright 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -from typing import Any, Dict, List, Tuple - -import benchgc.runner -import benchgc.util -import torch -from benchgc.mlir.util import MLIRCache -from gc_mlir import ir - - -def generic_loop( - cache: MLIRCache, - op: ir.OpView, - depth: int, - iterspace: Dict[str, Tuple[int, int, int]], - affine_from: List[str], - affine_to: List[List[str]], - var: Dict[str, torch.Tensor], - loop_var: Dict[str, torch.Tensor], - result_tensors: Tuple[torch.Tensor, ...], -): - if depth == len(affine_from): - # we need to execute the block here - # we will need to read the block argument name and save it into the cache - - if len(cache.next) == 0: - # region cache - cache.next.append(MLIRCache()) - - block: ir.Block = op.regions[0].blocks[0] - if len(cache.next[0].next) == 0: - # region->block cache - cache.next[0].next.append(MLIRCache()) - for arg in block.arguments: - cache.next[0].next[0].arg.append(arg.get_name()) - - block_cache = cache.next[0].next[0] - block_arg: Dict[str, torch.Tensor] = {} - for i in range(len(block.arguments)): - index: Tuple[int, ...] = tuple() - aff: List[str] = affine_to[i] - for d in aff: - index = index + (int(loop_var[d].item()),) - - if i + len(op.results) < len(op.regions[0].blocks[0].arguments): - # input argument - block_arg[block_cache.arg[i]] = var[cache.opr[i]][index] - else: - # output argument - block_arg[block_cache.arg[i]] = result_tensors[ - i + len(op.results) - len(block.arguments) - ][index] - - res: Tuple[Any, ...] = benchgc.runner.dfs_block( - cache.next[0].next[0], block, var | loop_var | block_arg - ) - - # perform the yield operation - for i in range(len(op.results)): - idx = -1 - i - aff: List[str] = affine_to[idx] - index: Tuple[int, ...] = tuple() - for d in aff: - index = index + (int(loop_var[d].item()),) - result_tensors[idx][index] = res[idx] - else: - it = iterspace[affine_from[depth]] - for i in range(it[0], it[1], it[2]): - loop_var[affine_from[depth]][0] = i - generic_loop( - cache, - op, - depth + 1, - iterspace, - affine_from, - affine_to, - var, - loop_var, - result_tensors, - ) - - -def ref_generic( - cache: MLIRCache, op: ir.OpView, tensors: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - affine_from: List[str] = [] - affine_to: List[List[str]] = [] - - for affine in op.attributes["indexing_maps"]: - aff = str(affine) - affine_from = aff[aff.find("<(") + 2 : aff.find(") ->")].split(", ") - affine_to.append(aff[aff.find("-> (") + 4 : aff.find(")>")].split(", ")) - - # try to find the iteration space - # TODO: support affine expression - - iterspace: Dict[str, Tuple[int, int, int]] = {} - operands: List[ir.OpOperand] = list(op.operands) - - loop_var: Dict[str, torch.Tensor] = {} - for d in affine_from: - iterspace[d] = (0, 0, 1) - loop_var[d] = torch.zeros(size=[1], dtype=torch.int) - - for i in range(len(operands)): - for j in range(len(operands[i].type.shape)): - iterspace[affine_to[i][j]] = (0, operands[i].type.shape[j], 1) - - result_tensors: Tuple[torch.Tensor, ...] = tuple() - # create the buffer for result tensors - for i in range(len(op.results)): - result_tensors = result_tensors + (tensors[cache.opr[-1 - i]].clone(),) - - generic_loop( - cache, - op, - 0, - iterspace, - affine_from, - affine_to, - tensors, - loop_var, - result_tensors, - ) - return result_tensors - - diff --git a/test/benchgc/src/benchgc/linalg/matmul.py b/test/benchgc/src/benchgc/linalg/matmul.py deleted file mode 100644 index 16ad0519c..000000000 --- a/test/benchgc/src/benchgc/linalg/matmul.py +++ /dev/null @@ -1,325 +0,0 @@ -################################################################################ -# Copyright 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -import argparse -from typing import Dict, List, Tuple - -import torch -from benchgc.arg import Arg -from benchgc.mlir.module import init_module -from benchgc.mlir.util import MLIRCache -from gc_mlir import ir -from gc_mlir.dialects import linalg -from gc_mlir.dialects.linalg.opdsl.lang.comprehension import TypeFnType - - -def ref_batch_matmul( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return (torch.matmul(var[cache.opr[0]], var[cache.opr[1]]),) - - -def mlir_batch_matmul(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.batch_matmul(arg0, arg1, outs=[args[2].get_zero_op(ctx)]) - ], - ) - - -def ref_batch_matmul_transpose_a( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return (torch.bmm(var[cache.opr[0]].transpose(-1, -2), var[cache.opr[1]]),) - - -def mlir_batch_matmul_transpose_a( - flags: argparse.Namespace, args: List[Arg] -) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.batch_matmul_transpose_a(arg0, arg1, outs=[args[2].get_zero_op(ctx)]) - ], - ) - - -def ref_batch_matmul_transpose_b( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return (torch.bmm(var[cache.opr[0]], var[cache.opr[1]].transpose(-1, -2)),) - - -def mlir_batch_matmul_transpose_b( - flags: argparse.Namespace, args: List[Arg] -) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.batch_matmul_transpose_b(arg0, arg1, outs=[args[2].get_zero_op(ctx)]) - ], - ) - - -def ref_batch_matvec( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - # pytorch does not support bmv - return ( - torch.matmul(var[cache.opr[0]], var[cache.opr[1]].unsqueeze(-1)).squeeze(-1), - ) - - -def mlir_batch_matvec(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.batch_matvec(arg0, arg1, outs=[args[2].get_zero_op(ctx)]) - ], - ) - - -def ref_batch_mmt4d( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - # [B, m, k, m0, k0] -> [B, m, m0, k, k0] - _src = var[cache.opr[0]].permute([0, 1, 3, 2, 4]).contiguous() - # [B, n, k, n0, k0] -> [B, k, k0, n, n0] - _wei = var[cache.opr[1]].permute([0, 2, 4, 1, 3]).contiguous() - - # [B, m, m0, k, k0] -> [B, M, K] - src = _src.reshape( - [_src.shape[0], _src.shape[1] * _src.shape[2], _src.shape[3] * _src.shape[4]] - ) - # [B, k, k0, n, n0] -> [B, K, N] - wei = _wei.reshape( - [_wei.shape[0], _wei.shape[1] * _wei.shape[2], _wei.shape[3] * _wei.shape[4]] - ) - - dst = torch.bmm(src, wei) - # [B, M, N] -> [B, m, m0, n, n0] - dst = dst.reshape( - [dst.shape[0], _src.shape[1], _src.shape[2], _wei.shape[-2], _wei.shape[-1]] - ) - - # [B, m, m0, n, n0] -> [B, m, n, m0, n0] - return (dst.transpose(2, 3).contiguous(),) - - -def mlir_batch_mmt4d(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.batch_mmt4d(arg0, arg1, outs=[args[2].get_zero_op(ctx)]) - ], - ) - - -def ref_batch_reduce_matmul( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return ( - torch.addbmm( - input=torch.zeros(tuple()), - batch1=var[cache.opr[0]], - batch2=var[cache.opr[1]], - beta=0, - alpha=1, - ), - ) - - -def mlir_batch_reduce_matmul(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.batch_reduce_matmul(arg0, arg1, outs=[args[2].get_zero_op(ctx)]) - ], - ) - - -def ref_batch_vecmat( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return ( - torch.matmul(var[cache.opr[0]].unsqueeze(-2), var[cache.opr[1]]).squeeze(-2), - ) - - -def mlir_batch_vecmat(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.batch_vecmat(arg0, arg1, outs=[args[2].get_zero_op(ctx)]) - ], - ) - - -def ref_dot( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return (torch.dot(var[cache.opr[0]], var[cache.opr[1]]),) - - -def mlir_dot(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.dot(arg0, arg1, outs=[args[2].get_zero_op(ctx)]) - ], - ) - - -def ref_matmul( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return (torch.mm(var[cache.opr[0]], var[cache.opr[1]]),) - - -def mlir_matmul(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.matmul( - arg0, arg1, outs=[args[2].get_zero_op(ctx)], cast=TypeFnType(flags.cast) - ) - ], - ) - - -def ref_matmul_transpose_a( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return (torch.mm(var[cache.opr[0]].transpose(-1, -2), var[cache.opr[1]]),) - - -def mlir_matmul_transpose_a(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.matmul_transpose_a( - arg0, arg1, outs=[args[2].get_zero_op(ctx)], cast=TypeFnType(flags.cast) - ) - ], - ) - - -def ref_matmul_transpose_b( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return (torch.mm(var[cache.opr[0]], var[cache.opr[1]].transpose(-1, -2)),) - - -def mlir_matmul_transpose_b(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.matmul_transpose_b( - arg0, arg1, outs=[args[2].get_zero_op(ctx)], cast=TypeFnType(flags.cast) - ) - ], - ) - - -def ref_matvec( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return (torch.mv(var[cache.opr[0]], var[cache.opr[1]]),) - - -def mlir_matvec(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.matvec(arg0, arg1, outs=[args[2].get_zero_op(ctx)]) - ], - ) - - -def ref_mmt4d( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - # [m, k, m0, k0] -> [m, m0, k, k0] - _src = var[cache.opr[0]].permute([0, 2, 1, 3]).contiguous() - # [n, k, n0, k0] -> [k, k0, n, n0] - _wei = var[cache.opr[1]].permute([1, 3, 0, 2]).contiguous() - - # [m, m0, k, k0] -> [M, K] - src = _src.reshape([_src.shape[0] * _src.shape[1], _src.shape[2] * _src.shape[3]]) - # [k, k0, n, n0] -> [K, N] - wei = _wei.reshape([_wei.shape[0] * _wei.shape[1], _wei.shape[2] * _wei.shape[3]]) - - dst = torch.mm(src, wei) - # [M, N] -> [m, m0, n, n0] - dst = dst.reshape([_src.shape[0], _src.shape[1], _wei.shape[-2], _wei.shape[-1]]) - - # [m, m0, n, n0] -> [m, n, m0, n0] - return (dst.transpose(1, 2).contiguous(),) - - -def mlir_mmt4d(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.mmt4d(arg0, arg1, outs=[args[2].get_zero_op(ctx)]) - ], - ) - - -def ref_vecmat( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return ( - torch.matmul(var[cache.opr[0]].unsqueeze(-2), var[cache.opr[1]]).squeeze(-2), - ) - - -def mlir_vecmat(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.vecmat(arg0, arg1, outs=[args[2].get_zero_op(ctx)]) - ], - ) diff --git a/test/benchgc/src/benchgc/linalg/misc.py b/test/benchgc/src/benchgc/linalg/misc.py deleted file mode 100644 index 6b899c5c5..000000000 --- a/test/benchgc/src/benchgc/linalg/misc.py +++ /dev/null @@ -1,119 +0,0 @@ -################################################################################ -# Copyright 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -import argparse -import copy -from typing import Dict, List, Tuple - -import benchgc.util -import torch -from benchgc.arg import Arg -from benchgc.mlir.module import init_module -from benchgc.mlir.util import MLIRCache -from gc_mlir import ir -from gc_mlir.dialects import linalg -from gc_mlir.dialects.linalg.opdsl.lang.comprehension import TypeFnType - - -# 1. use to reshape to match ndim -# 2. perform broadcast -def ref_broadcast( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - dst_shape: List[int] = op.results[0].type.shape - tmp_shape = copy.copy(dst_shape) - dimensions: ir.DenseI64ArrayAttr = op.attributes["dimensions"] - for d in dimensions: - tmp_shape[d] = 1 - - return (var[cache.opr[0]].reshape(tmp_shape).broadcast_to(dst_shape).contiguous(),) - - -def mlir_broadcast(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - - return init_module( - flags.entry, - (args[0],), - (args[1],), - lambda ctx, arg0: [ - linalg.broadcast( - arg0, outs=[args[1].get_zero_op(ctx)], dimensions=flags.dimensions - ) - ], - ) - - -def ref_fill( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return (torch.full(tuple(op.results[0].type.shape), var[cache.opr[0]]),) - - -def mlir_fill(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0],), - (args[1],), - lambda ctx, arg0: [ - linalg.fill( - arg0, outs=[args[1].get_zero_op(ctx)], dimensions=flags.dimensions - ) - ], - ) - - -def ref_copy( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return ( - var[cache.opr[0]] - .to(benchgc.util.get_dtype(str(op.result.type.element_type))) - .clone(), - ) - - -def mlir_copy(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - - return init_module( - flags.entry, - (args[0],), - (args[1],), - lambda ctx, arg0: [ - linalg.copy( - arg0, outs=[args[1].get_zero_op(ctx)], cast=TypeFnType(flags.cast) - ) - ], - ) - - -def ref_transpose( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - permutation: List[int] = [int(d) for d in op.attributes["permutation"]] - return (var[cache.opr[0]].permute(permutation).contiguous(),) - - -def mlir_transpose(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0],), - (args[1],), - lambda ctx, arg0: [ - linalg.transpose( - arg0, outs=[args[1].get_zero_op(ctx)], permutation=flags.permutation - ) - ], - ) diff --git a/test/benchgc/src/benchgc/linalg/pool.py b/test/benchgc/src/benchgc/linalg/pool.py deleted file mode 100644 index 755e4c76a..000000000 --- a/test/benchgc/src/benchgc/linalg/pool.py +++ /dev/null @@ -1,477 +0,0 @@ -################################################################################ -# Copyright 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -import argparse -from typing import Dict, List, Tuple - -import torch -from benchgc.arg import Arg -from benchgc.mlir.module import init_module -from benchgc.mlir.util import MLIRCache -from gc_mlir import ir -from gc_mlir.dialects import linalg - - -def ref_pooling_nchw_max( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - strides: ir.DenseIntElementsAttr = op.attributes["strides"] - dilations: ir.DenseIntElementsAttr = op.attributes["dilations"] - return ( - torch.max_pool2d( - var[cache.opr[0]], - kernel_size=var[cache.opr[1]].shape, - stride=[strides[i] for i in range(len(strides))], - dilation=[dilations[i] for i in range(len(dilations))], - ), - ) - - -def mlir_pooling_nchw_max(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.pooling_nchw_max( - arg0, - arg1, - outs=[args[2].get_min_value_op(ctx)], - strides=flags.strides, - dilations=flags.dilations, - ) - ], - ) - - -def ref_pooling_nchw_sum( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - strides: ir.DenseIntElementsAttr = op.attributes["strides"] - dilations: ir.DenseIntElementsAttr = op.attributes["dilations"] - - # pytorch does not support pooling on sum - # avg_pool2d or lp_pool2d with p = 1 does not support dilation - # we will use depthwise convolution with kernel equals to 1 to calculate the sum - - # FIXME: improve the code if pytorch support the sum pooling with dilation - - channel = var[cache.opr[0]].shape[1] - kernel = var[cache.opr[1]] - return ( - torch.conv2d( - var[cache.opr[0]], - torch.ones(channel, 1, kernel.shape[0], kernel.shape[1]), - stride=[strides[i] for i in range(len(strides))], - dilation=[dilations[i] for i in range(len(dilations))], - groups=channel, - ), - ) - - -def mlir_pooling_nchw_sum(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.pooling_nchw_sum( - arg0, - arg1, - outs=[args[2].get_zero_op(ctx)], - strides=flags.strides, - dilations=flags.dilations, - ) - ], - ) - - -def ref_pooling_ncw_max( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - strides: ir.DenseIntElementsAttr = op.attributes["strides"] - dilations: ir.DenseIntElementsAttr = op.attributes["dilations"] - return ( - torch.max_pool1d( - var[cache.opr[0]], - kernel_size=var[cache.opr[1]].shape, - stride=[strides[i] for i in range(len(strides))], - dilation=[dilations[i] for i in range(len(dilations))], - ), - ) - - -def mlir_pooling_ncw_max(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.pooling_ncw_max( - arg0, - arg1, - outs=[args[2].get_min_value_op(ctx)], - strides=flags.strides, - dilations=flags.dilations, - ) - ], - ) - - -def ref_pooling_ncw_sum( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - strides: ir.DenseIntElementsAttr = op.attributes["strides"] - dilations: ir.DenseIntElementsAttr = op.attributes["dilations"] - - # pytorch does not support pooling on sum - # avg_pool1d or lp_pool1d with p = 1 does not support dilation - # we will use depthwise convolution with kernel equals to 1 to calculate the sum - - # FIXME: improve the code if pytorch support the sum pooling with dilation - - channel = var[cache.opr[0]].shape[1] - kernel = var[cache.opr[1]] - return ( - torch.conv1d( - var[cache.opr[0]], - torch.ones(channel, 1, kernel.shape[0]), - stride=[strides[i] for i in range(len(strides))], - dilation=[dilations[i] for i in range(len(dilations))], - groups=channel, - ), - ) - - -def mlir_pooling_ncw_sum(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.pooling_ncw_sum( - arg0, - arg1, - outs=[args[2].get_zero_op(ctx)], - strides=flags.strides, - dilations=flags.dilations, - ) - ], - ) - - -def ref_pooling_ndhwc_max( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - strides: ir.DenseIntElementsAttr = op.attributes["strides"] - dilations: ir.DenseIntElementsAttr = op.attributes["dilations"] - return ( - torch.max_pool3d( - var[cache.opr[0]].permute([0, -1, 1, 2, 3]), - kernel_size=var[cache.opr[1]].shape, - stride=[strides[i] for i in range(len(strides))], - dilation=[dilations[i] for i in range(len(dilations))], - ) - .permute([0, 2, 3, 4, 1]) - .contiguous(), - ) - - -def mlir_pooling_ndhwc_max(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.pooling_ndhwc_max( - arg0, - arg1, - outs=[args[2].get_min_value_op(ctx)], - strides=flags.strides, - dilations=flags.dilations, - ) - ], - ) - - -def ref_pooling_ndhwc_sum( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - strides: ir.DenseIntElementsAttr = op.attributes["strides"] - dilations: ir.DenseIntElementsAttr = op.attributes["dilations"] - - # pytorch does not support pooling on sum - # avg_pool3d or lp_pool3d with p = 1 does not support dilation - # we will use depthwise convolution with kernel equals to 1 to calculate the sum - - # FIXME: improve the code if pytorch support the sum pooling with dilation - - channel = var[cache.opr[0]].shape[-1] - kernel = var[cache.opr[1]] - return ( - torch.conv3d( - var[cache.opr[0]].permute([0, -1, 1, 2, 3]), - torch.ones(channel, 1, kernel.shape[0], kernel.shape[1], kernel.shape[2]), - stride=[strides[i] for i in range(len(strides))], - dilation=[dilations[i] for i in range(len(dilations))], - groups=channel, - ) - .permute([0, 2, 3, 4, 1]) - .contiguous(), - ) - - -def mlir_pooling_ndhwc_sum(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.pooling_ndhwc_sum( - arg0, - arg1, - outs=[args[2].get_zero_op(ctx)], - strides=flags.strides, - dilations=flags.dilations, - ) - ], - ) - - -def ref_pooling_nhwc_max( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - strides: ir.DenseIntElementsAttr = op.attributes["strides"] - dilations: ir.DenseIntElementsAttr = op.attributes["dilations"] - return ( - torch.max_pool2d( - var[cache.opr[0]].permute([0, -1, 1, 2]), - kernel_size=var[cache.opr[1]].shape, - stride=[strides[i] for i in range(len(strides))], - dilation=[dilations[i] for i in range(len(dilations))], - ) - .permute([0, 2, 3, 1]) - .contiguous(), - ) - - -def mlir_pooling_nhwc_max(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.pooling_nhwc_max( - arg0, - arg1, - outs=[args[2].get_min_value_op(ctx)], - strides=flags.strides, - dilations=flags.dilations, - ) - ], - ) - - -def ref_pooling_nhwc_sum( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - strides: ir.DenseIntElementsAttr = op.attributes["strides"] - dilations: ir.DenseIntElementsAttr = op.attributes["dilations"] - - # pytorch does not support pooling on sum - # avg_pool2d or lp_pool2d with p = 1 does not support dilation - # we will use depthwise convolution with kernel equals to 1 to calculate the sum - - # FIXME: improve the code if pytorch support the sum pooling with dilation - - channel = var[cache.opr[0]].shape[-1] - kernel = var[cache.opr[1]] - return ( - torch.conv2d( - var[cache.opr[0]].permute([0, -1, 1, 2]), - torch.ones(channel, 1, kernel.shape[0], kernel.shape[1]), - stride=[strides[i] for i in range(len(strides))], - dilation=[dilations[i] for i in range(len(dilations))], - groups=channel, - ) - .permute([0, 2, 3, 1]) - .contiguous(), - ) - - -def mlir_pooling_nhwc_sum(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.pooling_nhwc_sum( - arg0, - arg1, - outs=[args[2].get_zero_op(ctx)], - strides=flags.strides, - dilations=flags.dilations, - ) - ], - ) - - -def ref_pooling_nhwc_min( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - strides: ir.DenseIntElementsAttr = op.attributes["strides"] - dilations: ir.DenseIntElementsAttr = op.attributes["dilations"] - return ( - torch.max_pool2d( - var[cache.opr[0]].permute([0, -1, 1, 2]).neg(), - kernel_size=var[cache.opr[1]].shape, - stride=[strides[i] for i in range(len(strides))], - dilation=[dilations[i] for i in range(len(dilations))], - ) - .permute([0, 2, 3, 1]) - .neg() - .contiguous(), - ) - - -def mlir_pooling_nhwc_min(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.pooling_nhwc_min( - arg0, - arg1, - outs=[args[2].get_max_value_op(ctx)], - strides=flags.strides, - dilations=flags.dilations, - ) - ], - ) - - -def ref_pooling_nwc_max( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - strides: ir.DenseIntElementsAttr = op.attributes["strides"] - dilations: ir.DenseIntElementsAttr = op.attributes["dilations"] - return ( - torch.max_pool1d( - var[cache.opr[0]].permute([0, -1, 1]), - kernel_size=var[cache.opr[1]].shape, - stride=[strides[i] for i in range(len(strides))], - dilation=[dilations[i] for i in range(len(dilations))], - ) - .permute([0, 2, 1]) - .contiguous(), - ) - - -def mlir_pooling_nwc_max(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.pooling_nwc_max( - arg0, - arg1, - outs=[args[2].get_min_value_op(ctx)], - strides=flags.strides, - dilations=flags.dilations, - ) - ], - ) - - -def ref_pooling_nwc_min( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - strides: ir.DenseIntElementsAttr = op.attributes["strides"] - dilations: ir.DenseIntElementsAttr = op.attributes["dilations"] - return ( - torch.max_pool1d( - var[cache.opr[0]].permute([0, -1, 1]).neg(), - kernel_size=var[cache.opr[1]].shape, - stride=[strides[i] for i in range(len(strides))], - dilation=[dilations[i] for i in range(len(dilations))], - ) - .permute([0, 2, 1]) - .contiguous() - .neg(), - ) - - -def mlir_pooling_nwc_min(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.pooling_nwc_min( - arg0, - arg1, - outs=[args[2].get_max_value_op(ctx)], - strides=flags.strides, - dilations=flags.dilations, - ) - ], - ) - - -def ref_pooling_nwc_sum( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - strides: ir.DenseIntElementsAttr = op.attributes["strides"] - dilations: ir.DenseIntElementsAttr = op.attributes["dilations"] - - # pytorch does not support pooling on sum - # avg_pool3d or lp_pool3d with p = 1 does not support dilation - # we will use depthwise convolution with kernel equals to 1 to calculate the sum - - # FIXME: improve the code if pytorch support the sum pooling with dilation - - channel = var[cache.opr[0]].shape[-1] - kernel = var[cache.opr[1]] - return ( - torch.conv1d( - var[cache.opr[0]].permute([0, -1, 1]), - torch.ones(channel, 1, kernel.shape[0]), - stride=[strides[i] for i in range(len(strides))], - dilation=[dilations[i] for i in range(len(dilations))], - groups=channel, - ) - .permute([0, 2, 1]) - .contiguous(), - ) - - -def mlir_pooling_nwc_sum(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0], args[1]), - (args[2],), - lambda ctx, arg0, arg1: [ - linalg.pooling_nwc_sum( - arg0, - arg1, - outs=[args[2].get_zero_op(ctx)], - strides=flags.strides, - dilations=flags.dilations, - ) - ], - ) diff --git a/test/benchgc/src/benchgc/linalg/reduce.py b/test/benchgc/src/benchgc/linalg/reduce.py deleted file mode 100644 index ea7430473..000000000 --- a/test/benchgc/src/benchgc/linalg/reduce.py +++ /dev/null @@ -1,278 +0,0 @@ -################################################################################ -# Copyright 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -import argparse -from typing import Callable, Dict, List, Tuple - -import benchgc.runner -import benchgc.util -import gc_mlir.ir as ir -import torch -from benchgc.arg.arg import Arg -from benchgc.mlir.module import init_reduce_module -from benchgc.mlir.util import MLIRCache -from gc_mlir.dialects import arith, math - - -def reduce_loop( - cache: MLIRCache, - op: ir.OpView, - depth: int, - in_shape: List[int], - var: Dict[str, torch.Tensor], - in_idx: List[int], - out_idx: List[int], - reduced_axis: int, - result_tensor: torch.Tensor, -): - if depth == len(in_shape): - # we need to execute the block here - # we will need to read the block argument name and save it into the cache - - block: ir.Block = op.regions[0].blocks[0] - - if len(cache.next) == 0: - # region cache - cache.next.append(MLIRCache()) - if len(cache.next[0].next) == 0: - # region->block cache - cache.next[0].next.append(MLIRCache()) - for arg in block.arguments: - cache.next[0].next[0].arg.append(arg.get_name()) - - block_arg: Dict[str, torch.Tensor] = { - # set input - cache.next[0].next[0].arg[0]: var[cache.opr[0]][tuple(in_idx)], - # set output - cache.next[0].next[0].arg[1]: result_tensor[tuple(out_idx)], - } - - res: Tuple[torch.Tensor, ...] = benchgc.runner.dfs_block( - cache.next[0].next[0], op.regions[0].blocks[0], var | block_arg - ) - - # perform the yield operation - result_tensor[tuple(out_idx)] = res[0] - else: - dimensions: ir.DenseI64ArrayAttr = op.attributes["dimensions"] - reduce_axis: bool = depth in list(dimensions) - - for i in range(in_shape[depth]): - if reduce_axis: - in_idx[depth] = i - reduce_loop( - cache, - op, - depth + 1, - in_shape, - var, - in_idx, - out_idx, - reduced_axis + 1, - result_tensor, - ) - else: - in_idx[depth] = i - out_idx[depth - reduced_axis] = i - reduce_loop( - cache, - op, - depth + 1, - in_shape, - var, - in_idx, - out_idx, - reduced_axis, - result_tensor, - ) - - -case_to_ops: Dict[str, List[str]] = { - "reduce.add": ["arith.addf", "arith.addi"], - "reduce.max": ["arith.maximumf", "arith.maxsi"], - "reduce.min": ["arith.minimumf", "arith.minsi"], - "reduce.mul": ["arith.mulf", "arith.muli"], -} - - -def check_commutative_operand(opr0: str, opr1: str, arg0: str, arg1: str) -> bool: - return opr0 == arg0 and opr1 == arg1 or opr0 == arg1 and opr1 == arg0 - - -# do not execute the block step by step if the pattern matches -def match_reduce_type(op: ir.OpView) -> str: - block = op.regions[0].blocks[0] - block_arg = block.arguments - arg0, arg1 = block_arg[0].get_name(), block_arg[1].get_name() - block_op = block.operations - if len(block_op) == 2 and len(block_op[0].operands) == 2: - # need to match the following pattern - # (%in: dt, %out: dt) { - # %0 = opcall %out, %in: dt / %in, %out - # linalg.yield %0: dt - # } - op0, op1 = block_op[0], block_op[1] - opr0, opr1 = op0.operands[0].get_name(), op0.operands[1].get_name() - - if ( - op1.name == "linalg.yield" - and op1.operands[0].get_name() == op0.result.get_name() - and check_commutative_operand(opr0, opr1, arg0, arg1) - ): - for case, ops in case_to_ops.items(): - if op0.name in ops: - return case - elif len(block_op) == 3 and len(block_op[1].operands) == 2: - op0, op1, op2 = block_op[0], block_op[1], block_op[2] - opr0, opr1 = op1.operands[0].get_name(), op1.operands[1].get_name() - # match complex reduce here - - # reduce L1 - # (%in: dt, %out: dt) { - # %0 = math.absi/absf %in - # %1 = arith.addf/addi %out, %0 / %0, %out - # linalg.yield %1: dt - # } - if ( - op0.name in ["math.absf", "math.absi"] - and op0.operands[0].get_name() == arg0 - and op1.name in ["arith.addf", "arith.addi"] - and check_commutative_operand(opr0, opr1, op0.result.get_name(), arg1) - and op2.name == "linalg.yield" - and op2.operands[0].get_name() == op1.result.get_name() - ): - return "reduce.l1" - - # reduce square L2 - # (%in: dt, %out: dt) { - # %0 = arith.muli/mulf %in, %in - # %1 = arith.addf/addi %out, %0 / %0, %out - # linalg.yield %1: dt - # } - - if ( - op0.name in ["arith.muli", "arith.mulf"] - and op0.operands[0].get_name() == arg0 - and op0.operands[1].get_name() == arg0 - and op1.name in ["arith.addf", "arith.addi"] - and check_commutative_operand(opr0, opr1, op0.result.get_name(), arg1) - and op2.name == "linalg.yield" - and op2.operands[0].get_name() == op1.result.get_name() - ): - return "reduce.l2_square" - # not matched - return "" - - -def ref_reduce( - cache: MLIRCache, op: ir.OpView, tensors: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - - reduce_type: str = match_reduce_type(op) - if reduce_type == "": - # create the buffer for result tensors - tensors[cache.res[0]] = tensors[cache.opr[-1]].clone() - in_shape: List[int] = list(op.operands[0].type.shape) - out_shape: List[int] = list(op.result.type.shape) - result_tensor: torch.Tensor = tensors[cache.opr[-1]].clone() - reduce_loop( - cache, - op, - 0, - in_shape, - tensors, - [0] * len(in_shape), - [0] * len(out_shape), - 0, - result_tensor, - ) - return (result_tensor,) - src = tensors[cache.opr[0]] - reduce_buf = tensors[cache.opr[1]] - dimensions: List[int] = [int(d) for d in op.attributes["dimensions"]] - if reduce_type == "reduce.add": - dst = reduce_buf + torch.sum(src, dimensions) - elif reduce_type == "reduce.max": - dst = torch.maximum(reduce_buf, torch.amax(src, dimensions)) - elif reduce_type == "reduce.min": - dst = torch.minimum(reduce_buf, torch.amin(src, dimensions)) - elif reduce_type == "reduce.mul": - dst = src - for d in dimensions: - dst = torch.prod(dst, d, keepdim=True) - dst = dst.reshape_as(reduce_buf) - dst = torch.mul(reduce_buf, dst) - elif reduce_type == "reduce.l1": - dst = torch.abs(src) - dst = reduce_buf + torch.sum(dst, dimensions) - elif reduce_type == "reduce.l2_square": - dst = torch.square(src) - dst = reduce_buf + torch.sum(dst, dimensions) - else: - raise Exception(f"Unsupport reduce type {reduce_type}") - return (dst,) - - -def mlir_reduce(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - - is_float: bool = benchgc.util.get_dtype(args[0].dtype).is_floating_point - is_signed: bool = benchgc.util.get_dtype(args[0].dtype).is_signed - - buf: Callable[[ir.Context], ir.OpView] | None = None - op: Callable[[ir.BlockArgument, ir.BlockArgument], ir.OpResult] | None = None - - if flags.case == "reduce.add" and is_float: - buf = lambda ctx: args[1].get_zero_op(ctx) - op = lambda arg0, arg1: arith.addf(arg0, arg1) - elif flags.case == "reduce.add": - buf = lambda ctx: args[1].get_zero_op(ctx) - op = lambda arg0, arg1: arith.addi(arg0, arg1) - elif flags.case == "reduce.mul" and is_float: - buf = lambda ctx: args[1].get_constant_op(ctx, 1) - op = lambda arg0, arg1: arith.mulf(arg0, arg1) - elif flags.case == "reduce.mul": - buf = lambda ctx: args[1].get_constant_op(ctx, 1) - op = lambda arg0, arg1: arith.muli(arg0, arg1) - elif flags.case == "reduce.max" and is_float: - buf = lambda ctx: args[1].get_min_value_op(ctx) - op = lambda arg0, arg1: arith.maximumf(arg0, arg1) - elif flags.case == "reduce.max" and is_signed: - buf = lambda ctx: args[1].get_min_value_op(ctx) - op = lambda arg0, arg1: arith.maxsi(arg0, arg1) - elif flags.case == "reduce.min" and is_float: - buf = lambda ctx: args[1].get_max_value_op(ctx) - op = lambda arg0, arg1: arith.minimumf(arg0, arg1) - elif flags.case == "reduce.min" and is_signed: - buf = lambda ctx: args[1].get_max_value_op(ctx) - op = lambda arg0, arg1: arith.minsi(arg0, arg1) - elif flags.case == "reduce.l1" and is_float: - buf = lambda ctx: args[1].get_zero_op(ctx) - op = lambda arg0, arg1: arith.addf(math.absf(arg0), arg1) - elif flags.case == "reduce.l1": - buf = lambda ctx: args[1].get_zero_op(ctx) - op = lambda arg0, arg1: arith.addi(math.absi(arg0), arg1) - elif flags.case == "reduce.l2_square" and is_float: - buf = lambda ctx: args[1].get_zero_op(ctx) - op = lambda arg0, arg1: arith.addf(arith.mulf(arg0, arg0), arg1) - elif flags.case == "reduce.l2_square": - buf = lambda ctx: args[1].get_zero_op(ctx) - op = lambda arg0, arg1: arith.addi(arith.muli(arg0, arg0), arg1) - else: - raise Exception( - f"not supported reduce case {flags.case} with data type {args[0].dtype}" - ) - - return init_reduce_module(flags.entry, args[0], args[1], flags.dimensions, buf, op) diff --git a/test/benchgc/src/benchgc/linalg/softmax.py b/test/benchgc/src/benchgc/linalg/softmax.py deleted file mode 100644 index e56376404..000000000 --- a/test/benchgc/src/benchgc/linalg/softmax.py +++ /dev/null @@ -1,48 +0,0 @@ -################################################################################ -# Copyright 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -import argparse -from typing import Dict, List, Tuple - -import torch -from benchgc.arg import Arg -from benchgc.mlir.module import init_module -from benchgc.mlir.util import MLIRCache -from gc_mlir import ir -from gc_mlir.dialects import linalg - - -def ref_softmax( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - dimension: ir.IntegerAttr = op.attributes["dimension"] - return (torch.softmax(var[cache.opr[0]], dimension.value),) - - -def mlir_softmax(flags: argparse.Namespace, args: List[Arg]) -> ir.Module: - return init_module( - flags.entry, - (args[0],), - (args[1],), - lambda ctx, arg0: [ - linalg.softmax( - result=[args[1].get_ranked_tensor_type(ctx)], - input=arg0, - output=args[1].get_zero_op(ctx), - dimension=flags.dimension, - ) - ], - ) diff --git a/test/benchgc/src/benchgc/math/CMakeLists.txt b/test/benchgc/src/benchgc/math/CMakeLists.txt deleted file mode 100644 index c2577a0ee..000000000 --- a/test/benchgc/src/benchgc/math/CMakeLists.txt +++ /dev/null @@ -1,22 +0,0 @@ -################################################################################ -# Copyright (C) 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions -# and limitations under the License. -# SPDX-License-Identifier: Apache-2.0 -################################################################################ - - -file(GLOB PYTHON_SCRIPTS "*.py") -foreach(PY_SCRIPT ${PYTHON_SCRIPTS}) - configure_file(${PY_SCRIPT} ${CMAKE_BINARY_DIR}/test/benchgc/src/benchgc/math/ COPYONLY) -endforeach() diff --git a/test/benchgc/src/benchgc/math/__init__.py b/test/benchgc/src/benchgc/math/__init__.py deleted file mode 100644 index 3dc9014fe..000000000 --- a/test/benchgc/src/benchgc/math/__init__.py +++ /dev/null @@ -1,45 +0,0 @@ -################################################################################ -# Copyright 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -import argparse -import importlib -from typing import Callable, Dict, List, Tuple - -import gc_mlir.ir -import torch -from benchgc.arg import Arg -from benchgc.mlir.util import MLIRCache - -ref_op: Dict[ - str, - Callable[ - [MLIRCache, gc_mlir.ir.OpView, Dict[str, torch.Tensor]], - Tuple[torch.Tensor, ...], - ], -] = {} -mlir_op: Dict[ - str, Callable[[argparse.Namespace, List[Arg], List[Arg]], gc_mlir.ir.Module] -] = {} - -for dri in ["piecewise"]: - mod = importlib.import_module(f"benchgc.math.{dri}") - for key in mod.__dict__: - if key.startswith("ref_"): - op: str = key.removeprefix("ref_") - ref_op[op] = mod.__dict__[key] - if key.startswith("mlir_"): - op: str = key.removeprefix("mlir_") - mlir_op[op] = mod.__dict__[key] diff --git a/test/benchgc/src/benchgc/math/piecewise.py b/test/benchgc/src/benchgc/math/piecewise.py deleted file mode 100644 index 9ca458f12..000000000 --- a/test/benchgc/src/benchgc/math/piecewise.py +++ /dev/null @@ -1,33 +0,0 @@ -################################################################################ -# Copyright 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -from typing import Dict, Tuple - -import gc_mlir.ir as ir -import torch -from benchgc.mlir.util import MLIRCache - - -def ref_absf( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return (torch.abs(var[cache.opr[0]]),) - - -def ref_absi( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return (torch.abs(var[cache.opr[0]]),) diff --git a/test/benchgc/src/benchgc/mlir/CMakeLists.txt b/test/benchgc/src/benchgc/mlir/CMakeLists.txt deleted file mode 100644 index 5f8d589b4..000000000 --- a/test/benchgc/src/benchgc/mlir/CMakeLists.txt +++ /dev/null @@ -1,22 +0,0 @@ -################################################################################ -# Copyright (C) 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions -# and limitations under the License. -# SPDX-License-Identifier: Apache-2.0 -################################################################################ - - -file(GLOB PYTHON_SCRIPTS "*.py") -foreach(PY_SCRIPT ${PYTHON_SCRIPTS}) - configure_file(${PY_SCRIPT} ${CMAKE_BINARY_DIR}/test/benchgc/src/benchgc/mlir/ COPYONLY) -endforeach() diff --git a/test/benchgc/src/benchgc/mlir/__init__.py b/test/benchgc/src/benchgc/mlir/__init__.py deleted file mode 100644 index 4d3e897ce..000000000 --- a/test/benchgc/src/benchgc/mlir/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -################################################################################ -# Copyright 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ diff --git a/test/benchgc/src/benchgc/mlir/arg.py b/test/benchgc/src/benchgc/mlir/arg.py deleted file mode 100644 index 2fb0a9871..000000000 --- a/test/benchgc/src/benchgc/mlir/arg.py +++ /dev/null @@ -1,163 +0,0 @@ -################################################################################ -# Copyright 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -import ctypes -from typing import Any, List - -import benchgc.util -import gc_mlir.dialects.arith -import gc_mlir.dialects.linalg -import gc_mlir.dialects.tensor -import torch -from benchgc.mlir.util import dtype_to_ctype, str_to_mlir_dtype, str_to_mlir_typed_attr -from gc_mlir import ir - - -# scalar should give a address -# map torch.Tensor -> memref -# map int address -> scalar value -def get_mlir_args(args: List[torch.Tensor | int]): - mlir_args: List[Any] = [] - - for arg in args: - if isinstance(arg, torch.Tensor): - mlir_args.append(ctypes.pointer(ctypes.pointer(get_md(arg)))) - else: - mlir_args.append(ctypes.c_void_p(arg)) - - return mlir_args - - -def get_md(tensor: torch.Tensor): - if tensor.ndim == 0: - - class _0dMemrefDescriptor(ctypes.Structure): - _fields_ = [ - ("allocated", ctypes.c_longlong), - ("aligned", ctypes.POINTER(dtype_to_ctype(tensor.dtype))), - ("offset", ctypes.c_longlong), - ] - - md = _0dMemrefDescriptor() - else: - ctype_shape = ctypes.c_longlong * tensor.ndim - ctype_strides = ctypes.c_longlong * tensor.ndim - - class _ndMemrefDescriptor(ctypes.Structure): - _fields_ = [ - ("allocated", ctypes.c_longlong), - ("aligned", ctypes.POINTER(dtype_to_ctype(tensor.dtype))), - ("offset", ctypes.c_longlong), - ("shape", ctype_shape), - ("strides", ctype_strides), - ] - - md = _ndMemrefDescriptor() - md.shape = ctype_shape(*tensor.shape) - md.strides = ctype_strides(*tensor.stride()) - - md.allocated = tensor.data_ptr() - md.aligned = ctypes.cast( - ctypes.c_void_p(tensor.data_ptr()), ctypes.POINTER(dtype_to_ctype(tensor.dtype)) - ) - md.offset = ctypes.c_longlong(0) - return md - - -class MLIRArg: - dtype: str - shape: List[int] - - scalar: bool - - def __init__(self) -> None: - self.dtype = "" - - # md format: - # 0d memref/tensor: 0xf32 - # nd memref/tensor: 2x3xf32 - # scalar: f32 - def set_md(self, md: str): - splited: List[str] = md.split("x") - self.dtype = splited[-1] - self.shape = [] - - self.shape = [int(x) for x in splited[:-1]] - self.set_scalar() - - def set_scalar(self): - # use 0xf32 to represent memref - # use f32 to represent f32 - if self.shape == [0]: - self.shape = [] - self.scalar = False - elif self.shape == []: - self.scalar = True - else: - self.scalar = False - - def nelem(self) -> int: - if self.scalar or self.shape == [] or self.shape[0] == 0: - return 1 - ret: int = 1 - for dim in self.shape: - ret = ret * dim - return ret - - def get_mlir_type(self, ctx: ir.Context) -> ir.Type: - if self.scalar: - return str_to_mlir_dtype(ctx, self.dtype) - else: - return ir.RankedTensorType.get( - self.shape, str_to_mlir_dtype(ctx, self.dtype) - ) - - def get_ranked_tensor_type(self, ctx: ir.Context) -> ir.RankedTensorType: - return ir.RankedTensorType.get(self.shape, str_to_mlir_dtype(ctx, self.dtype)) - - def get_constant_op(self, ctx: ir.Context, cst: Any) -> ir.OpView: - zero = gc_mlir.dialects.arith.ConstantOp( - value=str_to_mlir_typed_attr(ctx, self.dtype, cst), - result=str_to_mlir_dtype(ctx, self.dtype), - ) - if self.scalar: - return zero - else: - return gc_mlir.dialects.linalg.fill( - zero, - outs=[ - gc_mlir.dialects.tensor.EmptyOp( - self.shape, str_to_mlir_dtype(ctx, self.dtype) - ) - ], - ) - - def get_zero_op(self, ctx: ir.Context) -> ir.OpView: - return self.get_constant_op(ctx, 0) - - def get_max_value_op(self, ctx: ir.Context) -> ir.OpView: - dtype = benchgc.util.get_dtype(self.dtype) - if dtype.is_floating_point: - return self.get_constant_op(ctx, torch.finfo(dtype).max) - else: - return self.get_constant_op(ctx, torch.iinfo(dtype).max) - - def get_min_value_op(self, ctx: ir.Context) -> ir.OpView: - dtype = benchgc.util.get_dtype(self.dtype) - if dtype.is_floating_point: - return self.get_constant_op(ctx, torch.finfo(dtype).min) - else: - return self.get_constant_op(ctx, torch.iinfo(dtype).min) diff --git a/test/benchgc/src/benchgc/mlir/module.py b/test/benchgc/src/benchgc/mlir/module.py deleted file mode 100644 index 98addf941..000000000 --- a/test/benchgc/src/benchgc/mlir/module.py +++ /dev/null @@ -1,86 +0,0 @@ -################################################################################ -# Copyright 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -from typing import Callable, List, Tuple - -from benchgc.mlir.arg import MLIRArg -from benchgc.mlir.util import str_to_mlir_dtype -from gc_mlir import ir -from gc_mlir.dialects import func, linalg - - -def init_module( - entry_name: str, - inputs: Tuple[MLIRArg, ...], - outputs: Tuple[MLIRArg, ...], - op_func: Callable[ - [ir.Context, Tuple[ir.BlockArgument, ...]], - List[ir.OpResult], - ], -) -> ir.Module: - with ir.Context() as ctx, ir.Location.unknown(): - module = ir.Module.create() - with ir.InsertionPoint(module.body): - f = func.FuncOp( - name=entry_name, - type=ir.FunctionType.get( - inputs=[x.get_mlir_type(ctx) for x in inputs], - results=[x.get_mlir_type(ctx) for x in outputs], - ), - ) - with ir.InsertionPoint(f.add_entry_block()): - block_args = f.entry_block.arguments - func.ReturnOp(op_func(ctx, *block_args)) - return module - - -def init_reduce_module( - entry_name: str, - input: MLIRArg, - output: MLIRArg, - dimensions: List[int], - reduce_buf: Callable[[ir.Context], ir.OpView], - reduce_func: Callable[[ir.BlockArgument, ir.BlockArgument], ir.OpResult], -) -> ir.Module: - with ir.Context() as ctx, ir.Location.unknown(): - module = ir.Module.create() - with ir.InsertionPoint(module.body): - f = func.FuncOp( - name=entry_name, - type=ir.FunctionType.get( - inputs=[input.get_mlir_type(ctx)], - results=[output.get_mlir_type(ctx)], - ), - ) - with ir.InsertionPoint(f.add_entry_block()): - block_args = f.entry_block.arguments - - # op_empty = outputs[0].get_zero_op(ctx) - op_reduce = linalg.ReduceOp( - result=[output.get_ranked_tensor_type(ctx)], - inputs=[block_args[0]], - inits=[reduce_buf(ctx)], - dimensions=dimensions, - ) - block = op_reduce.regions[0].blocks.append( - str_to_mlir_dtype(ctx, input.dtype), - str_to_mlir_dtype(ctx, output.dtype), - ) - with ir.InsertionPoint(block): - arg0, arg1 = block.arguments - linalg.YieldOp([reduce_func(arg0, arg1)]) - func.ReturnOp(op_reduce) - return module diff --git a/test/benchgc/src/benchgc/mlir/util.py b/test/benchgc/src/benchgc/mlir/util.py deleted file mode 100644 index 9ff5b8f45..000000000 --- a/test/benchgc/src/benchgc/mlir/util.py +++ /dev/null @@ -1,202 +0,0 @@ -################################################################################ -# Copyright 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -import argparse -import ctypes -import os -from typing import Any, List - -import torch -from gc_mlir import ir -from gc_mlir.dialects import arith, func, memref -from gc_mlir.tools import cpuinfo - - -# calling python binding consumes a lot of time e.g. get_name() -# we need to cache some result to avoid duplicate call -class MLIRCache: - # operand name cache - opr: List[str] - # result name cache - res: List[str] - # argument name cache - arg: List[str] - # next hierarchy - next = [] # List[Self] - - def __init__(self): - self.opr = [] - self.res = [] - self.arg = [] - self.next = [] - - -def dtype_to_ctype(dtype: torch.dtype): - if dtype == torch.float32: - return ctypes.c_float - elif dtype == torch.float64: - return ctypes.c_double - elif dtype == torch.int32: - return ctypes.c_int - elif dtype == torch.int64: - return ctypes.c_longlong - elif dtype == torch.uint8: - return ctypes.c_ubyte - elif dtype == torch.int8: - return ctypes.c_byte - elif dtype == torch.int16 or dtype == torch.bfloat16 or torch.float16: - return ctypes.c_short - elif dtype == torch.bool: - return ctypes.c_bool - elif dtype == torch.uint32: - return ctypes.c_uint32 - else: - raise ValueError(f"Unsupported torch dtype: {dtype}") - - -def str_to_mlir_dtype(ctx: ir.Context, dtype: str) -> ir.Type: - if dtype == "f32": - return ir.F32Type.get(ctx) - elif dtype == "f64": - return ir.F64Type.get(ctx) - elif dtype == "f16": - return ir.F16Type.get(ctx) - elif dtype == "bf16": - return ir.BF16Type.get(ctx) - elif dtype == "u8": - return ir.IntegerType.get_unsigned(8, ctx) - elif dtype == "s8": - return ir.IntegerType.get_signed(8, ctx) - elif dtype == "boolean": - return ir.IntegerType.get_unsigned(1, ctx) - elif dtype == "f8_e4m3": - return ir.Float8E4M3FNType.get(ctx) - elif dtype == "f8_e5m2": - return ir.Float8E5M2Type.get(ctx) - elif dtype == "s32": - return ir.IntegerType.get_signed(32, ctx) - elif dtype == "i32": - return ir.IntegerType.get_signless(32, ctx) - elif dtype == "u32": - return ir.IntegerType.get_unsigned(32, ctx) - else: - raise Exception(f"data type not support: {dtype}") - - -def str_to_mlir_typed_attr(ctx: ir.Context, dtype: str, value: Any) -> ir.Attribute: - mlir_dtype = str_to_mlir_dtype(ctx, dtype) - if dtype in ["f32", "f64", "bf16", "f16", "f8_e4m3", "f8_e5m2"]: - return ir.FloatAttr.get(mlir_dtype, value) - elif dtype in ["u8", "s8", "s32", "i32", "u32"]: - return ir.IntegerAttr.get(mlir_dtype, value) - elif dtype == "boolean": - return ir.BoolAttr.get(value) - else: - raise Exception(f"data type not support: {dtype}") - - -def emit_nano_time() -> func.FuncOp: - """Emit a nanoTime function that returns the current time in nanoseconds.""" - nanoTime = func.FuncOp( - "nanoTime", ([], [ir.IntegerType.get_signless(64)]), visibility="private" - ) - nanoTime.attributes["llvm.emit_c_interface"] = ir.UnitAttr.get() - return nanoTime - - -def emit_benchmark_wrapped_main_func( - kernel_func: func.FuncOp, timer_func: func.FuncOp -) -> func.FuncOp: - """Emit a wrapped main function that calls the kernel function and records the time taken.""" - memref_of_i64_type = ir.MemRefType.get([1], ir.IntegerType.get_signless(64)) - wrapped_func_name = "wrapped_main" - assert wrapped_func_name != str( - kernel_func.name - ), "wrapped function name should be different from kernel function name" - wrapped_func = func.FuncOp( - wrapped_func_name, - ([memref_of_i64_type] + kernel_func.arguments.types, kernel_func.type.results), - visibility="public", - ) - wrapped_func.attributes["llvm.emit_c_interface"] = ir.UnitAttr.get() - with ir.InsertionPoint(wrapped_func.add_entry_block()): - timer_buffer = wrapped_func.arguments[0] - start = func.CallOp(timer_func, []) - call_op = func.CallOp( - kernel_func, - list(wrapped_func.arguments[1:]), - ) - end = func.CallOp(timer_func, []) - time_taken = arith.SubIOp(end, start) - zero = arith.ConstantOp.create_index(0) - memref.StoreOp(time_taken, timer_buffer, [zero]) - func.ReturnOp(call_op.results) - return wrapped_func - - -def get_kernel_func_from_module( - module: ir.Module, func_name: str = "entry" -) -> func.FuncOp: - """Get the func op by the name from a module""" - for f in module.operation.regions[0].blocks[0].operations: - if type(f) is func.FuncOp and str(f.name).strip('"') == func_name: - return f - raise ValueError("can not find the entry function") - - -def attach_dlti(flags: argparse.Namespace, module: ir.Module): - # the moudle already had dlti attr - if "dlti.target_system_spec" in module.operation.attributes: - return - if flags.cpu_cache_sizes: - caches_sizes = [int(x) for x in flags.cpu_cache_sizes.strip().split(":")] - else: - caches_sizes = cpuinfo.get_cache_sizes() - if not caches_sizes or len(caches_sizes) != 3: - print( - "Failed to get CPU cache sizes, please added them manually br --cpu_cache_sizes" - ) - return - if flags.max_vector_width: - max_vector_width = flags.max_vector_width - else: - max_vector_width = cpuinfo.get_max_vector_width() - if not max_vector_width: - print( - "Failed to get CPU max vector width, please added them manually br --max_vector_width" - ) - return - l1_data_cache_size, l2_cache_size, l3_cache_size = caches_sizes - if "OMP_NUM_THREADS" not in os.environ: - print("OMP_NUM_THREADS is not found, using 1 as default") - num_threads = os.environ.get("OMP_NUM_THREADS", 1) - - dlti_template = f""" - module attributes {{ - dlti.target_system_spec = #dlti.target_system_spec< - "CPU": #dlti.target_device_spec< - #dlti.dl_entry<"L1_cache_size_in_bytes", {l1_data_cache_size} : ui32>, - #dlti.dl_entry<"L2_cache_size_in_bytes", {l2_cache_size} : ui64>, - #dlti.dl_entry<"L3_cache_size_in_bytes", {l3_cache_size} : ui64>, - #dlti.dl_entry<"num_threads", {num_threads} : i32>, - #dlti.dl_entry<"max_vector_width", {max_vector_width} : i64>> - >}} {{}} - """ - with module.context: - template_module = ir.Module.parse(dlti_template) - module.operation.attributes["dlti.target_system_spec"] = ( - template_module.operation.attributes["dlti.target_system_spec"] - ) diff --git a/test/benchgc/src/benchgc/pattern/CMakeLists.txt b/test/benchgc/src/benchgc/pattern/CMakeLists.txt deleted file mode 100644 index 51683ac19..000000000 --- a/test/benchgc/src/benchgc/pattern/CMakeLists.txt +++ /dev/null @@ -1,22 +0,0 @@ -################################################################################ -# Copyright (C) 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions -# and limitations under the License. -# SPDX-License-Identifier: Apache-2.0 -################################################################################ - - -file(GLOB PYTHON_SCRIPTS "*.py") -foreach(PY_SCRIPT ${PYTHON_SCRIPTS}) - configure_file(${PY_SCRIPT} ${CMAKE_BINARY_DIR}/test/benchgc/src/benchgc/pattern/ COPYONLY) -endforeach() \ No newline at end of file diff --git a/test/benchgc/src/benchgc/pattern/__init__.py b/test/benchgc/src/benchgc/pattern/__init__.py deleted file mode 100644 index b0f5cbce0..000000000 --- a/test/benchgc/src/benchgc/pattern/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -################################################################################ -# Copyright 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -from .base import Pattern -from .mlp import MLP - -__all__ = ["Pattern", "MLP", "get_pattern_clz"] - - -def get_pattern_clz(name: str): - """Function getting pattern class by name.""" - clz = {"mlp": MLP}[name] - return clz diff --git a/test/benchgc/src/benchgc/pattern/base.py b/test/benchgc/src/benchgc/pattern/base.py deleted file mode 100644 index 42527efc3..000000000 --- a/test/benchgc/src/benchgc/pattern/base.py +++ /dev/null @@ -1,43 +0,0 @@ -################################################################################ -# Copyright (C) 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions -# and limitations under the License. -# SPDX-License-Identifier: Apache-2.0 -################################################################################ - -import argparse -from abc import ABC, abstractmethod - -from gc_mlir import ir - - -class Pattern(ABC): - """Abstract class for pattern.""" - - @staticmethod - @abstractmethod - def add_args(parser: argparse.ArgumentParser): - """Add arguments to parser""" - - @abstractmethod - def handle_args(self, args: argparse.Namespace): - """Get and handle the args""" - - def __init__(self, ctx: ir.Context, flags: argparse.Namespace): - self.main_entry = flags.entry - self.handle_args(flags) - self.ir_module = self.init_module(ctx) - - @abstractmethod - def init_module(self, ctx: ir.Context) -> ir.Module: - """Create MLIR moudule by args""" diff --git a/test/benchgc/src/benchgc/pattern/mlp.py b/test/benchgc/src/benchgc/pattern/mlp.py deleted file mode 100644 index 13e60ac9f..000000000 --- a/test/benchgc/src/benchgc/pattern/mlp.py +++ /dev/null @@ -1,173 +0,0 @@ -################################################################################ -# Copyright (C) 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions -# and limitations under the License. -# SPDX-License-Identifier: Apache-2.0 -################################################################################ - -import argparse -from re import U -from typing import List - -from benchgc.arg.arg import Arg -from benchgc.mlir.util import str_to_mlir_dtype -from benchgc.util import to_bool_list, to_int_list -from gc_mlir import ir -from gc_mlir.dialects import arith, func, linalg, tensor - -from .base import Pattern - - -class MLP(Pattern): - @staticmethod - def add_args(parser: argparse.ArgumentParser): - parser.add_argument("--batch_size", type=int, default=1) - parser.add_argument("--hidden_size_list", type=str, default="") - parser.add_argument("--has_bias", required=False, type=str) - parser.add_argument( - "--act_type", type=str, choices=["noop", "relu", "sigmoid"], default="noop" - ) - parser.add_argument( - "--dtype", - type=str, - choices=[ - "f32", - "bf16", - ], - default="f32", - ) - - def handle_args(self, args: argparse.Namespace): - self.batch_size = args.batch_size - assert self.batch_size > 0, "batch size should be greater than 0" - - self.hidden_size_list = to_int_list(args.hidden_size_list) - layers = len(self.hidden_size_list) - 1 - assert layers >= 1, "hidden_size_list should have at least 2 elements" - - self.has_bias = ( - [False] * layers if args.has_bias is None else to_bool_list(args.has_bias) - ) - - assert ( - len(self.has_bias) == layers - ), "has_bias should have the same length as hidden_size_list" - - self.act_type = args.act_type - self.dtype = args.dtype - - def init_module(self, ctx: ir.Context) -> ir.Module: - with ctx, ir.Location.unknown(): - layers = len(self.hidden_size_list) - 1 - module = ir.Module.create() - dtype = str_to_mlir_dtype(ctx, self.dtype) - src = ir.RankedTensorType.get( - [self.batch_size, self.hidden_size_list[0]], dtype - ) - weights = [] - bias = [] - for i in range(layers): - weights.append( - ir.RankedTensorType.get( - [ - self.hidden_size_list[i], - self.hidden_size_list[i + 1], - ], - dtype, - ) - ) - if self.has_bias[i]: - bias.append( - ir.RankedTensorType.get([self.hidden_size_list[i + 1]], dtype) - ) - result = ir.RankedTensorType.get( - [ - self.batch_size, - self.hidden_size_list[-1], - ], - dtype, - ) - with ir.InsertionPoint(module.body): - f = func.FuncOp( - name=self.main_entry, - type=ir.FunctionType.get( - inputs=[src] + weights + bias, results=[result] - ), - ) - with ir.InsertionPoint(f.add_entry_block()): - data = f.entry_block.arguments[0] - bias_idx = len(weights) + 1 - zero = arith.ConstantOp( - value=ir.FloatAttr.get(dtype, 0.0), result=dtype - ).result - for i in range(layers): - weight = f.entry_block.arguments[i + 1] - if self.has_bias[i]: - bias = f.entry_block.arguments[bias_idx] - bias_idx += 1 - else: - bias = None - layer_out_shape = [ - self.batch_size, - self.hidden_size_list[i + 1], - ] - out = linalg.fill( - zero, outs=[tensor.EmptyOp(layer_out_shape, dtype)] - ) - data = linalg.matmul(data, weight, outs=[out]) - if bias: - broadcast_bias = linalg.broadcast( - bias, - outs=[tensor.EmptyOp(layer_out_shape, dtype)], - dimensions=[0], - ) - data = linalg.add( - data, - broadcast_bias, - outs=[tensor.EmptyOp(layer_out_shape, dtype)], - ) - - if self.act_type == "relu": - element = ir.FloatAttr.get(dtype, 0) - tensor_type = ir.RankedTensorType.get( - layer_out_shape, dtype - ) - attr = ir.DenseElementsAttr.get_splat(tensor_type, element) - cst = arith.ConstantOp(tensor_type, attr) - data = linalg.max( - data, cst, outs=[tensor.EmptyOp(layer_out_shape, dtype)] - ) - func.ReturnOp([data]) - return module - - def default_fill( - flags: argparse.Namespace, - arg: Arg, - arglist: List[Arg], - ): - arg.fill_type = "U" - arg.fill_param = ["0", "1"] - - def default_compare( - flags: argparse.Namespace, - arg: Arg, - arglist: List[Arg], - ): - arg.cmp_type = "P" - if arg.dtype == "f32": - arg.cmp_param = [str(1e-5)] - elif arg.dtype == "bf16": - arg.cmp_param = [str(5e-2)] - else: - raise Exception("Unsupported dtype for mlp pattern") - arg.cmp_param.append("100.0") diff --git a/test/benchgc/src/benchgc/pattern/util.py b/test/benchgc/src/benchgc/pattern/util.py deleted file mode 100644 index 62bf74ca5..000000000 --- a/test/benchgc/src/benchgc/pattern/util.py +++ /dev/null @@ -1,48 +0,0 @@ -################################################################################ -# Copyright (C) 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions -# and limitations under the License. -# SPDX-License-Identifier: Apache-2.0 -################################################################################ - -from typing import List - - -def to_int_list(s: str) -> List[int]: - """ - Parsing the cmd for list of int values - - Args: - s (str): int values in cmd, example: 2x3x4 - - Returns: - List[int]: int values in list, example: [2, 3, 4] - """ - if not s or len(s) == 0: - return [] - return [int(i) for i in s.strip().split("x")] - - -def to_bool_list(s: str) -> List[bool]: - """ - Parsing the cmd for list of bool values - - Args: - s (str): bools in cmd, example: 1x0x1 - - Returns: - List[bool]: bools in list, example: [True, False, True] - """ - if not s or len(s) == 0: - return [] - return [bool(int(i)) for i in s.strip().split("x")] diff --git a/test/benchgc/src/benchgc/runner.py b/test/benchgc/src/benchgc/runner.py deleted file mode 100644 index 06eae595a..000000000 --- a/test/benchgc/src/benchgc/runner.py +++ /dev/null @@ -1,112 +0,0 @@ -################################################################################ -# Copyright 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -from typing import Dict, Tuple - -import gc_mlir._mlir_libs -import gc_mlir.dialects -import gc_mlir.dialects.func -import torch -from benchgc.arith import ref_op as arith_ref_op -from benchgc.linalg import ref_op as linalg_ref_op -from benchgc.math import ref_op as math_ref_op -from benchgc.mlir.util import MLIRCache -from benchgc.tensor import ref_op as tensor_ref_op -from gc_mlir import ir - - -def dfs_op( - cache: MLIRCache, op: ir.OpView, tensors: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - - dialect_call: str = str(op.name) - if dialect_call in ["func.return", "linalg.yield"]: - ret: Tuple[torch.Tensor, ...] = tuple() - for name in cache.opr: - ret = ret + (tensors[name],) - return ret - if dialect_call.startswith("linalg"): - ref_op = linalg_ref_op - elif dialect_call.startswith("tensor"): - ref_op = tensor_ref_op - elif dialect_call.startswith("arith"): - ref_op = arith_ref_op - elif dialect_call.startswith("math"): - ref_op = math_ref_op - else: - build_cache = len(cache.next) == 0 - for i in range(len(op.regions)): - if build_cache: - # we do not need to cache things for region - # keep an empty cache - cache.next.append(MLIRCache()) - ret = dfs_region(cache.next[i], op.regions[i], tensors) - if len(ret) != 0: - return ret - return tuple() - - dialect_op: str = dialect_call.split(".")[1] - if dialect_op not in ref_op: - raise Exception(f"unknown op call {dialect_call}") - ref_func = ref_op[dialect_op] - for i, res in enumerate(ref_func(cache, op, tensors)): - tensors[cache.res[i]] = res - return tuple() - - -def dfs_region( - cache: MLIRCache, region: ir.Region, tensors: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - build_cache = len(cache.next) == 0 - for i in range(len(region.blocks)): - if build_cache: - _cache = MLIRCache() - # we need to cache argument name for block object - for arg in region.blocks[i].arguments: - _cache.arg.append(arg.get_name()) - cache.next.append(_cache) - ret = dfs_block(cache.next[i], region.blocks[i], tensors) - if len(ret) != 0: - return ret - return tuple() - - -def dfs_block( - cache: MLIRCache, block: ir.Block, tensors: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - build_cache = len(cache.next) == 0 - for i in range(len(block.operations)): - if build_cache: - _cache = MLIRCache() - # we need to cache operand name and result name - for opr in block.operations[i].operands: - _cache.opr.append(opr.get_name()) - - for res in block.operations[i].results: - _cache.res.append(res.get_name()) - cache.next.append(_cache) - - ret = dfs_op(cache.next[i], block.operations[i], tensors) - if len(ret) != 0: - return ret - return tuple() - - -def ref_run( - entry: gc_mlir.dialects.func.FuncOp, tensors: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - # cache some information of block & op - return dfs_op(MLIRCache(), entry, tensors) diff --git a/test/benchgc/src/benchgc/tensor/CMakeLists.txt b/test/benchgc/src/benchgc/tensor/CMakeLists.txt deleted file mode 100644 index 7b1b990dc..000000000 --- a/test/benchgc/src/benchgc/tensor/CMakeLists.txt +++ /dev/null @@ -1,22 +0,0 @@ -################################################################################ -# Copyright (C) 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions -# and limitations under the License. -# SPDX-License-Identifier: Apache-2.0 -################################################################################ - - -file(GLOB PYTHON_SCRIPTS "*.py") -foreach(PY_SCRIPT ${PYTHON_SCRIPTS}) - configure_file(${PY_SCRIPT} ${CMAKE_BINARY_DIR}/test/benchgc/src/benchgc/tensor/ COPYONLY) -endforeach() diff --git a/test/benchgc/src/benchgc/tensor/__init__.py b/test/benchgc/src/benchgc/tensor/__init__.py deleted file mode 100644 index 9bfd9b09d..000000000 --- a/test/benchgc/src/benchgc/tensor/__init__.py +++ /dev/null @@ -1,43 +0,0 @@ -################################################################################ -# Copyright 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -import argparse -import importlib -from typing import Callable, Dict, Tuple - -import torch -from benchgc.arg import Arg -from benchgc.mlir.util import MLIRCache -from gc_mlir import ir - -ref_op: Dict[ - str, - Callable[ - [MLIRCache, ir.OpView, Dict[str, torch.Tensor]], - Tuple[torch.Tensor, ...], - ], -] = {} -mlir_op: Dict[str, Callable[[argparse.Namespace, Dict[str, Arg]], ir.Module]] = {} - -for dri in ["basic", "shape"]: - mod = importlib.import_module(f"benchgc.tensor.{dri}") - for key in mod.__dict__: - if key.startswith("ref_"): - op: str = key.removeprefix("ref_") - ref_op[op] = mod.__dict__[key] - if key.startswith("mlir_"): - op: str = key.removeprefix("mlir_") - mlir_op[op] = mod.__dict__[key] diff --git a/test/benchgc/src/benchgc/tensor/basic.py b/test/benchgc/src/benchgc/tensor/basic.py deleted file mode 100644 index a424a7bb2..000000000 --- a/test/benchgc/src/benchgc/tensor/basic.py +++ /dev/null @@ -1,33 +0,0 @@ -################################################################################ -# Copyright 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -from typing import Dict, Tuple - -import benchgc.util -import torch -from benchgc.mlir.util import MLIRCache -from gc_mlir import ir - - -def ref_empty( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - return ( - torch.zeros( - size=op.results[0].type.shape, - dtype=benchgc.util.get_dtype(str(op.results[0].type.element_type)), - ), - ) diff --git a/test/benchgc/src/benchgc/tensor/shape.py b/test/benchgc/src/benchgc/tensor/shape.py deleted file mode 100644 index 25fc20e53..000000000 --- a/test/benchgc/src/benchgc/tensor/shape.py +++ /dev/null @@ -1,59 +0,0 @@ -################################################################################ -# Copyright 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -from typing import Dict, List, Tuple - -import torch -from benchgc.mlir.util import MLIRCache -from gc_mlir import ir - - -def ref_collapse_shape( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - # permute axis and do reshape - reassociation: ir.ArrayAttr = op.attributes["reassociation"] - permutation: List[int] = [] - shape: List[int] = [] - for outdim in reassociation: - d: int = 1 - for indim in outdim: - permutation.append(int(indim)) - d = d * int(op.operands[0].type.shape[int(indim)]) - shape.append(d) - return ( - torch.permute(var[cache.opr[0]], tuple(permutation)) - .contiguous() - .reshape(shape) - .contiguous(), - ) - - -def ref_expand_shape( - cache: MLIRCache, op: ir.OpView, var: Dict[str, torch.Tensor] -) -> Tuple[torch.Tensor, ...]: - # permute axis and do reshape - reassociation: ir.ArrayAttr = op.attributes["reassociation"] - permutation: List[int] = [0] * len(op.result.type.shape) - shape: List[int] = [] - - d: int = 0 - for indim in reassociation: - for outdim in indim: - shape.append(int(op.result.type.shape[int(outdim)])) - permutation[int(outdim)] = d - d = d + 1 - return (torch.reshape(var[cache.opr[0]], shape).permute(permutation).contiguous(),) diff --git a/test/benchgc/src/benchgc/util.py b/test/benchgc/src/benchgc/util.py deleted file mode 100644 index 1f8963dc5..000000000 --- a/test/benchgc/src/benchgc/util.py +++ /dev/null @@ -1,367 +0,0 @@ -################################################################################ -# Copyright 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -import operator -import random -from functools import reduce -from typing import Any, Callable, List, Tuple, Union - -import numpy -import torch - -# verbose level -NO_VERBOSE = 0 -MODULE_VERBOSE = 1 # print the module will be executed -ARG_VERBOSE = 2 # + print arg information -COMPARE_VERBOSE = 3 # + print threshold for comparison -ERROR_OUTPUT_VERBOSE = 4 # + print all error data points if failed -OUTPUT_VERBOSE = 5 # + print all result including passed tensor -INPUT_VERBOSE = 6 # + print input torch tensors - -""" -acc | acc | elems | value_range | worst case -s32 | mul | 10 | 3 | 3^10=2^16, out of 2^30 (max integer) -f16 | mul | 10 | 1 | (2^1)^10=2^10, out of 2^16 (max exponent) -f32 | mul | 30 | 3 | (2^3)^30=2^90, out of 2^128 (max exponent) -s32 | sum | 10000 | 50 | 10000*50=2^19, out of 2^30 (max integer) -f16 | sum | 1000 | 8 | 1000*8=2^13, out of 2^10 (max mantissa/integer) -f32 | sum | 10000 | 16 | 10000*16=2^18, out of 2^23 (max mantissa/integer) - min/max | all | 1000 | no limits on accumulation chain - -In f16 cases, the worst case exceeds the data type bounds, however it's rare -to reach these extreme cases as long as they're close (can't just use f32 bounds) -""" -# first: nonneutral elements -# second: maximum range -_problem_bounds = { - "mul_int": (10, 3), - "mul_fp16": (10, 1), - "mul_fp32": (30, 3), - "sum_int": (10000, 50), - "sum_fp16": (1000, 8), - "sum_fp32": (10000, 16), - "minmax_int": (-1, 1000), - "minmax_fp": (-1, 1000), -} -_dtype_2_range = { - "f32": (-16777216, 16777216), - "f64": (-16777216, 16777216), - "f16": (-2048, 2048), - "bf16": (-16777216, 16777216), - "f8_e5m2": (-2048, 2048), - "f8_e4m3": (-2048, 2048), - "u8": (0, 255), - "s8": (-128, 127), - "s32": (-2147483648, 2147483520), - "i32": (-2147483648, 2147483520), -} - - -def flip_coin( - seed: Union[Any, torch.Tensor], prob: Union[float, torch.Tensor] -) -> Union[bool, torch.Tensor]: - big_prime: int = 1000003 - prime: int = 753737 - seed = seed * prime - return (seed % big_prime) < (prob * big_prime) - - -def get_problem_bounds(kind: str, dt: torch.dtype) -> Tuple[int, int]: - if not dt.is_floating_point: - if kind in ["max", "min"]: - return _problem_bounds["minmax_int"] - elif kind == "mul": - return _problem_bounds["mul_int"] - else: - return _problem_bounds["sum_int"] - elif kind in ["max", "min"]: - return _problem_bounds["minmax_fp"] - elif kind == "mul": - return ( - _problem_bounds["mul_fp16"] - if dt == torch.float16 - else _problem_bounds["mul_fp32"] - ) - else: - return ( - _problem_bounds["sum_fp16"] - if dt == torch.float16 - else _problem_bounds["sum_fp32"] - ) - - -def get_type_range(dt: str) -> Tuple[float, float]: - return _dtype_2_range[dt] - - -# Lnorm, Bnorm & Conv -def get_digits(dtype: str) -> int: - return { - "f32": 24, - "f64": 53, - "s8": 7, - "u8": 8, - "f16": 11, - "bf16": 8, - "f8_e5m2": 3, - "f8_e4m3": 4, - }[dtype] - - -def get_dtype(dtype: str) -> torch.dtype: - if dtype == "f32": - return torch.float32 - elif dtype == "f64": - return torch.float64 - elif dtype == "f16": - return torch.float16 - elif dtype == "bf16": - return torch.bfloat16 - elif dtype == "u8" or dtype == "ui8": - return torch.uint8 - elif dtype in ["s8", "i8", "si8"]: - return torch.int8 - elif dtype == "boolean": - return torch.uint8 - elif dtype == "f8_e4m3": - return torch.float8_e4m3fn - elif dtype == "f8_e5m2": - return torch.float8_e5m2 - elif dtype in ["s32", "i32", "si32"]: - return torch.int32 - elif dtype in ["u32", "ui32"]: - return torch.uint32 - else: - raise Exception(f"data type not support: {dtype}") - - -def get_eps(dtype: torch.dtype) -> float: - return torch.finfo(dtype).eps if dtype.is_floating_point else 0.0 - - -_seed: int = 0 - - -def set_seed(seed: int): - global _seed - _seed = seed - - -def torch_seed(seed_scale: int = 1, seed_shift: int = 0): - seed: int = _seed * seed_scale + seed_shift - torch.manual_seed(seed) - torch.cuda.manual_seed(seed) - torch.cuda.manual_seed_all(seed) - numpy.random.seed(seed) - random.seed(seed) - torch.backends.cudnn.deterministic = True - torch.backends.cudnn.benchmark = False - - -def iterate_tensor(tensor: torch.Tensor, fn: Callable[[Tuple[int, ...]], None]): - index: List[int] = [0] * tensor.ndim - - def dfs(depth: int): - if depth == tensor.ndim: - fn(tuple(index)) - else: - for i in range(tensor.shape[depth]): - index[depth] = i - dfs(depth + 1) - - -# indicate how to check the result -class Checker: - use_norm: bool - # set if negative result is trancated to zero - truncate_negative: bool - eltwise_relax: bool - threshold: float - zero_percent: float - # args: [ref, res, abs_diff, rel_diff] - customized_checker: ( - Callable[[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor], torch.Tensor] - | None - ) - - def __init__( - self, - threshold: float, - zero_percent: float, - use_norm: bool = False, - eltwise_relax: bool = False, - truncate_negative: bool = False, - checker: ( - Callable[ - [torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor], torch.Tensor - ] - | None - ) = None, - ) -> None: - self.use_norm = use_norm - self.eltwise_relax = eltwise_relax - self.threshold = threshold - self.zero_percent = zero_percent - self.truncate_negative = truncate_negative - self.customized_checker = checker - - def check( - self, ref: torch.Tensor, res: torch.Tensor, verbose: int - ) -> Tuple[bool, bool | None]: - if self.use_norm: - return self.norm(ref, res, verbose) - else: - return self.p2p(ref, res, verbose) - - def norm( - self, ref: torch.Tensor, res: torch.Tensor, verbose: int - ) -> Tuple[bool, bool | None]: - - f32_ref = ref.to(torch.float32) - f32_res = res.to(torch.float32) - if f32_ref.nelement() == 0: - return (True, None) - - diff_square_sum = torch.square(torch.subtract(f32_ref, f32_res)).sum() - square_sum = torch.square(f32_ref).sum() - - l2_diff_norm = torch.sqrt(diff_square_sum / square_sum).item() - if verbose >= COMPARE_VERBOSE: - print(f"norm check: {l2_diff_norm:.10f} / threshold: {self.threshold:.10f}") - - return (l2_diff_norm < self.threshold, None) - - def p2p( - self, ref: torch.Tensor, res: torch.Tensor, verbose: int - ) -> Tuple[bool, bool | None]: - - if verbose >= COMPARE_VERBOSE: - print(f"p2p check: threshold: {self.threshold:.7f}") - f32_ref = ref.to(torch.float32) - f32_res = res.to(torch.float32) - - check = torch.BoolTensor([False]) - - check = check.bitwise_or(torch.bitwise_and(f32_ref.isnan(), f32_res.isnan())) - check = check.bitwise_or( - torch.bitwise_and(f32_ref.isneginf(), f32_res.isneginf()) - ) - check = check.bitwise_or( - torch.bitwise_and(f32_ref.isposinf(), f32_res.isposinf()) - ) - - # choose diff/rel_diff based on value - abs_diff = (f32_ref - f32_res).abs() - rel_diff = abs_diff / torch.where( - f32_ref.abs() > numpy.finfo(numpy.float32).smallest_subnormal, - f32_ref.abs(), - 1, - ) - # pick a diff for comparison - diff = torch.where(f32_ref.abs() > 1e-5, rel_diff, abs_diff) - - check = check.bitwise_or(diff <= self.threshold) - - if self.eltwise_relax: - check = check.bitwise_or(abs_diff <= max(torch.finfo(res.dtype).eps, 2e-5)) - - if self.customized_checker is not None: - check = check.bitwise_or( - self.customized_checker(ref, res, abs_diff, rel_diff) - ) - - if verbose >= OUTPUT_VERBOSE: - iterate_tensor( - check, - lambda idx: print( - "%20s: ref: %12.7f res: %12.7f abs_diff: %12.7f rel_diff: %12.7f" - % ( - idx, - f32_ref[idx].item(), - f32_res[idx].item(), - abs_diff[idx].item(), - rel_diff[idx].item(), - ) - ), - ) - if check.all(): - # check mistrusted - zero = res.nelement() - res.count_nonzero().item() - if res.nelement() < 10: - mistrust = False - elif self.truncate_negative: - mistrust = ( - zero * 100.0 / res.nelement() > 50.0 + self.zero_percent / 2.0 - ) - else: - mistrust = zero * 100.0 / res.nelement() > self.zero_percent - return (True, mistrust) - else: - if ( - verbose < OUTPUT_VERBOSE - ): # skip verbose print if full output tensor is alrady printed - fail = torch.argwhere(torch.where(check, 0, 1)) - if verbose < ERROR_OUTPUT_VERBOSE: - fail = fail[ - :10 - ] # only print top 10 failed data points if verbose level does not satisfied - for idx in fail: - index: Tuple[int, ...] = tuple(idx.tolist()) - print( - "%20s: ref: %12.7f res: %12.7f abs_diff: %12.7f rel_diff: %12.7f" - % ( - index, - f32_ref[index].item(), - f32_res[index].item(), - abs_diff[index].item(), - rel_diff[index].item(), - ) - ) - return (False, None) - - -def nelem(shape: List[int]) -> int: - return reduce(operator.mul, shape) - - -def to_int_list(s: str) -> List[int]: - """ - Parsing the cmd for list of int values - - Args: - s (str): int values in cmd, example: 2x3x4 - - Returns: - List[int]: int values in list, example: [2, 3, 4] - """ - if not s or len(s) == 0: - return [] - return [int(i) for i in s.strip().split("x")] - - -def to_bool_list(s: str) -> List[bool]: - """ - Parsing the cmd for list of bool values - - Args: - s (str): bools in cmd, example: 1x0x1 - - Returns: - List[bool]: bools in list, example: [True, False, True] - """ - if not s or len(s) == 0: - return [] - return [bool(int(i)) for i in s.strip().split("x")] diff --git a/test/dnnl/CMakeLists.txt b/test/dnnl/CMakeLists.txt deleted file mode 100644 index 8a6b537e2..000000000 --- a/test/dnnl/CMakeLists.txt +++ /dev/null @@ -1,26 +0,0 @@ -if (NOT GC_ENABLE_TEST OR NOT GC_ENABLE_TEST_DNNL_API OR NOT GC_ENABLE_DNNL_API) - message(STATUS "The dnnl tests are not enabled.") - return() -endif () - -# Copy resources to the build directory -file(GLOB_RECURSE TEST_RESOURCES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} resources/*) -foreach (TEST_RESOURCE ${TEST_RESOURCES}) - configure_file(${TEST_RESOURCE} ${TEST_RESOURCE} COPYONLY) -endforeach () - -file(GLOB TEST_SOURCES Test*.cpp) -foreach (TEST_SOURCE ${TEST_SOURCES}) - get_filename_component(TEST_NAME ${TEST_SOURCE} NAME_WE) - add_executable(${TEST_NAME} ${TEST_SOURCE}) - if (${TEST_NAME} MATCHES "^TestApi.*") - # The API tests are linked with the shared lib - target_link_libraries(${TEST_NAME} PRIVATE LLVMSupport GcDnnlApi) - else () - # The other tests are linked with the static lib and have non-public includes - target_link_libraries(${TEST_NAME} PRIVATE GcDnnlApiStatic) - target_include_directories(${TEST_NAME} PRIVATE ${PROJECT_SOURCE_DIR}/src/dnnl) - endif () - target_link_libraries(${TEST_NAME} PRIVATE gtest gtest_main) - add_test(NAME ${TEST_NAME} COMMAND ${TEST_NAME}) -endforeach () diff --git a/test/dnnl/DnnlTestUtils.h b/test/dnnl/DnnlTestUtils.h deleted file mode 100644 index 0306d23d0..000000000 --- a/test/dnnl/DnnlTestUtils.h +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (C) 2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include -#include -#include - -#if __cplusplus > 202002L -#include -#else -namespace std { -#if defined(__SIZEOF_FLOAT__) && __SIZEOF_FLOAT__ == 4 -using float32_t = float; -#elif defined(__SIZEOF_DOUBLE__) && __SIZEOF_DOUBLE__ == 4 -using float32_t = double; -#else -static_assert(false, "No 32-bit floating point type available"); -#endif -} // namespace std -#endif - -static std::string readStrResource(const std::string &name) { - std::filesystem::path res_dir{"resources"}; - auto path = std::filesystem::absolute(res_dir / name); - std::ifstream file(path); - - if (!file) { - throw std::runtime_error("Unable to open file " + path.string()); - } - - std::stringstream buffer; - buffer << file.rdbuf(); - file.close(); - return buffer.str(); -} diff --git a/test/dnnl/TestApiBasic.cpp b/test/dnnl/TestApiBasic.cpp deleted file mode 100644 index e54b9b727..000000000 --- a/test/dnnl/TestApiBasic.cpp +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright (C) 2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -#include - -#include "DnnlTestUtils.h" -#include "gc_version.h" -#include "graph/backend/elyzor/include/dnnl_graph_compiler.h" - -TEST(TestApiBasic, basicWorkflow) { - auto json = readStrResource("add.json"); - - const struct dnnl_graph_compiler_context ctx = {.num_threads = 4}; - const struct dnnl_graph_compiler *gc; - const struct dnnl_graph_compiler_executable *exe; - - ASSERT_EQ(dnnl_graph_compiler_create(&ctx, &gc), dnnl_success); - - ASSERT_EQ(dnnl_graph_compiler_compile(gc, json.c_str(), &exe), dnnl_success); - - // Initialize inputs and outputs - dnnl_graph_compiler_tensor inputs[2]; - dnnl_graph_compiler_tensor outputs[1]; - float arg1[128]{1.f}; - float arg2[128]; - float arg3[128]; - int64_t dims[1] = {128}; - inputs[0] = {.id = 0, .ndims = 1, .dims = dims, .data = arg1}; - inputs[1] = {.id = 1, .ndims = 1, .dims = dims, .data = arg2}; - outputs[0] = {.id = 2, .ndims = 1, .dims = dims, .data = arg3}; - for (size_t i = 0; i < 128; i++) { - arg2[i] = i; - } - - ASSERT_EQ(dnnl_graph_compiler_execute(exe, inputs, outputs), dnnl_success); - - dnnl_graph_compiler_destroy_executable(exe); - dnnl_graph_compiler_destroy(gc); - - for (size_t i = 0; i < 128; i++) { - ASSERT_FLOAT_EQ(arg3[i], arg1[i] + arg2[i]); - } -} - -TEST(TestApiBasic, get_version) { - auto v = dnnl_graph_compiler_get_version(); - - ASSERT_NE(v, nullptr); - - ASSERT_EQ(v->api_version.major, DNNL_GC_API_V_MAJOR); - ASSERT_EQ(v->api_version.minor, DNNL_GC_API_V_MINOR); - ASSERT_EQ(v->api_version.patch, DNNL_GC_API_V_PATCH); - ASSERT_STREQ(v->api_version.hash, DNNL_GC_API_V_HASH); - - // check if the version is valid - ASSERT_NE(v->gc_version.major, std::numeric_limits::max()); - ASSERT_NE(v->gc_version.minor, std::numeric_limits::max()); - ASSERT_NE(v->gc_version.patch, std::numeric_limits::max()); - - ASSERT_EQ(v->gc_version.major, GC_VERSION_MAJOR); - ASSERT_EQ(v->gc_version.minor, GC_VERSION_MINOR); - ASSERT_EQ(v->gc_version.patch, GC_VERSION_PATCH); - ASSERT_STREQ(v->gc_version.hash, GC_VERSION_HASH); -} - -int main(int argc, char **argv) { - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/test/dnnl/TestApiOps.cpp b/test/dnnl/TestApiOps.cpp deleted file mode 100644 index acb8252b2..000000000 --- a/test/dnnl/TestApiOps.cpp +++ /dev/null @@ -1,325 +0,0 @@ -/* - * Copyright (C) 2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include - -#include - -#include "DnnlTestUtils.h" -#include "graph/backend/elyzor/include/dnnl_graph_compiler.h" - -static void exec(const char *fileName, dnnl_graph_compiler_tensor *inputs, - dnnl_graph_compiler_tensor *outputs) { - auto json = readStrResource(fileName); - const struct dnnl_graph_compiler *gc; - const struct dnnl_graph_compiler_executable *exe; - ASSERT_EQ(dnnl_graph_compiler_create(nullptr, &gc), dnnl_success); - ASSERT_EQ(dnnl_graph_compiler_compile(gc, json.c_str(), &exe), dnnl_success); - ASSERT_EQ(dnnl_graph_compiler_execute(exe, inputs, outputs), dnnl_success); - dnnl_graph_compiler_destroy_executable(exe); - dnnl_graph_compiler_destroy(gc); -} - -TEST(TestApiOps, div) { - dnnl_graph_compiler_tensor inputs[2]; - dnnl_graph_compiler_tensor outputs[1]; - int64_t dims[3] = {64, 32, 32}; - std::float32_t arg1[64][32][32]; - std::float32_t arg2[64][32][32]; - std::float32_t arg3[64][32][32]; - inputs[0] = {.id = 0, .ndims = 3, .dims = dims, .data = arg1}; - inputs[1] = {.id = 1, .ndims = 3, .dims = dims, .data = arg2}; - outputs[0] = {.id = 2, .ndims = 3, .dims = dims, .data = arg3}; - for (auto i = 0; i < 64; i++) { - for (auto j = 0; j < 32; j++) { - for (auto k = 0; k < 32; k++) { - arg1[i][j][k] = static_cast(i * 32 * 32 + j * 32 + k); - arg2[i][j][k] = arg1[i][j][k] + 1; - } - } - } - - exec("div.json", inputs, outputs); - - for (auto i = 0; i < 64; i++) { - for (auto j = 0; j < 32; j++) { - for (auto k = 0; k < 32; k++) { - ASSERT_FLOAT_EQ(arg1[i][j][k] / arg2[i][j][k], arg3[i][j][k]); - } - } - } -} - -TEST(TestApiOps, matMul) { - dnnl_graph_compiler_tensor inputs[3]; - dnnl_graph_compiler_tensor outputs[1]; - int64_t dimsA[2] = {128, 512}; - int64_t dimsB[2] = {128, 64}; - int64_t dimsBias[1] = {64}; - int64_t dimsOut[2] = {512, 64}; - std::float32_t argA[128][512]; - std::float32_t argB[128][64]; - std::float32_t argBias[64]; - std::float32_t argOut[512][64]; - inputs[0] = {.id = 0, .ndims = 2, .dims = dimsA, .data = argA}; - inputs[1] = {.id = 1, .ndims = 2, .dims = dimsB, .data = argB}; - inputs[2] = {.id = 2, .ndims = 1, .dims = dimsBias, .data = argBias}; - outputs[0] = {.id = 3, .ndims = 2, .dims = dimsOut, .data = argOut}; - - // Initialize input tensors - for (auto i = 0; i < 128; i++) { - for (auto j = 0; j < 512; j++) { - argA[i][j] = static_cast(i + j); - } - } - for (auto i = 0; i < 128; i++) { - for (auto j = 0; j < 64; j++) { - argB[i][j] = static_cast(i - j); - } - } - for (auto i = 0; i < 64; i++) { - argBias[i] = static_cast(i); - } - - exec("matmul.json", inputs, outputs); - - // Calculate expected output - std::float32_t expected[512][64]; - for (auto i = 0; i < 512; i++) { - for (auto j = 0; j < 64; j++) { - expected[i][j] = argBias[j]; - for (auto k = 0; k < 128; k++) { - expected[i][j] += argA[k][i] * argB[k][j]; // transpose_a = true - } - } - } - - // Compare the results - for (auto i = 0; i < 512; i++) { - for (auto j = 0; j < 64; j++) { - ASSERT_FLOAT_EQ(expected[i][j], argOut[i][j]); - } - } -} - -TEST(TestApiOps, mul) { - dnnl_graph_compiler_tensor inputs[2]; - dnnl_graph_compiler_tensor outputs[1]; - int64_t dims[1] = {128}; - std::float32_t arg1[128]; - std::float32_t arg2[128]; - std::float32_t arg3[128]; - inputs[0] = {.id = 0, .ndims = 1, .dims = dims, .data = arg1}; - inputs[1] = {.id = 1, .ndims = 1, .dims = dims, .data = arg2}; - outputs[0] = {.id = 2, .ndims = 1, .dims = dims, .data = arg3}; - for (auto i = 0; i < 128; i++) { - arg1[i] = arg2[i] = static_cast(i); - } - - exec("mul.json", inputs, outputs); - - for (auto i = 0; i < 128; i++) { - ASSERT_FLOAT_EQ(arg3[i], static_cast(i * i)); - } -} - -TEST(TestApiOps, sub) { - dnnl_graph_compiler_tensor inputs[2]; - dnnl_graph_compiler_tensor outputs[1]; - int64_t dims[1] = {128}; - std::float32_t arg1[128]; - std::float32_t arg2[128]; - std::float32_t arg3[128]; - inputs[0] = {.id = 0, .ndims = 1, .dims = dims, .data = arg1}; - inputs[1] = {.id = 1, .ndims = 1, .dims = dims, .data = arg2}; - outputs[0] = {.id = 2, .ndims = 1, .dims = dims, .data = arg3}; - for (auto i = 0; i < 128; i++) { - arg1[i] = static_cast(i); - arg2[i] = arg1[i] * arg1[i]; - } - - exec("sub.json", inputs, outputs); - - for (auto i = 0; i < 128; i++) { - ASSERT_FLOAT_EQ(arg3[i], arg1[i] - arg2[i]); - } -} - -TEST(TestApiOps, pow) { - dnnl_graph_compiler_tensor inputs[1]; - dnnl_graph_compiler_tensor outputs[1]; - int64_t dims[1] = {64}; - std::float32_t arg1[64]; - std::float32_t arg2[64]; - inputs[0] = {.id = 0, .ndims = 1, .dims = dims, .data = arg1}; - outputs[0] = {.id = 1, .ndims = 1, .dims = dims, .data = arg2}; - for (auto i = 0; i < 64; i++) { - arg1[i] = static_cast(i); - } - - exec("pow.json", inputs, outputs); - - for (auto i = 0; i < 64; i++) { - ASSERT_FLOAT_EQ(arg1[i] * arg1[i], arg2[i]); - } -} - -TEST(TestApiOps, relu) { - dnnl_graph_compiler_tensor inputs[1]; - dnnl_graph_compiler_tensor outputs[1]; - int64_t dims[1] = {128}; - std::float32_t arg1[128]; - std::float32_t arg2[128]; - inputs[0] = {.id = 0, .ndims = 1, .dims = dims, .data = arg1}; - outputs[0] = {.id = 1, .ndims = 1, .dims = dims, .data = arg2}; - - for (auto i = 0; i < 128; i++) { - arg1[i] = static_cast(i - 64); - } - - exec("relu.json", inputs, outputs); - - for (auto i = 0; i < 128; i++) { - ASSERT_FLOAT_EQ(arg1[i] < 0 ? 0 : arg1[i], arg2[i]); - } -} - -TEST(TestApiOps, reduceMean) { - dnnl_graph_compiler_tensor inputs[1]; - dnnl_graph_compiler_tensor outputs[1]; - int64_t dims1[3] = {16, 64, 32}; - int64_t dims2[3] = {16, 1, 32}; - std::float32_t arg1[16][64][32]; - std::float32_t arg2[16][1][32]; - inputs[0] = {.id = 0, .ndims = 3, .dims = dims1, .data = arg1}; - outputs[0] = {.id = 1, .ndims = 3, .dims = dims2, .data = arg2}; - - for (auto i = 0; i < 16; i++) { - for (auto y = 0; y < 64; y++) { - for (auto z = 0; z < 32; z++) { - arg1[i][y][z] = static_cast(i * 64 * 32 + y * 32 + z); - } - } - } - - exec("reduce_mean.json", inputs, outputs); - - std::float32_t expected[16][1][32]; - for (auto x = 0; x < 16; x++) { - for (auto z = 0; z < 32; z++) { - expected[x][0][z] = 0; - for (auto y = 0; y < 64; y++) { - expected[x][0][z] += arg1[x][y][z]; - } - expected[x][0][z] /= 64; - } - } - - for (auto x = 0; x < 16; x++) { - for (auto z = 0; z < 32; z++) { - ASSERT_FLOAT_EQ(expected[x][0][z], arg2[x][0][z]); - } - } -} - -TEST(TestApiOps, reduceSum) { - dnnl_graph_compiler_tensor inputs[1]; - dnnl_graph_compiler_tensor outputs[1]; - int64_t dims1[3] = {16, 64, 32}; - int64_t dims2[3] = {16, 1, 32}; - std::float32_t arg1[16][64][32]; - std::float32_t arg2[16][1][32]; - inputs[0] = {.id = 0, .ndims = 3, .dims = dims1, .data = arg1}; - outputs[0] = {.id = 1, .ndims = 3, .dims = dims2, .data = arg2}; - - for (auto i = 0; i < 16; i++) { - for (auto y = 0; y < 64; y++) { - for (auto z = 0; z < 32; z++) { - arg1[i][y][z] = static_cast(i * 64 * 32 + y * 32 + z); - } - } - } - - exec("reduce_sum.json", inputs, outputs); - - std::float32_t expected[16][1][32]; - for (auto x = 0; x < 16; x++) { - for (auto z = 0; z < 32; z++) { - expected[x][0][z] = 0; - for (auto y = 0; y < 64; y++) { - expected[x][0][z] += arg1[x][y][z]; - } - } - } - - for (auto x = 0; x < 16; x++) { - for (auto z = 0; z < 32; z++) { - ASSERT_FLOAT_EQ(expected[x][0][z], arg2[x][0][z]); - } - } -} - -TEST(TestApiOps, sigmoid) { - dnnl_graph_compiler_tensor inputs[1]; - dnnl_graph_compiler_tensor outputs[1]; - int64_t dims[1] = {128}; - std::float32_t arg1[128]; - std::float32_t arg2[128]; - inputs[0] = {.id = 0, .ndims = 1, .dims = dims, .data = arg1}; - outputs[0] = {.id = 1, .ndims = 1, .dims = dims, .data = arg2}; - for (auto i = 0; i < 128; i++) { - arg1[i] = static_cast(i - 64); - } - - exec("sigmoid.json", inputs, outputs); - - for (auto i = 0; i < 128; i++) { - ASSERT_FLOAT_EQ(1.f / (1.f + std::exp(-arg1[i])), arg2[i]); - } -} - -TEST(TestApiOps, typecast) { - dnnl_graph_compiler_tensor inputs[1]; - dnnl_graph_compiler_tensor outputs[1]; - int64_t dims[1] = {128}; - std::float32_t arg1[128]; - uint16_t arg2[128]; - inputs[0] = {.id = 0, .ndims = 1, .dims = dims, .data = arg1}; - outputs[0] = {.id = 1, .ndims = 1, .dims = dims, .data = arg2}; - for (auto i = 0; i < 128; i++) { - auto x = i - 64; - arg1[i] = static_cast(x / (std::exp(-x))); - } - - exec("typecast.json", inputs, outputs); - - for (auto i = 0; i < 128; i++) { - llvm::APFloat f(arg1[i]); - bool losesInfo; - f.convert(llvm::APFloat::IEEEhalf(), llvm::APFloat::rmNearestTiesToEven, - &losesInfo); - ASSERT_EQ(static_cast(f.bitcastToAPInt().getZExtValue()), - arg2[i]); - } -} - -int main(int argc, char **argv) { - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/test/dnnl/TestJsonParser.cpp b/test/dnnl/TestJsonParser.cpp deleted file mode 100644 index d7f7097cf..000000000 --- a/test/dnnl/TestJsonParser.cpp +++ /dev/null @@ -1,230 +0,0 @@ -/* - * Copyright (C) 2024 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -#include - -#include "DnnlTestUtils.h" -#include "JsonParser.h" - -#include "gc/Transforms/Passes.h" - -static mlir::ModuleOp parse(const char *fileName, - llvm::SmallVector &inputIds, - llvm::SmallVector &outputIds, - std::unordered_map &strides) { - static auto registry = []() { - mlir::DialectRegistry registry; - mlir::gc::registerGraphCompilerPasses(); - registry.insert(); - registry.insert(); - registry.insert(); - registry.insert(); - return registry; - }(); - static mlir::MLIRContext context(registry); - static bool printModule = []() { - context.loadAllAvailableDialects(); -#ifndef NDEBUG - return true; -#else - return false; -#endif - }(); - - auto json = readStrResource(fileName); - mlir::ModuleOp module = - JsonParser::parse(context, json, inputIds, outputIds, strides); - - if (printModule) { - auto &out = llvm::outs(); - out << "OneDNN JSON to MLIR:\n"; - module.print(out); - out << "Input IDs: "; - for (auto i : inputIds) { - out << i << ' '; - } - out << "\nOutput IDs: "; - for (auto i : outputIds) { - out << i << ' '; - } - out << '\n'; - } - - return module; -} - -TEST(TestJsonParser, AddRelu) { - llvm::SmallVector inputIds; - llvm::SmallVector outputIds; - std::unordered_map strides; - mlir::ModuleOp module = parse("add_relu.json", inputIds, outputIds, strides); - - ASSERT_EQ(inputIds.size(), 2); - ASSERT_EQ(outputIds.size(), 1); - ASSERT_EQ(strides.size(), 1); - ASSERT_EQ(inputIds[0], 0); - ASSERT_EQ(inputIds[1], 1); - ASSERT_EQ(outputIds[0], 3); - ASSERT_EQ(strides[1], Strides({1, 2, 3, 4})); - - auto functions = module.getOps(); - ASSERT_EQ(std::distance(functions.begin(), functions.end()), 1); - auto &&func = *functions.begin(); - auto funcType = func.getFunctionType(); - ASSERT_EQ(funcType.getNumInputs(), 2); - ASSERT_EQ(funcType.getNumResults(), 1); - - auto checkTensorType = [](mlir::Type type) { - ASSERT_EQ(type.getTypeID(), mlir::RankedTensorType::getTypeID()); - auto tensorType = mlir::dyn_cast(type); - ASSERT_TRUE(tensorType.getElementType().isBF16()); - ASSERT_EQ(tensorType.getRank(), 4); - ASSERT_EQ(tensorType.getDimSize(0), 1); - ASSERT_EQ(tensorType.getDimSize(1), 32); - ASSERT_EQ(tensorType.getDimSize(2), 28); - ASSERT_EQ(tensorType.getDimSize(3), 28); - }; - - checkTensorType(funcType.getInput(0)); - checkTensorType(funcType.getInput(1)); - checkTensorType(funcType.getResult(0)); - - auto &body = func.getBody(); - auto ops = body.getOps(); - ASSERT_EQ(std::distance(ops.begin(), ops.end()), 3); - - auto addOp = ops.begin(); - auto attrs = addOp->getAttrs(); - ASSERT_EQ(addOp->getName().getStringRef(), "onednn_graph.add"); - ASSERT_EQ(addOp->getNumOperands(), 2); - ASSERT_EQ(addOp->getNumResults(), 1); - ASSERT_EQ(attrs.size(), 1); - ASSERT_EQ(attrs.begin()->getName(), - mlir::StringAttr::get(addOp->getContext(), "auto_broadcast")); - ASSERT_FALSE( - mlir::cast(attrs.begin()->getValue()).getValue()); - checkTensorType(addOp->getOperandTypes()[0]); - checkTensorType(addOp->getOperandTypes()[1]); - checkTensorType(addOp->getResultTypes()[0]); - - auto reluOp = std::next(addOp); - ASSERT_EQ(reluOp->getName().getStringRef(), "onednn_graph.relu"); - ASSERT_EQ(reluOp->getNumOperands(), 1); - ASSERT_EQ(reluOp->getNumResults(), 1); - checkTensorType(reluOp->getOperandTypes()[0]); - checkTensorType(reluOp->getResultTypes()[0]); - - auto returnOp = std::next(reluOp); - ASSERT_EQ(returnOp->getName().getStringRef(), "func.return"); - ASSERT_EQ(returnOp->getNumOperands(), 1); - checkTensorType(returnOp->getOperandTypes()[0]); -} - -TEST(TestJsonParser, Mpl) { - llvm::SmallVector inputIds; - llvm::SmallVector outputIds; - std::unordered_map strides; - mlir::ModuleOp module = parse("mpl.json", inputIds, outputIds, strides); - - ASSERT_EQ(inputIds.size(), 5); - ASSERT_EQ(outputIds.size(), 1); - ASSERT_EQ(strides.size(), 0); - ASSERT_EQ(inputIds[0], 0); - ASSERT_EQ(inputIds[1], 1); - ASSERT_EQ(inputIds[2], 2); - ASSERT_EQ(inputIds[3], 5); - ASSERT_EQ(inputIds[4], 7); - ASSERT_EQ(outputIds[0], 9); - - auto functions = module.getOps(); - ASSERT_EQ(std::distance(functions.begin(), functions.end()), 1); - auto &&func = *functions.begin(); - auto funcType = func.getFunctionType(); - ASSERT_EQ(funcType.getNumInputs(), 5); - ASSERT_EQ(funcType.getNumResults(), 1); - - auto checkTensorType = [](mlir::Type type, llvm::SmallVector dims) { - ASSERT_EQ(type.getTypeID(), mlir::RankedTensorType::getTypeID()); - auto tensorType = mlir::dyn_cast(type); - ASSERT_TRUE(tensorType.getElementType().isBF16()); - ASSERT_EQ(tensorType.getRank(), dims.size()); - for (size_t i = 0; i < dims.size(); ++i) { - ASSERT_EQ(tensorType.getDimSize(i), dims[i]); - } - }; - - checkTensorType(funcType.getInput(0), {128, 512}); - checkTensorType(funcType.getInput(1), {512, 64}); - checkTensorType(funcType.getInput(2), {64}); - checkTensorType(funcType.getInput(3), {64, 256}); - checkTensorType(funcType.getInput(4), {256}); - checkTensorType(funcType.getResult(0), {128, 256}); - - auto &body = func.getBody(); - auto ops = body.getOps(); - ASSERT_EQ(std::distance(ops.begin(), ops.end()), 6); - - auto matmulOp1 = ops.begin(); - ASSERT_EQ(matmulOp1->getName().getStringRef(), "onednn_graph.matmul"); - ASSERT_EQ(matmulOp1->getNumOperands(), 3); - ASSERT_EQ(matmulOp1->getNumResults(), 1); - checkTensorType(matmulOp1->getOperandTypes()[0], {128, 512}); - checkTensorType(matmulOp1->getOperandTypes()[1], {512, 64}); - checkTensorType(matmulOp1->getOperandTypes()[2], {64}); - checkTensorType(matmulOp1->getResultTypes()[0], {128, 64}); - - auto reluOp1 = std::next(matmulOp1); - ASSERT_EQ(reluOp1->getName().getStringRef(), "onednn_graph.relu"); - ASSERT_EQ(reluOp1->getNumOperands(), 1); - ASSERT_EQ(reluOp1->getNumResults(), 1); - checkTensorType(reluOp1->getOperandTypes()[0], {128, 64}); - checkTensorType(reluOp1->getResultTypes()[0], {128, 64}); - - auto matmulOp2 = std::next(reluOp1); - ASSERT_EQ(matmulOp2->getName().getStringRef(), "onednn_graph.matmul"); - ASSERT_EQ(matmulOp2->getNumOperands(), 2); - ASSERT_EQ(matmulOp2->getNumResults(), 1); - checkTensorType(matmulOp2->getOperandTypes()[0], {128, 64}); - checkTensorType(matmulOp2->getOperandTypes()[1], {64, 256}); - checkTensorType(matmulOp2->getResultTypes()[0], {128, 256}); - - auto addOp = std::next(matmulOp2); - ASSERT_EQ(addOp->getName().getStringRef(), "onednn_graph.add"); - ASSERT_EQ(addOp->getNumOperands(), 2); - ASSERT_EQ(addOp->getNumResults(), 1); - checkTensorType(addOp->getOperandTypes()[0], {128, 256}); - checkTensorType(addOp->getOperandTypes()[1], {256}); - checkTensorType(addOp->getResultTypes()[0], {128, 256}); - - auto reluOp2 = std::next(addOp); - ASSERT_EQ(reluOp2->getName().getStringRef(), "onednn_graph.relu"); - ASSERT_EQ(reluOp2->getNumOperands(), 1); - ASSERT_EQ(reluOp2->getNumResults(), 1); - checkTensorType(reluOp2->getOperandTypes()[0], {128, 256}); - checkTensorType(reluOp2->getResultTypes()[0], {128, 256}); - - auto returnOp = std::next(reluOp2); - ASSERT_EQ(returnOp->getName().getStringRef(), "func.return"); - ASSERT_EQ(returnOp->getNumOperands(), 1); - checkTensorType(returnOp->getOperandTypes()[0], {128, 256}); -} - -int main(int argc, char **argv) { - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/test/dnnl/resources/add.json b/test/dnnl/resources/add.json deleted file mode 100644 index 4df0470b2..000000000 --- a/test/dnnl/resources/add.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "version": "3.5.0", - "engine_kind": "cpu", - "fpmath_mode": "any", - "input_ports": [ - 0, - 1 - ], - "output_ports": [ - 2 - ], - "graph": [ - { - "id": 0, - "kind": "Add", - "inputs": [ - { - "id": 0, - "dtype": "f32", - "shape": [ - 128 - ] - }, - { - "id": 1, - "dtype": "f32", - "shape": [ - 128 - ] - } - ], - "outputs": [ - { - "id": 2, - "dtype": "f32", - "shape": [ - 128 - ] - } - ] - } - ] -} diff --git a/test/dnnl/resources/add_relu.json b/test/dnnl/resources/add_relu.json deleted file mode 100644 index 6d0759b42..000000000 --- a/test/dnnl/resources/add_relu.json +++ /dev/null @@ -1,114 +0,0 @@ -{ - "version": "3.5.0", - "engine_kind": "cpu", - "fpmath_mode": "any", - "input_ports": [ - 0, - 1 - ], - "output_ports": [ - 3 - ], - "graph": [ - { - "id": 0, - "name": "add", - "kind": "Add", - "attrs": { - "auto_broadcast": { - "type": "string", - "value": "none" - } - }, - "inputs": [ - { - "id": 0, - "dtype": "bf16", - "shape": [ - 1, - 32, - 28, - 28 - ], - "stride": [ - -9223372036854775808 - ], - "layout_type": "undef", - "property_type": "undef" - }, - { - "id": 1, - "dtype": "bf16", - "shape": [ - 1, - 32, - 28, - 28 - ], - "stride": [ - 1, 2, 3, 4 - ], - "layout_type": "strided", - "property_type": "undef" - } - ], - "outputs": [ - { - "id": 2, - "dtype": "bf16", - "shape": [ - 1, - 32, - 28, - 28 - ], - "stride": [ - -9223372036854775808 - ], - "layout_type": "undef", - "property_type": "undef" - } - ] - }, - { - "id": 1, - "name": "relu", - "kind": "ReLU", - "attrs": {}, - "inputs": [ - { - "id": 2, - "dtype": "bf16", - "shape": [ - 1, - 32, - 28, - 28 - ], - "stride": [ - -9223372036854775808 - ], - "layout_type": "undef", - "property_type": "undef" - } - ], - "outputs": [ - { - "id": 3, - "dtype": "bf16", - "shape": [ - 1, - 32, - 28, - 28 - ], - "stride": [ - -9223372036854775808 - ], - "layout_type": "undef", - "property_type": "undef" - } - ] - } - ] -} diff --git a/test/dnnl/resources/div.json b/test/dnnl/resources/div.json deleted file mode 100644 index 4cca12450..000000000 --- a/test/dnnl/resources/div.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "version": "3.5.0", - "engine_kind": "cpu", - "fpmath_mode": "any", - "input_ports": [ - 0, - 1 - ], - "output_ports": [ - 2 - ], - "graph": [ - { - "id": 0, - "kind": "Divide", - "inputs": [ - { - "id": 0, - "dtype": "f32", - "shape": [ - 64, 32, 32 - ] - }, - { - "id": 1, - "dtype": "f32", - "shape": [ - 64, 32, 32 - ] - } - ], - "outputs": [ - { - "id": 2, - "dtype": "f32", - "shape": [ - 64, 32, 32 - ] - } - ] - } - ] -} diff --git a/test/dnnl/resources/matmul.json b/test/dnnl/resources/matmul.json deleted file mode 100644 index 2fcd89a37..000000000 --- a/test/dnnl/resources/matmul.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "version": "3.5.0", - "engine_kind": "cpu", - "fpmath_mode": "any", - "input_ports": [ - 0, - 1, - 2 - ], - "output_ports": [ - 3 - ], - "graph": [ - { - "kind": "MatMul", - "attrs": { - "transpose_a": { - "type": "bool", - "value": 1 - }, - "transpose_b": { - "type": "bool", - "value": 0 - } - }, - "inputs": [ - { - "id": 0, - "dtype": "f32", - "shape": [ - 128, - 512 - ] - }, - { - "id": 1, - "dtype": "f32", - "shape": [ - 128, - 64 - ] - }, - { - "id": 2, - "dtype": "f32", - "shape": [ - 64 - ] - } - ], - "outputs": [ - { - "id": 3, - "dtype": "f32", - "shape": [ - 512, - 64 - ] - } - ] - } - ] -} diff --git a/test/dnnl/resources/mpl.json b/test/dnnl/resources/mpl.json deleted file mode 100644 index 7179e006a..000000000 --- a/test/dnnl/resources/mpl.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "version": "3.5.0", - "engine_kind": "cpu", - "fpmath_mode": "any", - "input_ports": [ - 0, - 1, - 2, - 5, - 7 - ], - "output_ports": [ - 9 - ], - "graph": [ - { - "kind": "MatMul", - "inputs": [ - { - "id": 0, - "dtype": "bf16", - "shape": [ - 128, - 512 - ] - }, - { - "id": 1, - "dtype": "bf16", - "shape": [ - 512, - 64 - ] - }, - { - "id": 2, - "dtype": "bf16", - "shape": [ - 64 - ] - } - ], - "outputs": [ - { - "id": 3, - "dtype": "bf16", - "shape": [ - 128, - 64 - ] - } - ] - }, - { - "kind": "ReLU", - "inputs": [ - { - "id": 3, - "dtype": "bf16", - "shape": [ - 128, - 64 - ] - } - ], - "outputs": [ - { - "id": 4, - "dtype": "bf16", - "shape": [ - 128, - 64 - ] - } - ] - }, - { - "kind": "MatMul", - "inputs": [ - { - "id": 4, - "dtype": "bf16", - "shape": [ - 128, - 64 - ] - }, - { - "id": 5, - "dtype": "bf16", - "shape": [ - 64, - 256 - ] - } - ], - "outputs": [ - { - "id": 6, - "dtype": "bf16", - "shape": [ - 128, - 256 - ] - } - ] - }, - { - "kind": "Add", - "inputs": [ - { - "id": 6, - "dtype": "bf16", - "shape": [ - 128, - 256 - ] - }, - { - "id": 7, - "dtype": "bf16", - "shape": [ - 256 - ] - } - ], - "outputs": [ - { - "id": 8, - "dtype": "bf16", - "shape": [ - 128, - 256 - ] - } - ] - }, - { - "kind": "ReLU", - "inputs": [ - { - "id": 8, - "dtype": "bf16", - "shape": [ - 128, - 256 - ] - } - ], - "outputs": [ - { - "id": 9, - "dtype": "bf16", - "shape": [ - 128, - 256 - ] - } - ] - } - ] -} diff --git a/test/dnnl/resources/mul.json b/test/dnnl/resources/mul.json deleted file mode 100644 index 48b59ee1e..000000000 --- a/test/dnnl/resources/mul.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "version": "3.5.0", - "engine_kind": "cpu", - "fpmath_mode": "any", - "input_ports": [ - 0, - 1 - ], - "output_ports": [ - 2 - ], - "graph": [ - { - "id": 0, - "kind": "Multiply", - "inputs": [ - { - "id": 0, - "dtype": "f32", - "shape": [ - 128 - ] - }, - { - "id": 1, - "dtype": "f32", - "shape": [ - 128 - ] - } - ], - "outputs": [ - { - "id": 2, - "dtype": "f32", - "shape": [ - 128 - ] - } - ] - } - ] -} diff --git a/test/dnnl/resources/pow.json b/test/dnnl/resources/pow.json deleted file mode 100644 index 61ab1b55f..000000000 --- a/test/dnnl/resources/pow.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "version": "3.5.0", - "engine_kind": "cpu", - "fpmath_mode": "any", - "input_ports": [ - 0 - ], - "output_ports": [ - 1 - ], - "graph": [ - { - "id": 0, - "kind": "Pow", - "attrs": { - "beta": { - "type": "f32", - "value": 2 - } - }, - "inputs": [ - { - "id": 0, - "dtype": "f32", - "shape": [ - 64 - ] - } - ], - "outputs": [ - { - "id": 1, - "dtype": "f32", - "shape": [ - 64 - ] - } - ] - } - ] -} diff --git a/test/dnnl/resources/reduce_mean.json b/test/dnnl/resources/reduce_mean.json deleted file mode 100644 index 330fe8c3d..000000000 --- a/test/dnnl/resources/reduce_mean.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "version": "3.5.0", - "engine_kind": "cpu", - "fpmath_mode": "any", - "input_ports": [ - 0 - ], - "output_ports": [ - 1 - ], - "graph": [ - { - "id": 0, - "kind": "ReduceMean", - "attrs": { - "axes": { - "type": "s64[]", - "value": [ - 1 - ] - }, - "keep_dims": { - "type": "bool", - "value": 1 - } - }, - "inputs": [ - { - "id": 0, - "dtype": "f32", - "shape": [ - 16, - 64, - 32 - ] - } - ], - "outputs": [ - { - "id": 1, - "dtype": "f32", - "shape": [ - 16, - 1, - 32 - ] - } - ] - } - ] -} diff --git a/test/dnnl/resources/reduce_sum.json b/test/dnnl/resources/reduce_sum.json deleted file mode 100644 index 708376791..000000000 --- a/test/dnnl/resources/reduce_sum.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "version": "3.5.0", - "engine_kind": "cpu", - "fpmath_mode": "any", - "input_ports": [ - 0 - ], - "output_ports": [ - 1 - ], - "graph": [ - { - "id": 0, - "kind": "ReduceSum", - "attrs": { - "axes": { - "type": "s64[]", - "value": [ - 1 - ] - }, - "keep_dims": { - "type": "bool", - "value": 1 - } - }, - "inputs": [ - { - "id": 0, - "dtype": "f32", - "shape": [ - 16, - 64, - 32 - ] - } - ], - "outputs": [ - { - "id": 1, - "dtype": "f32", - "shape": [ - 16, - 1, - 32 - ] - } - ] - } - ] -} diff --git a/test/dnnl/resources/relu.json b/test/dnnl/resources/relu.json deleted file mode 100644 index 8edb0722c..000000000 --- a/test/dnnl/resources/relu.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "version": "3.5.0", - "engine_kind": "cpu", - "fpmath_mode": "any", - "input_ports": [ - 0 - ], - "output_ports": [ - 1 - ], - "graph": [ - { - "id": 0, - "kind": "ReLU", - "inputs": [ - { - "id": 0, - "dtype": "f32", - "shape": [ - 128 - ] - } - ], - "outputs": [ - { - "id": 1, - "dtype": "f32", - "shape": [ - 128 - ] - } - ] - } - ] -} diff --git a/test/dnnl/resources/sigmoid.json b/test/dnnl/resources/sigmoid.json deleted file mode 100644 index 37f5fc68d..000000000 --- a/test/dnnl/resources/sigmoid.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "version": "3.5.0", - "engine_kind": "cpu", - "fpmath_mode": "any", - "input_ports": [ - 0 - ], - "output_ports": [ - 1 - ], - "graph": [ - { - "id": 0, - "kind": "Sigmoid", - "inputs": [ - { - "id": 0, - "dtype": "f32", - "shape": [ - 128 - ] - } - ], - "outputs": [ - { - "id": 1, - "dtype": "f32", - "shape": [ - 128 - ] - } - ] - } - ] -} diff --git a/test/dnnl/resources/sub.json b/test/dnnl/resources/sub.json deleted file mode 100644 index fe2388f65..000000000 --- a/test/dnnl/resources/sub.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "version": "3.5.0", - "engine_kind": "cpu", - "fpmath_mode": "any", - "input_ports": [ - 0, - 1 - ], - "output_ports": [ - 2 - ], - "graph": [ - { - "id": 0, - "kind": "Subtract", - "inputs": [ - { - "id": 0, - "dtype": "f32", - "shape": [ - 128 - ] - }, - { - "id": 1, - "dtype": "f32", - "shape": [ - 128 - ] - } - ], - "outputs": [ - { - "id": 2, - "dtype": "f32", - "shape": [ - 128 - ] - } - ] - } - ] -} diff --git a/test/dnnl/resources/typecast.json b/test/dnnl/resources/typecast.json deleted file mode 100644 index 963008b62..000000000 --- a/test/dnnl/resources/typecast.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "version": "3.5.0", - "engine_kind": "cpu", - "fpmath_mode": "any", - "input_ports": [ - 0 - ], - "output_ports": [ - 1 - ], - "graph": [ - { - "id": 0, - "kind": "Typecast", - "inputs": [ - { - "id": 0, - "dtype": "f32", - "shape": [ - 128 - ] - } - ], - "outputs": [ - { - "id": 1, - "dtype": "f16", - "shape": [ - 128 - ] - } - ] - } - ] -} diff --git a/test/mlir/CMakeLists.txt b/test/mlir/CMakeLists.txt index a15ffe0e5..af236fce2 100644 --- a/test/mlir/CMakeLists.txt +++ b/test/mlir/CMakeLists.txt @@ -1,7 +1,23 @@ -if (NOT GC_ENABLE_TEST_MLIR) - message(STATUS "The mlir tests are not enabled.") - return() -endif () +if(WIN32) + set(MLIR_RUNNER_UTILS_DIR ${LLVM_BINARY_DIR}/bin) +else() + set(MLIR_RUNNER_UTILS_DIR ${LLVM_LIBRARY_DIR}) +endif() -add_subdirectory(test) -add_subdirectory(unittests) +llvm_canonicalize_cmake_booleans(GC_ENABLE_BINDINGS_PYTHON) + +configure_lit_site_cfg( + ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in + ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py MAIN_CONFIG + ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py) + +set(GC_OPT_TEST_DEPENDS FileCheck count not split-file gc-opt) + +add_lit_testsuite(gc-check "Running the regression tests" + ${CMAKE_CURRENT_BINARY_DIR} DEPENDS ${GC_OPT_TEST_DEPENDS}) + +# Looks that this property is suitable for IDE TODO: Check is this fine for IDE +set_target_properties(gc-check PROPERTIES FOLDER "Tests") + +add_lit_testsuites(GC_OPT ${CMAKE_CURRENT_SOURCE_DIR} DEPENDS + ${GC_OPT_TEST_DEPENDS}) diff --git a/test/mlir/gc/Dialect/Linalgx/linalgx-attention-tiling.mlir b/test/mlir/gc/Dialect/Linalgx/linalgx-attention-tiling.mlir new file mode 100644 index 000000000..de41f1902 --- /dev/null +++ b/test/mlir/gc/Dialect/Linalgx/linalgx-attention-tiling.mlir @@ -0,0 +1,37 @@ +// RUN: gc-opt --transform-interpreter --split-input-file %s | FileCheck %s + +module attributes { transform.with_named_sequence } { + transform.named_sequence @__transform_main(%module_op: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match ops{["linalgx.attention"]} in %module_op : (!transform.any_op) -> !transform.any_op + %tiled_op, %forall_op = transform.structured.tile_using_forall %0 num_threads [8, 16] + : (!transform.any_op) -> (!transform.any_op, !transform.any_op) + transform.yield + } +} + +#mapQ = affine_map<(batch, m, k1, k2, n) -> (batch, m, k1)> +#mapK = affine_map<(batch, m, k1, k2, n) -> (batch, k2, k1)> +#mapV = affine_map<(batch, m, k1, k2, n) -> (batch, k2, n)> +#mapS = affine_map<(batch, m, k1, k2, n) -> ()> +#mapO = affine_map<(batch, m, k1, k2, n) -> (batch, m, n)> +#mapR = affine_map<(batch, m, k1, k2, n) -> (batch, m)> + +func.func @attention_f16(%query: tensor<192x1024x64xf16>, + %key: tensor<192x1024x64xf16>, + %value: tensor<192x1024x64xf16>, + %output: tensor<192x1024x64xf32>) + -> (tensor<192x1024x64xf32>) { + %scale = arith.constant 1.0 : f16 + + %out = linalgx.attention + <{ indexing_maps = [#mapQ, #mapK, #mapV, #mapS, #mapO] }> + ins(%query, %key, %value, %scale : tensor<192x1024x64xf16>, tensor<192x1024x64xf16>, tensor<192x1024x64xf16>, f16) + outs(%output : tensor<192x1024x64xf32>) + -> tensor<192x1024x64xf32> + + return %out : tensor<192x1024x64xf32> +} + +// CHECK: scf.forall +// CHECK: linalgx.attention +// CHECK: scf.forall.in_parallel diff --git a/test/mlir/gc/Transforms/GPU/decompose-attention-dyn.mlir b/test/mlir/gc/Transforms/GPU/decompose-attention-dyn.mlir new file mode 100644 index 000000000..fc557f62f --- /dev/null +++ b/test/mlir/gc/Transforms/GPU/decompose-attention-dyn.mlir @@ -0,0 +1,74 @@ +// RUN: gc-opt %s --decomposition --split-input-file | FileCheck %s + +// Decompose a tiled, dynamic-shape linalgx.attention and check that the +// generated ops carry the propagated gc.tiling.wg_tile_sizes attribute with +// the reduction (k-loop) step folded into the K2 dimension. + +#map = affine_map<(d0)[s0] -> (-d0 + s0, 128)> +#map1 = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d1, d2, d3)> +#map2 = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d1, d4, d3)> +#map3 = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d1, d4, d5)> +#map4 = affine_map<(d0, d1, d2, d3, d4, d5) -> ()> +#map5 = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d1, d2, d5)> + +// CHECK-LABEL: func.func @entry +// The accumulator-init fills before the k-loop must NOT get tiling attributes. +// CHECK: linalg.fill ins +// CHECK-NOT: gc.tiling +// CHECK: linalg.fill ins +// CHECK-NOT: gc.tiling +// CHECK: linalg.fill ins +// CHECK-NOT: gc.tiling + +// The k-loop with the original 6D wg tile sizes preserved verbatim. +// CHECK: scf.for +// CHECK: linalg.transpose +// CHECK-SAME: gc.tiling.wg_tile_sizes = array +// QK matmul: iteration (m, k2, k1) -> [128, 64, 0] +// CHECK: linalg.matmul {{.*}}gc.tiling.wg_tile_sizes = array +// CHECK: linalg.reduce +// CHECK-SAME: gc.tiling.wg_tile_sizes = array +// CHECK: linalg.generic {{.*}}gc.tiling.wg_tile_sizes = array +// CHECK-NEXT: ^bb0 +// CHECK-NEXT: %{{.*}} = arith.maximumf +// PV matmul: iteration (m, n, k2) -> [128, 0, 64] +// CHECK: linalg.matmul {{.*}}gc.tiling.wg_tile_sizes = array +// CHECK: } {gc.tiling.level = 1 : i8, gc.tiling.wg_tile_sizes = array} + +// Suffix ops (after the loop) use the 2D (m, n) tile space. +// CHECK: linalg.broadcast {{.*}}gc.tiling.wg_tile_sizes = array +// CHECK: linalg.generic {{.*}}gc.tiling.wg_tile_sizes = array +// CHECK-NEXT: ^bb0 +// CHECK-NEXT: %{{.*}} = arith.divf + +module @fragment_name attributes {gc.module = {device = {arch = "bmg", max_wg_size = 1024 : i64, sg_sizes = [16, 32]}, kernels = {entry_kernel = {threads = [128, 1, 1]}}}} { + func.func @entry(%arg0: memref, %arg1: memref, %arg2: memref, %arg3: memref) attributes {gc.num_kernels = 1 : i32} { + %cst = arith.constant 0.000000e+00 : f16 + %cst_0 = arith.constant 1.250000e-01 : f16 + %c2 = arith.constant 2 : index + %c0 = arith.constant 0 : index + %0 = bufferization.to_tensor %arg0 restrict writable : memref to tensor + %dim = memref.dim %arg0, %c0 : memref + %1 = bufferization.to_tensor %arg1 restrict writable : memref to tensor + %2 = bufferization.to_tensor %arg2 restrict writable : memref to tensor + %dim_1 = memref.dim %arg2, %c2 : memref + %3 = tensor.empty(%dim, %dim_1) : tensor + %dim_2 = memref.dim %arg2, %c0 : memref + %dim_3 = memref.dim %arg2, %c2 : memref + %dim_4 = memref.dim %arg1, %c2 : memref + %4 = scf.forall (%arg4, %arg5, %arg6) = (0, 0, 0) to (%dim_2, 16, %dim_3) step (1, 1, 128) shared_outs(%arg7 = %3) -> (tensor) { + %5 = affine.min #map(%arg6)[%dim_3] + %extracted_slice = tensor.extract_slice %2[%arg4, %arg5, %arg6, 0] [1, 1, %5, 64] [1, 1, 1, 1] : tensor to tensor<1x1x?x64xf16> + %extracted_slice_5 = tensor.extract_slice %1[%arg4, %arg5, 0, 0] [1, 1, %dim_4, 64] [1, 1, 1, 1] : tensor to tensor<1x1x?x64xf16> + %extracted_slice_6 = tensor.extract_slice %0[%arg4, %arg5, 0, 0] [1, 1, %dim_4, 64] [1, 1, 1, 1] : tensor to tensor<1x1x?x64xf16> + %extracted_slice_7 = tensor.extract_slice %arg7[%arg4, %arg5, %arg6, 0] [1, 1, %5, 64] [1, 1, 1, 1] : tensor to tensor<1x1x?x64xf16> + %6 = linalg.fill ins(%cst : f16) outs(%extracted_slice_7 : tensor<1x1x?x64xf16>) -> tensor<1x1x?x64xf16> + %7 = linalgx.attention {gc.tiling.level = 1 : i8, gc.tiling.wg_tile_sizes = array} <{indexing_maps = [#map1, #map2, #map3, #map4, #map5]}> ins(%extracted_slice, %extracted_slice_5, %extracted_slice_6, %cst_0 : tensor<1x1x?x64xf16>, tensor<1x1x?x64xf16>, tensor<1x1x?x64xf16>, f16) outs(%6 : tensor<1x1x?x64xf16>) -> tensor<1x1x?x64xf16> + scf.forall.in_parallel { + tensor.parallel_insert_slice %7 into %arg7[%arg4, %arg5, %arg6, 0] [1, 1, %5, 64] [1, 1, 1, 1] : tensor<1x1x?x64xf16> into tensor + } + } {gc.kernel_name = "entry_kernel", gc.tiling.stamp = 1 : i64} + bufferization.materialize_in_destination %4 in restrict writable %arg3 : (tensor, memref) -> () + return + } +} diff --git a/test/mlir/gc/Transforms/GPU/gpu-kernel-outline-target.mlir b/test/mlir/gc/Transforms/GPU/gpu-kernel-outline-target.mlir new file mode 100644 index 000000000..d4151b27f --- /dev/null +++ b/test/mlir/gc/Transforms/GPU/gpu-kernel-outline-target.mlir @@ -0,0 +1,17 @@ +// RUN: gc-opt %s --gpu-dev-props=arch=bmg --gpu-kernel-outline=cmd-options=-ze-opt-large-register-file | FileCheck %s + +// The outlined kernel module must get a single xevm target with the command +// options as flags. The chip is taken from the device attributes; "bmg" is the +// default chip of the xevm target and is therefore not printed. + +// CHECK: gpu.module @kernel [#xevm.target] { +// CHECK: gpu.func @kernel +module attributes {gc.module = {kernels = {kernel = {threads = [16, 1, 1]}}}} { + func.func @entry(%arg0: memref<64x64xf16>) { + %cst = arith.constant 0.0 : f16 + scf.forall (%i, %j) = (0, 0) to (64, 64) step (16, 16) { + memref.store %cst, %arg0[%i, %j] : memref<64x64xf16> + } {gc.kernel_name = "kernel"} + return + } +} diff --git a/test/mlir/gc/Transforms/GPU/gpu-to-gc-gpu-events.mlir b/test/mlir/gc/Transforms/GPU/gpu-to-gc-gpu-events.mlir new file mode 100644 index 000000000..7efd47d43 --- /dev/null +++ b/test/mlir/gc/Transforms/GPU/gpu-to-gc-gpu-events.mlir @@ -0,0 +1,32 @@ +// RUN: gc-opt %s --gpu-to-gc-gpu=wait=false | FileCheck %s + +// Multiple owned events are stored to the array, passed as the last argument. +// CHECK: llvm.func @main(%{{.+}}: !llvm.ptr, %{{.+}}: !llvm.ptr, %{{.+}}: !llvm.ptr, %{{.+}}: i32, %[[EVENTS:.+]]: !llvm.ptr) { +// CHECK: %[[K0:.+]] = llvm.call @gcGpuLaunch({{.*}}) {gc.kernel_name = "k0"} +// CHECK: %[[K1:.+]] = llvm.call @gcGpuLaunch({{.*}}) {gc.kernel_name = "k1"} +// CHECK: llvm.store %[[K0]], %[[EVENTS]] : !llvm.ptr, !llvm.ptr +// CHECK-NEXT: %[[SLOT1:.+]] = llvm.getelementptr %[[EVENTS]][1] +// CHECK-NEXT: llvm.store %[[K1]], %[[SLOT1]] : !llvm.ptr, !llvm.ptr +// CHECK-NEXT: llvm.return + +// CHECK: llvm.func @eventNum() -> i32 +// CHECK-NEXT: %[[NUM:.+]] = llvm.mlir.constant(2 : i32) : i32 +// CHECK-NEXT: llvm.return %[[NUM]] + +module attributes {gpu.container_module} { + llvm.func @entry(%arg0: !llvm.ptr) { + %0 = llvm.mlir.constant(64 : i64) : i64 + %1 = llvm.mlir.constant(1 : i64) : i64 + %2 = llvm.call @mgpuStreamCreate() : () -> !llvm.ptr + gpu.launch_func <%2 : !llvm.ptr> @k0::@k0 blocks in (%0, %1, %1) threads in (%1, %1, %1) : i64 args(%arg0 : !llvm.ptr) + gpu.launch_func <%2 : !llvm.ptr> @k1::@k1 blocks in (%0, %1, %1) threads in (%1, %1, %1) : i64 args(%arg0 : !llvm.ptr) + llvm.call @mgpuStreamSynchronize(%2) : (!llvm.ptr) -> () + llvm.call @mgpuStreamDestroy(%2) : (!llvm.ptr) -> () + llvm.return + } + gpu.binary @k0 [#gpu.object<#xevm.target, bin = "k0bin">] + gpu.binary @k1 [#gpu.object<#xevm.target, bin = "k1bin">] + llvm.func @mgpuStreamCreate() -> !llvm.ptr + llvm.func @mgpuStreamSynchronize(!llvm.ptr) + llvm.func @mgpuStreamDestroy(!llvm.ptr) +} diff --git a/test/mlir/gc/Transforms/GPU/gpu-to-gc-gpu.mlir b/test/mlir/gc/Transforms/GPU/gpu-to-gc-gpu.mlir new file mode 100644 index 000000000..9d1216caa --- /dev/null +++ b/test/mlir/gc/Transforms/GPU/gpu-to-gc-gpu.mlir @@ -0,0 +1,126 @@ +// RUN: gc-opt %s --gpu-to-gc-gpu=wait=true | FileCheck %s +// RUN: gc-opt %s --gpu-to-gc-gpu=wait=false | FileCheck %s --check-prefix=RETURN +// RUN: gc-opt %s --gpu-to-gc-gpu=wait=true | gc-opt | FileCheck %s --check-prefix=PARSE + +// The kernels are created lazily and cached in the global variables. +// CHECK-DAG: llvm.mlir.global internal constant @gcGpuKernel_k0_Bin("k0bin") +// CHECK-DAG: llvm.mlir.global internal constant @gcGpuKernel_k0_Name("k0\00") +// CHECK-DAG: llvm.mlir.global internal constant @gcGpuKernel_k0_ArgTypes(dense<[10, 6]> : tensor<2xi8>) +// CHECK-DAG: llvm.mlir.global internal @gcGpuKernel_k0_Ptr(#llvm.zero) +// CHECK-DAG: llvm.mlir.global internal @gcGpuKernel_k1_Ptr(#llvm.zero) +// CHECK-DAG: llvm.mlir.global internal @gcGpuKernel_k2_Ptr(#llvm.zero) +// CHECK: llvm.func internal @gcGpuCreateKernel_k0(%[[RT:.+]]: !llvm.ptr) -> !llvm.ptr +// CHECK: llvm.call @gcGpuCreateKernel(%[[RT]], %{{.+}}, %{{.+}}, %{{.+}}, %{{.+}}, %{{.+}}) : (!llvm.ptr, !llvm.ptr, i64, !llvm.ptr, i32, !llvm.ptr) -> !llvm.ptr +// CHECK: llvm.cmpxchg +// CHECK: llvm.call @gcGpuDestroyKernel +// CHECK: llvm.func internal @gcGpuGetKernel_k0(%{{.+}}: !llvm.ptr) -> !llvm.ptr attributes {always_inline} +// CHECK: llvm.load %{{.+}} atomic acquire : !llvm.ptr -> !llvm.ptr +// CHECK: llvm.call @gcGpuCreateKernel_k0 + +// The entry function is renamed to main. The runtime pointer and the external +// dependencies are appended to its arguments. +// CHECK: llvm.func @main(%[[A:.+]]: !llvm.ptr, %[[B:.+]]: !llvm.ptr, %[[RUNTIME:.+]]: !llvm.ptr, %[[DEPS:.+]]: !llvm.ptr, %[[NDEPS:.+]]: i32) { + +// A single array of nDeps + 1 elements is allocated for the merged dependencies +// and the external ones are copied to its beginning. Only k1 has events to +// merge, thus a single slot is reserved. +// CHECK-NEXT: %[[OFF:.+]] = llvm.zext %[[NDEPS]] : i32 to i64 +// CHECK-NEXT: %[[ONE:.+]] = llvm.mlir.constant(1 : i64) : i64 +// CHECK-NEXT: %[[SIZE:.+]] = llvm.add %[[OFF]], %[[ONE]] : i64 +// CHECK-NEXT: %[[ARR:.+]] = llvm.alloca %[[SIZE]] x !llvm.ptr +// CHECK: "llvm.intr.memcpy"(%[[ARR]], %[[DEPS]], %{{.+}}) +// CHECK-NEXT: %[[SLOT:.+]] = llvm.getelementptr %[[ARR]][%[[OFF]]] + +// All the calls share a single buffer of the pointer arrays (the kernel +// argument pointers and the event lists) and a single buffer of the kernel +// argument values. Two slots are enough for both. +// CHECK-NEXT: %{{.+}} = llvm.mlir.constant(2 : i64) : i64 +// CHECK-NEXT: %[[PTRS:.+]] = llvm.alloca %{{.+}} x !llvm.ptr : (i64) -> !llvm.ptr +// CHECK-NEXT: %{{.+}} = llvm.mlir.constant(2 : i64) : i64 +// CHECK-NEXT: %[[VALS:.+]] = llvm.alloca %{{.+}} x !llvm.ptr {alignment = 16 : i64} + +// k0 takes a pointer and an i64, that are stored in the value buffer, and the +// pointers to the values are stored in the pointer buffer. k0 has no event +// dependencies, the external ones are passed as is. +// CHECK: llvm.store %[[A]], %[[VALS]] : !llvm.ptr, !llvm.ptr +// CHECK-NEXT: %[[VAL1:.+]] = llvm.getelementptr %[[VALS]][1] +// CHECK-NEXT: llvm.store %{{.+}}, %[[VAL1]] : i64, !llvm.ptr +// CHECK-NEXT: llvm.store %[[VALS]], %[[PTRS]] : !llvm.ptr, !llvm.ptr +// CHECK-NEXT: %[[PTR1:.+]] = llvm.getelementptr %[[PTRS]][1] +// CHECK-NEXT: llvm.store %[[VAL1]], %[[PTR1]] : !llvm.ptr, !llvm.ptr +// CHECK: %[[K0:.+]] = llvm.call @gcGpuLaunch(%[[RUNTIME]], %{{.+}}, %[[PTRS]], {{.*}}, %[[DEPS]], %[[NDEPS]]) {gc.kernel_name = "k0"} + +// The allocation waits for the k0 event, k1 depends on. +// CHECK: llvm.store %[[K0]], %[[PTRS]] : !llvm.ptr, !llvm.ptr +// CHECK: llvm.call @gcGpuWait(%[[RUNTIME]], %[[PTRS]], %{{.+}}) : (!llvm.ptr, !llvm.ptr, i32) -> () +// CHECK: %[[PTR:.+]] = llvm.call @gcGpuAllocDevice(%[[RUNTIME]], %{{.+}}, %{{.+}}) : (!llvm.ptr, i64, i64) -> !llvm.ptr + +// The copy has no event dependencies, k1 depends on the copy. +// CHECK: %[[CPY:.+]] = llvm.call @gcGpuMemcpy(%[[RUNTIME]], %[[B]], %[[PTR]], %{{.+}}, %[[DEPS]], %[[NDEPS]]) + +// The k1 event is stored to the shared deps array, following the external deps. +// CHECK: llvm.store %[[CPY]], %[[SLOT]] : !llvm.ptr, !llvm.ptr +// CHECK: %[[N:.+]] = llvm.add %[[NDEPS]], %{{.+}} : i32 +// CHECK-NEXT: %[[K1:.+]] = llvm.call @gcGpuLaunch(%[[RUNTIME]], %{{.+}}, %[[PTRS]], {{.*}}, %[[ARR]], %[[N]]) {gc.kernel_name = "k1"} + +// The memory is not released before the copy and k1 are complete. +// CHECK: llvm.store %[[CPY]], %[[PTRS]] : !llvm.ptr, !llvm.ptr +// CHECK: llvm.store %[[K1]], %{{.+}} : !llvm.ptr, !llvm.ptr +// CHECK: llvm.call @gcGpuWait(%[[RUNTIME]] +// CHECK: llvm.call @gcGpuFree(%[[RUNTIME]], %[[PTR]]) + +// The k2 event is the only one we still own. +// CHECK: %[[K2:.+]] = llvm.call @gcGpuLaunch(%[[RUNTIME]], %{{.+}}, %[[PTRS]], {{.*}}, %[[DEPS]], %[[NDEPS]]) {gc.kernel_name = "k2"} +// CHECK: llvm.store %[[K2]], %[[PTRS]] : !llvm.ptr, !llvm.ptr +// CHECK: llvm.call @gcGpuWait(%[[RUNTIME]] +// CHECK: llvm.return + +// The three buffers are the only stack allocations in the function. +// CHECK-NOT: llvm.alloca + +// CHECK: llvm.func @eventNum() -> i32 +// CHECK-NEXT: %[[NUM:.+]] = llvm.mlir.constant(0 : i32) : i32 +// CHECK-NEXT: llvm.return %[[NUM]] +// CHECK: llvm.func @destructor() +// CHECK-NEXT: llvm.fence acquire +// CHECK-COUNT-3: llvm.call @gcGpuDestroyKernel + +// Without the wait option, the owned events are returned to the caller. +// RETURN: llvm.func @main(%{{.+}}: !llvm.ptr, %{{.+}}: !llvm.ptr, %{{.+}}: !llvm.ptr, %{{.+}}: !llvm.ptr, %{{.+}}: i32, %[[EVENT:.+]]: !llvm.ptr) { +// RETURN: %[[K2:.+]] = llvm.call @gcGpuLaunch({{.*}}) {gc.kernel_name = "k2"} +// RETURN: llvm.store %[[K2]], %[[EVENT]] : !llvm.ptr, !llvm.ptr +// RETURN-NEXT: llvm.return + +// RETURN: llvm.func @eventNum() -> i32 +// RETURN-NEXT: %[[NUM:.+]] = llvm.mlir.constant(1 : i32) : i32 +// RETURN-NEXT: llvm.return %[[NUM]] + +// The generated module must be valid. +// PARSE: llvm.call @gcGpuLaunch + +module attributes {gc.module = {kernels = {k1 = {depends = ["k0"]}}}, gpu.container_module} { + llvm.func @entry(%arg0: !llvm.ptr, %arg1: !llvm.ptr) { + %0 = llvm.mlir.constant(64 : i64) : i64 + %1 = llvm.mlir.constant(1 : i64) : i64 + %2 = llvm.mlir.constant(false) : i1 + %3 = llvm.call @mgpuStreamCreate() : () -> !llvm.ptr + gpu.launch_func <%3 : !llvm.ptr> @k0::@k0 blocks in (%0, %1, %1) threads in (%1, %1, %1) : i64 args(%arg0 : !llvm.ptr, %0 : i64) + %4 = llvm.call @mgpuMemAlloc(%0, %3, %2) : (i64, !llvm.ptr, i1) -> !llvm.ptr + llvm.call @mgpuMemcpy(%4, %arg1, %0, %3) : (!llvm.ptr, !llvm.ptr, i64, !llvm.ptr) -> () + gpu.launch_func <%3 : !llvm.ptr> @k1::@k1 blocks in (%0, %1, %1) threads in (%1, %1, %1) : i64 args(%arg0 : !llvm.ptr, %4 : !llvm.ptr) + llvm.call @mgpuMemFree(%4, %3) : (!llvm.ptr, !llvm.ptr) -> () + gpu.launch_func <%3 : !llvm.ptr> @k2::@k2 blocks in (%0, %1, %1) threads in (%1, %1, %1) : i64 args(%arg1 : !llvm.ptr) + llvm.call @mgpuStreamSynchronize(%3) : (!llvm.ptr) -> () + llvm.call @mgpuStreamDestroy(%3) : (!llvm.ptr) -> () + llvm.return + } + gpu.binary @k0 [#gpu.object<#xevm.target, bin = "k0bin">] + gpu.binary @k1 [#gpu.object<#xevm.target, bin = "k1bin">] + gpu.binary @k2 [#gpu.object<#xevm.target, bin = "k2bin">] + llvm.func @mgpuStreamCreate() -> !llvm.ptr + llvm.func @mgpuStreamSynchronize(!llvm.ptr) + llvm.func @mgpuStreamDestroy(!llvm.ptr) + llvm.func @mgpuMemAlloc(i64, !llvm.ptr, i1) -> !llvm.ptr + llvm.func @mgpuMemcpy(!llvm.ptr, !llvm.ptr, i64, !llvm.ptr) + llvm.func @mgpuMemFree(!llvm.ptr, !llvm.ptr) +} diff --git a/test/mlir/gc/Transforms/GPU/remove-allocs-late-subview-offsets.mlir b/test/mlir/gc/Transforms/GPU/remove-allocs-late-subview-offsets.mlir new file mode 100644 index 000000000..f8a0bdfa4 --- /dev/null +++ b/test/mlir/gc/Transforms/GPU/remove-allocs-late-subview-offsets.mlir @@ -0,0 +1,63 @@ +// RUN: gc-opt %s --remove-allocs --split-input-file | FileCheck %s + +// The copy target is a subview whose offsets are computed after the alloc. +// Removing the alloc must move the whole index computation, not just the +// subview op, otherwise the result violates SSA dominance. + +// CHECK-LABEL: func.func @alloc_copy_to_subview_with_late_offsets +// CHECK-NOT: memref.alloc +// CHECK: %[[OFF:.+]] = affine.apply +// CHECK: %[[VIEW:.+]] = memref.subview %arg0[0, %[[OFF]]] +// CHECK: vector.transfer_write %arg2, %[[VIEW]] +// CHECK-NOT: memref.copy +func.func @alloc_copy_to_subview_with_late_offsets(%arg0: memref<8x64xf16>, %arg1: index, %arg2: vector<1x64xf16>) { + %c0 = arith.constant 0 : index + %alloc = memref.alloc() alignment = 64 : memref<1x64xf16> + vector.transfer_write %arg2, %alloc[%c0, %c0] {in_bounds = [true, true]} : vector<1x64xf16>, memref<1x64xf16> + %off = affine.apply affine_map<()[s0] -> (s0 + 4)>()[%arg1] + %subview = memref.subview %arg0[0, %off] [1, 64] [1, 1] : memref<8x64xf16> to memref<1x64xf16, strided<[64, 1], offset: ?>> + memref.copy %alloc, %subview : memref<1x64xf16> to memref<1x64xf16, strided<[64, 1], offset: ?>> + return +} + +// ----- + +// The offset computation is not movable (it reads memory), so the alloc must be +// kept instead of producing invalid IR. + +// CHECK-LABEL: func.func @alloc_copy_to_subview_with_unmovable_offsets +// CHECK: memref.alloc +// CHECK: memref.copy +func.func @alloc_copy_to_subview_with_unmovable_offsets(%arg0: memref<8x64xf16>, %arg1: memref, %arg2: vector<1x64xf16>) { + %c0 = arith.constant 0 : index + %alloc = memref.alloc() alignment = 64 : memref<1x64xf16> + vector.transfer_write %arg2, %alloc[%c0, %c0] {in_bounds = [true, true]} : vector<1x64xf16>, memref<1x64xf16> + %off = memref.load %arg1[] : memref + %subview = memref.subview %arg0[0, %off] [1, 64] [1, 1] : memref<8x64xf16> to memref<1x64xf16, strided<[64, 1], offset: ?>> + memref.copy %alloc, %subview : memref<1x64xf16> to memref<1x64xf16, strided<[64, 1], offset: ?>> + return +} + +// ----- + +// An unsupported view chain must fail before moving the late offset +// computation. Returning failure after moving it would violate the rewrite +// pattern contract. + +// CHECK-LABEL: func.func @alloc_copy_through_unsupported_reshape +// CHECK: %[[ALLOC:.+]] = memref.alloc +// CHECK: vector.transfer_write %arg2, %[[ALLOC]] +// CHECK: %[[RESHAPE:.+]] = memref.reshape %[[ALLOC]] +// CHECK: %[[OFF:.+]] = affine.apply +// CHECK: %[[VIEW:.+]] = memref.subview %arg0[%[[OFF]], 0] +// CHECK: memref.copy %[[RESHAPE]], %[[VIEW]] +func.func @alloc_copy_through_unsupported_reshape(%arg0: memref<64x64xf16>, %arg1: index, %arg2: vector<1x64xf16>, %shape: memref<2xindex>) { + %c0 = arith.constant 0 : index + %alloc = memref.alloc() alignment = 64 : memref<1x64xf16> + vector.transfer_write %arg2, %alloc[%c0, %c0] {in_bounds = [true, true]} : vector<1x64xf16>, memref<1x64xf16> + %reshape = memref.reshape %alloc(%shape) : (memref<1x64xf16>, memref<2xindex>) -> memref<64x1xf16> + %off = affine.apply affine_map<()[s0] -> (s0 + 4)>()[%arg1] + %subview = memref.subview %arg0[%off, 0] [64, 1] [1, 1] : memref<64x64xf16> to memref<64x1xf16, strided<[64, 1], offset: ?>> + memref.copy %reshape, %subview : memref<64x1xf16> to memref<64x1xf16, strided<[64, 1], offset: ?>> + return +} diff --git a/test/mlir/gc/Transforms/GPU/set-attention-layouts-static-tail.mlir b/test/mlir/gc/Transforms/GPU/set-attention-layouts-static-tail.mlir new file mode 100644 index 000000000..62e5693e4 --- /dev/null +++ b/test/mlir/gc/Transforms/GPU/set-attention-layouts-static-tail.mlir @@ -0,0 +1,54 @@ +// RUN: gc-opt %s --set-attention-layouts | FileCheck %s +// RUN: gc-opt %s --set-attention-layouts --set-layouts | FileCheck %s + +// With a static seq_len that is not a multiple of the k-tile, the canonicalizer +// folds away the scf.if guarding the peeled remainder, so the tail iteration is +// inlined after the k-loop. It must get the same layouts as the loop body, and +// set-layouts must not override them with its defaults. + +// CHECK-LABEL: gpu.func @sdpa_kernel +// CHECK: %[[Q:.+]] = xegpu.load_nd {{.*}}sg_layout = [8, 1], sg_data = [16, 64] +// CHECK: %[[LOOP:.+]] = scf.for +// CHECK: xegpu.dpas %[[Q]], {{.*}}layout_cd = #xegpu.layout +// CHECK: xegpu.dpas {{.*}}layout_cd = #xegpu.layout +// CHECK: } +// CHECK: xegpu.load_nd {{.*}}sg_layout = [1, 1], sg_data = [64, 64], order = [0, 1] +// CHECK: xegpu.dpas %[[Q]], {{.*}}layout_a = #xegpu.layout, layout_b = #xegpu.layout, layout_cd = #xegpu.layout +// CHECK: xegpu.load_nd {{.*}}layout = #xegpu.layout}> +// CHECK: xegpu.dpas {{.*}}%[[LOOP]] {{.*}}layout_a = #xegpu.layout, layout_b = #xegpu.layout, layout_cd = #xegpu.layout +// CHECK: xegpu.store_nd {{.*}}sg_layout = [8, 1], sg_data = [16, 64] + +module attributes {gc.module = {kernels = {sdpa_kernel = {sg_layout = [4, 2]}}}, gpu.container_module} { + gpu.module @sdpa_kernel { + gpu.func @sdpa_kernel(%q: memref<128x64xf16>, %k: memref<1178x64xf16>, %v: memref<1178x64xf16>, %out: memref<128x64xf16>) kernel attributes {known_block_size = array} { + %c0 = arith.constant 0 : index + %c64 = arith.constant 64 : index + %c1152 = arith.constant 1152 : index + %zero = arith.constant dense<0.000000e+00> : vector<128x64xf32> + %q_td = xegpu.create_nd_tdesc %q : memref<128x64xf16> -> !xegpu.tensor_desc<128x64xf16> + %q_val = xegpu.load_nd %q_td[0, 0] : !xegpu.tensor_desc<128x64xf16> -> vector<128x64xf16> + %k_td = xegpu.create_nd_tdesc %k : memref<1178x64xf16> -> !xegpu.tensor_desc<64x64xf16> + %v_td = xegpu.create_nd_tdesc %v : memref<1178x64xf16> -> !xegpu.tensor_desc<64x64xf16> + %loop = scf.for %iv = %c0 to %c1152 step %c64 iter_args(%acc = %zero) -> (vector<128x64xf32>) { + %k_val = xegpu.load_nd %k_td[%iv, 0] : !xegpu.tensor_desc<64x64xf16> -> vector<64x64xf16> + %k_t = vector.transpose %k_val, [1, 0] : vector<64x64xf16> to vector<64x64xf16> + %s = xegpu.dpas %q_val, %k_t, %zero : vector<128x64xf16>, vector<64x64xf16>, vector<128x64xf32> -> vector<128x64xf32> + %p = arith.truncf %s : vector<128x64xf32> to vector<128x64xf16> + %v_val = xegpu.load_nd %v_td[%iv, 0] : !xegpu.tensor_desc<64x64xf16> -> vector<64x64xf16> + %o = xegpu.dpas %p, %v_val, %acc : vector<128x64xf16>, vector<64x64xf16>, vector<128x64xf32> -> vector<128x64xf32> + scf.yield %o : vector<128x64xf32> + } + // Peeled tail (rows 1152..1177), inlined after the loop. + %k_tail = xegpu.load_nd %k_td[%c1152, 0] : !xegpu.tensor_desc<64x64xf16> -> vector<64x64xf16> + %k_tail_t = vector.transpose %k_tail, [1, 0] : vector<64x64xf16> to vector<64x64xf16> + %s_tail = xegpu.dpas %q_val, %k_tail_t, %zero : vector<128x64xf16>, vector<64x64xf16>, vector<128x64xf32> -> vector<128x64xf32> + %p_tail = arith.truncf %s_tail : vector<128x64xf32> to vector<128x64xf16> + %v_tail = xegpu.load_nd %v_td[%c1152, 0] : !xegpu.tensor_desc<64x64xf16> -> vector<64x64xf16> + %o_tail = xegpu.dpas %p_tail, %v_tail, %loop : vector<128x64xf16>, vector<64x64xf16>, vector<128x64xf32> -> vector<128x64xf32> + %res = arith.truncf %o_tail : vector<128x64xf32> to vector<128x64xf16> + %out_td = xegpu.create_nd_tdesc %out : memref<128x64xf16> -> !xegpu.tensor_desc<128x64xf16> + xegpu.store_nd %res, %out_td[0, 0] : vector<128x64xf16>, !xegpu.tensor_desc<128x64xf16> + gpu.return + } + } +} diff --git a/test/mlir/gc/Transforms/GPU/tile-dynamic-shapes.mlir b/test/mlir/gc/Transforms/GPU/tile-dynamic-shapes.mlir new file mode 100644 index 000000000..c9247e416 --- /dev/null +++ b/test/mlir/gc/Transforms/GPU/tile-dynamic-shapes.mlir @@ -0,0 +1,60 @@ +// RUN: gc-opt %s --gpu-dev-props --tile-contract --tile-attention --split-input-file | FileCheck %s + + +// CHECK-LABEL: func.func @entry +// CHECK-DAG: %[[C64:.+]] = arith.constant 64 : index +// CHECK-DAG: %[[C0:.+]] = arith.constant 0 : index +// CHECK: %[[M:.+]] = tensor.dim +// CHECK: %[[K:.+]] = tensor.dim +// CHECK: %[[N:.+]] = tensor.dim +// CHECK: scf.forall (%{{.+}}, %{{.+}}) = (0, 0) to (%[[M]], %[[N]]) step (32, 64) +// CHECK: scf.for %{{.+}} = %[[C0]] to %[[K]] step %[[C64]] +// CHECK: linalg.matmul {gc.tiling.level = 1 +// CHECK: linalg.generic {{.*}}gc.tiling.level = 0 +func.func @entry(%arg0: memref, %arg1: memref, %arg2: memref) { + %cst = arith.constant 0.000000e+00 : f16 + %c1 = arith.constant 1 : index + %c0 = arith.constant 0 : index + %0 = bufferization.to_tensor %arg0 restrict writable : memref to tensor + %1 = bufferization.to_tensor %arg1 restrict writable : memref to tensor + %dim = memref.dim %arg1, %c0 : memref + %dim_0 = memref.dim %arg0, %c1 : memref + %2 = tensor.empty(%dim, %dim_0) : tensor + %3 = linalg.fill ins(%cst : f16) outs(%2 : tensor) -> tensor + %4 = linalg.matmul ins(%1, %0 : tensor, tensor) outs(%3 : tensor) -> tensor + %5 = linalg.generic {indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>, affine_map<(d0, d1) -> (d0, d1)>, affine_map<(d0, d1) -> (d0, d1)>], iterator_types = ["parallel", "parallel"]} + ins(%4, %1 : tensor, tensor) outs(%2 : tensor) { + ^bb0(%lhs: f16, %rhs: f16, %out: f16): + %sum = arith.addf %lhs, %rhs : f16 + linalg.yield %sum : f16 + } -> tensor + bufferization.materialize_in_destination %5 in restrict writable %arg2 : (tensor, memref) -> () + return +} + +// ----- + +// CHECK-LABEL: func.func @entry +// CHECK-DAG: %[[C0:.+]] = arith.constant 0 : index +// CHECK-DAG: %[[C2:.+]] = arith.constant 2 : index +// CHECK: %[[DB:.+]] = tensor.dim %{{.+}}, %[[C0]] +// CHECK: %[[DS:.+]] = tensor.dim %{{.+}}, %[[C2]] +// CHECK: scf.forall (%{{.+}}, %{{.+}}, %{{.+}}) = (0, 0, 0) to (%[[DB]], 16, %[[DS]]) step (1, 1, 128) +// CHECK: linalgx.attention {gc.tiling.level = 1 +// CHECK-SAME: tensor<1x1x?x64xf16> +func.func @entry(%arg0: memref, %arg1: memref, %arg2: memref, %arg3: memref) { + %cst = arith.constant 0.000000e+00 : f16 + %cst_0 = arith.constant 1.250000e-01 : f16 + %c2 = arith.constant 2 : index + %c0 = arith.constant 0 : index + %0 = bufferization.to_tensor %arg0 restrict writable : memref to tensor + %dim = memref.dim %arg0, %c0 : memref + %1 = bufferization.to_tensor %arg1 restrict writable : memref to tensor + %2 = bufferization.to_tensor %arg2 restrict writable : memref to tensor + %dim_1 = memref.dim %arg2, %c2 : memref + %3 = tensor.empty(%dim, %dim_1) : tensor + %4 = linalg.fill ins(%cst : f16) outs(%3 : tensor) -> tensor + %5 = linalgx.attention <{indexing_maps = [affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d1, d2, d3)>, affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d1, d4, d3)>, affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d1, d4, d5)>, affine_map<(d0, d1, d2, d3, d4, d5) -> ()>, affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d1, d2, d5)>]}> ins(%2, %1, %0, %cst_0 : tensor, tensor, tensor, f16) outs(%4 : tensor) -> tensor + bufferization.materialize_in_destination %5 in restrict writable %arg3 : (tensor, memref) -> () + return +} diff --git a/test/mlir/gc/Transforms/GPU/tile-grouped-matmul.mlir b/test/mlir/gc/Transforms/GPU/tile-grouped-matmul.mlir new file mode 100644 index 000000000..ac30f13e7 --- /dev/null +++ b/test/mlir/gc/Transforms/GPU/tile-grouped-matmul.mlir @@ -0,0 +1,91 @@ +// RUN: gc-opt %s --gpu-dev-props --tile-contract --tile-grouped-matmul --split-input-file | FileCheck %s + +// Without 'max_group_size' the row loop runs over all rows of 'lhs'. Every +// tile covers one expert: its rows are clamped to the expert's group, which is +// read from 'offsets', and the tile itself is a plain linalg.matmul with the +// expert's weight matrix. + +#mapL = affine_map<(e, m, n, k) -> (m, k)> +#mapW = affine_map<(e, m, n, k) -> (e, n, k)> +#mapO = affine_map<(e, m, n, k) -> (e)> +#mapR = affine_map<(e, m, n, k) -> (m, n)> + +// Maps of the plain matmul: out[m, n] += lhs[m, k] * wei[n, k]. +// CHECK-DAG: #[[MAP_A:.+]] = affine_map<(d0, d1, d2) -> (d0, d2)> +// CHECK-DAG: #[[MAP_B:.+]] = affine_map<(d0, d1, d2) -> (d1, d2)> +// CHECK-DAG: #[[MAP_C:.+]] = affine_map<(d0, d1, d2) -> (d0, d1)> +// CHECK-LABEL: func.func @grouped_matmul +// CHECK-SAME: %[[LHS:[^:]+]]: tensor, %[[WEI:[^:]+]]: tensor<8x512x256xf16>, %[[OFFS:[^:]+]]: tensor<8xi32> +// CHECK: scf.forall (%[[E:[^,]+]], %[[M:[^,]+]], %[[N:[^)]+]]) = (0, 0, 0) to (8, %[[TOTAL:[^,]+]], 512) step (1, [[#TM:]], [[#TN:]]) +// CHECK-SAME: shared_outs(%[[OUT:[^ ]+]] = %{{[^)]+}}) +// The tile's rows, clamped to the bounds of the row loop. +// CHECK: %[[TILE:.+]] = affine.min {{.*}}(%[[M]])[%[[TOTAL]]] +// Group range: start = e == 0 ? 0 : offsets[e - 1], end = offsets[e]. +// CHECK: %[[FIRST:.+]] = arith.cmpi eq, %[[E]], %{{.+}} +// CHECK: %[[PREV_IDX:.+]] = arith.select %[[FIRST]], %{{.+}}, %{{.+}} +// CHECK: %[[PREV_I32:.+]] = tensor.extract %[[OFFS]][%[[PREV_IDX]]] +// CHECK: %[[PREV:.+]] = arith.index_cast %[[PREV_I32]] +// CHECK: %[[START:.+]] = arith.select %[[FIRST]], %{{.+}}, %[[PREV]] +// CHECK: %[[END_I32:.+]] = tensor.extract %[[OFFS]][%[[E]]] +// CHECK: %[[END:.+]] = arith.index_cast %[[END_I32]] +// row0 = min(start + m, end), size = clamp(tile, 0, end - row0). +// CHECK: %[[ABS:.+]] = arith.addi %[[START]], %[[M]] +// CHECK: %[[ROW0:.+]] = arith.minsi %[[ABS]], %[[END]] +// CHECK: %[[AVAIL:.+]] = arith.subi %[[END]], %[[ROW0]] +// CHECK: %[[CLAMPED:.+]] = arith.minsi %[[TILE]], %[[AVAIL]] +// CHECK: %[[SIZE:.+]] = arith.maxsi {{.*}}%[[CLAMPED]] +// CHECK: %[[LHS_T:.+]] = tensor.extract_slice %[[LHS]][%[[ROW0]], 0] [%[[SIZE]], 256] [1, 1] +// CHECK: %[[OUT_T:.+]] = tensor.extract_slice %[[OUT]][%[[ROW0]], %[[N]]] [%[[SIZE]], [[#TN]]] [1, 1] +// CHECK: %[[INIT:.+]] = linalg.fill {{.*}}gc.tiling.wg_tile_sizes = array{{.*}} outs(%[[OUT_T]] +// The expert dim of the weights is sliced to 1 and dropped. +// CHECK: %[[WEI_T:.+]] = tensor.extract_slice %[[WEI]][%[[E]], %[[N]], 0] [1, [[#TN]], 256] [1, 1, 1] +// CHECK-SAME: : tensor<8x512x256xf16> to tensor<[[#TN]]x256xf16> +// The K loop of the plain matmul. +// CHECK: %[[RES:.+]] = scf.for %[[K:[^ ]+]] = %{{.+}} to %{{.+}} step %{{.+}} iter_args(%{{[^ ]+}} = %[[INIT]]) +// CHECK: tensor.extract_slice %[[LHS_T]][0, %[[K]]] [%[[SIZE]], 64] +// CHECK: tensor.extract_slice %[[WEI_T]][0, %[[K]]] {{\[}}[[#TN]], 64] +// CHECK: linalg.matmul indexing_maps = [#[[MAP_A]], #[[MAP_B]], #[[MAP_C]]] +// CHECK-SAME: gc.tiling.level = 1 +// CHECK-SAME: gc.tiling.wg_tile_sizes = array +// CHECK: scf.forall.in_parallel +// CHECK: tensor.parallel_insert_slice %[[RES]] into %[[OUT]][%{{.+}}, %[[N]]] [%{{.+}}, [[#TN]]] [1, 1] +// CHECK-NOT: linalgx.grouped_matmul +func.func @grouped_matmul(%lhs: tensor, %wei: tensor<8x512x256xf16>, %offs: tensor<8xi32>) -> tensor { + %c0 = arith.constant 0 : index + %cst = arith.constant 0.000000e+00 : f16 + %total = tensor.dim %lhs, %c0 : tensor + %0 = tensor.empty(%total) : tensor + %1 = linalg.fill ins(%cst : f16) outs(%0 : tensor) -> tensor + %2 = linalgx.grouped_matmul <{indexing_maps = [#mapL, #mapW, #mapO, #mapR]}> + ins(%lhs, %wei, %offs : tensor, tensor<8x512x256xf16>, tensor<8xi32>) + outs(%1 : tensor) -> tensor + return %2 : tensor +} + +// ----- + +// With 'max_group_size' the row loop only runs up to the hint. + +#mapL = affine_map<(e, m, n, k) -> (m, k)> +#mapW = affine_map<(e, m, n, k) -> (e, n, k)> +#mapO = affine_map<(e, m, n, k) -> (e)> +#mapR = affine_map<(e, m, n, k) -> (m, n)> + +// CHECK-LABEL: func.func @grouped_matmul_max_group_size +// CHECK-SAME: %{{[^:]+}}: tensor, %{{[^:]+}}: tensor<8x512x256xf16>, %{{[^:]+}}: tensor<8xi32>, %[[MAX:[^:]+]]: index +// CHECK: scf.forall (%{{[^,]+}}, %[[M:[^,]+]], %{{[^)]+}}) = (0, 0, 0) to (8, %[[MAX]], 512) step (1, {{[0-9]+}}, {{[0-9]+}}) +// CHECK: affine.min {{.*}}(%[[M]])[%[[MAX]]] +// CHECK: linalg.matmul +// CHECK-NOT: linalgx.grouped_matmul +func.func @grouped_matmul_max_group_size(%lhs: tensor, %wei: tensor<8x512x256xf16>, %offs: tensor<8xi32>, %max: index) -> tensor { + %c0 = arith.constant 0 : index + %cst = arith.constant 0.000000e+00 : f16 + %total = tensor.dim %lhs, %c0 : tensor + %0 = tensor.empty(%total) : tensor + %1 = linalg.fill ins(%cst : f16) outs(%0 : tensor) -> tensor + %2 = linalgx.grouped_matmul <{indexing_maps = [#mapL, #mapW, #mapO, #mapR]}> + ins(%lhs, %wei, %offs : tensor, tensor<8x512x256xf16>, tensor<8xi32>) + max_group_size(%max) + outs(%1 : tensor) -> tensor + return %2 : tensor +} diff --git a/test/mlir/gc/Transforms/GPU/tile-wg-tile-sizes.mlir b/test/mlir/gc/Transforms/GPU/tile-wg-tile-sizes.mlir new file mode 100644 index 000000000..6313fe9aa --- /dev/null +++ b/test/mlir/gc/Transforms/GPU/tile-wg-tile-sizes.mlir @@ -0,0 +1,68 @@ +// RUN: gc-opt %s --gpu-dev-props --tile-contract --tile-attention --split-input-file | FileCheck %s + +// CHECK-LABEL: func.func @matmul_add +// CHECK: scf.forall +// CHECK: linalg.fill {{.*}}gc.tiling.wg_tile_sizes = array +// CHECK: scf.for +// CHECK: linalg.matmul {{.*}}gc.tiling.wg_tile_sizes = array +// CHECK: linalg.generic {{.*}}gc.tiling.wg_tile_sizes = array +func.func @matmul_add(%arg0: tensor<1024x512xf16>, %arg1: tensor<512x1024xf16>) -> tensor<1024x1024xf16> { + %cst = arith.constant 0.000000e+00 : f16 + %0 = tensor.empty() : tensor<1024x1024xf16> + %1 = linalg.fill ins(%cst : f16) outs(%0 : tensor<1024x1024xf16>) -> tensor<1024x1024xf16> + %2 = linalg.matmul ins(%arg0, %arg1 : tensor<1024x512xf16>, tensor<512x1024xf16>) outs(%1 : tensor<1024x1024xf16>) -> tensor<1024x1024xf16> + %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>, affine_map<(d0, d1) -> (d0, d1)>, affine_map<(d0, d1) -> (d0, d1)>], iterator_types = ["parallel", "parallel"]} + ins(%2, %1 : tensor<1024x1024xf16>, tensor<1024x1024xf16>) outs(%0 : tensor<1024x1024xf16>) { + ^bb0(%lhs: f16, %rhs: f16, %out: f16): + %sum = arith.addf %lhs, %rhs : f16 + linalg.yield %sum : f16 + } -> tensor<1024x1024xf16> + return %3 : tensor<1024x1024xf16> +} + +// ----- + +// CHECK-LABEL: func.func @transpose_matmul_add +// CHECK: scf.forall +// CHECK: linalg.fill {{.*}}gc.tiling.wg_tile_sizes = array +// CHECK: scf.for +// CHECK: linalg.transpose {{.*}}gc.tiling.wg_tile_sizes = array +// CHECK: linalg.matmul {{.*}}gc.tiling.wg_tile_sizes = array +// CHECK: linalg.generic {{.*}}gc.tiling.wg_tile_sizes = array +func.func @transpose_matmul_add(%arg0: tensor<1024x512xf16>, %argB: tensor<1024x512xf16>) -> tensor<1024x1024xf16> { + %cst = arith.constant 0.000000e+00 : f16 + %0 = tensor.empty() : tensor<1024x1024xf16> + %bt_init = tensor.empty() : tensor<512x1024xf16> + %bt = linalg.transpose ins(%argB : tensor<1024x512xf16>) outs(%bt_init : tensor<512x1024xf16>) permutation = [1, 0] + %1 = linalg.fill ins(%cst : f16) outs(%0 : tensor<1024x1024xf16>) -> tensor<1024x1024xf16> + %2 = linalg.matmul ins(%arg0, %bt : tensor<1024x512xf16>, tensor<512x1024xf16>) outs(%1 : tensor<1024x1024xf16>) -> tensor<1024x1024xf16> + %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>, affine_map<(d0, d1) -> (d0, d1)>, affine_map<(d0, d1) -> (d0, d1)>], iterator_types = ["parallel", "parallel"]} + ins(%2, %1 : tensor<1024x1024xf16>, tensor<1024x1024xf16>) outs(%0 : tensor<1024x1024xf16>) { + ^bb0(%lhs: f16, %rhs: f16, %out: f16): + %sum = arith.addf %lhs, %rhs : f16 + linalg.yield %sum : f16 + } -> tensor<1024x1024xf16> + return %3 : tensor<1024x1024xf16> +} + +// ----- + +// CHECK-LABEL: func.func @attention +// CHECK: scf.forall +// CHECK: linalgx.attention {{.*}}gc.tiling.wg_tile_sizes = array +#mapQ = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d1, d2, d3)> +#mapK = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d1, d4, d3)> +#mapV = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d1, d4, d5)> +#mapS = affine_map<(d0, d1, d2, d3, d4, d5) -> ()> +#mapO = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d1, d2, d5)> +func.func @attention(%q: tensor<2x16x1024x64xf16>, %k: tensor<2x16x1024x64xf16>, %v: tensor<2x16x1024x64xf16>) -> tensor<2x16x1024x64xf16> { + %scale = arith.constant 1.250000e-01 : f16 + %cst = arith.constant 0.000000e+00 : f16 + %0 = tensor.empty() : tensor<2x16x1024x64xf16> + %1 = linalg.fill ins(%cst : f16) outs(%0 : tensor<2x16x1024x64xf16>) -> tensor<2x16x1024x64xf16> + %2 = linalgx.attention + <{ indexing_maps = [#mapQ, #mapK, #mapV, #mapS, #mapO] }> + ins(%q, %k, %v, %scale : tensor<2x16x1024x64xf16>, tensor<2x16x1024x64xf16>, tensor<2x16x1024x64xf16>, f16) + outs(%1 : tensor<2x16x1024x64xf16>) -> tensor<2x16x1024x64xf16> + return %2 : tensor<2x16x1024x64xf16> +} diff --git a/test/mlir/test/lit.cfg.py b/test/mlir/lit.cfg.py similarity index 82% rename from test/mlir/test/lit.cfg.py rename to test/mlir/lit.cfg.py index eb1bbbcb1..b206656c9 100644 --- a/test/mlir/test/lit.cfg.py +++ b/test/mlir/lit.cfg.py @@ -14,7 +14,7 @@ # name: The name of this test suite. config.name = "GRAPH_COMPILER_OPT" -config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell) +config.test_format = lit.formats.ShTest() # suffixes: A list of file extensions to treat as test files. config.suffixes = [".mlir"] @@ -23,18 +23,15 @@ config.test_source_root = os.path.dirname(__file__) # test_exec_root: The root path where tests should be run. -config.test_exec_root = os.path.join(config.gc_obj_root, "test", "mlir", "test") +config.test_exec_root = os.path.join(config.gc_obj_root, "test", "mlir") config.gc_tools_dir = os.path.join(config.gc_obj_root, "bin") config.substitutions.append(("%PATH%", config.environment["PATH"])) config.substitutions.append(("%shlibext", config.llvm_shlib_ext)) config.substitutions.append(("%llvmlibdir", config.llvm_lib_dir)) config.substitutions.append(("%gclibdir", config.gc_obj_root + "/lib/")) -config.substitutions.append(('%mlir_runner_utils', config.mlir_runner_utils)) -config.substitutions.append(('%mlir_c_runner_utils', config.mlir_c_runner_utils)) - -if config.gc_use_gpu: - config.substitutions.append(('%opencl_runtime', config.opencl_runtime)) +config.substitutions.append(("%mlir_runner_utils", config.mlir_runner_utils)) +config.substitutions.append(("%mlir_c_runner_utils", config.mlir_c_runner_utils)) llvm_config.with_system_environment(["HOME", "INCLUDE", "LIB", "TMP", "TEMP"]) @@ -49,7 +46,7 @@ llvm_config.with_environment("PATH", config.llvm_tools_dir, append_path=True) tool_dirs = [config.gc_tools_dir, config.llvm_tools_dir] -tools = ["gc-opt", "gc-cpu-runner", "gc-gpu-runner"] +tools = ["gc-opt"] llvm_config.add_tool_substitutions(tools, tool_dirs) @@ -59,4 +56,4 @@ os.path.join(config.mlir_obj_dir, "python_packages", "gc_mlir_core"), ], append_path=True, -) \ No newline at end of file +) diff --git a/test/mlir/test/lit.site.cfg.py.in b/test/mlir/lit.site.cfg.py.in similarity index 80% rename from test/mlir/test/lit.site.cfg.py.in rename to test/mlir/lit.site.cfg.py.in index 3ca900909..c08c1e933 100644 --- a/test/mlir/test/lit.site.cfg.py.in +++ b/test/mlir/lit.site.cfg.py.in @@ -34,20 +34,14 @@ config.gc_src_root = "@PROJECT_SOURCE_DIR@" config.gc_obj_root = "@PROJECT_BINARY_DIR@" config.gc_lib_dir = os.path.join(config.gc_obj_root, "lib") config.mlir_obj_dir = "@MLIR_BINARY_DIR@" -config.gc_use_imex = "@GC_ENABLE_IMEX@" in ["ON", "1"] -config.gc_use_gpu = "@GC_ENABLE_GPU@" in ["ON", "1"] -config.enable_bindings_python = @GC_ENABLE_BINDINGS_PYTHON@ config.llvm_shlib_dir = "@SHLIBDIR@" config.llvm_shlib_ext = "@SHLIBEXT@" config.mlir_runner_utils_dir = "@MLIR_RUNNER_UTILS_DIR@" config.mlir_runner_utils = os.path.normpath(os.path.join(config.mlir_runner_utils_dir, config.shlib_prefix + "mlir_runner_utils" + config.llvm_shlib_ext)) config.mlir_c_runner_utils = os.path.normpath(os.path.join(config.mlir_runner_utils_dir, config.shlib_prefix + "mlir_c_runner_utils" + config.llvm_shlib_ext)) -config.opencl_runtime = os.path.normpath(os.path.join(config.gc_lib_dir, config.shlib_prefix + "GcOpenclRuntime" + config.llvm_shlib_ext)) -config.gc_use_dnnl = "@GC_ENABLE_DNNL_API@" in ["ON", "1"] - import lit.llvm lit.llvm.initialize(lit_config, config) # Let the main config do the real work. -lit_config.load_config(config, "@PROJECT_SOURCE_DIR@/test/mlir/test/lit.cfg.py") +lit_config.load_config(config, "@PROJECT_SOURCE_DIR@/test/mlir/lit.cfg.py") diff --git a/test/mlir/test/CMakeLists.txt b/test/mlir/test/CMakeLists.txt deleted file mode 100644 index e2a1ee621..000000000 --- a/test/mlir/test/CMakeLists.txt +++ /dev/null @@ -1,53 +0,0 @@ -if(WIN32) - set(MLIR_RUNNER_UTILS_DIR ${LLVM_BINARY_DIR}/bin) -else() - set(MLIR_RUNNER_UTILS_DIR ${LLVM_LIBRARY_DIR}) -endif() - -llvm_canonicalize_cmake_booleans( - GC_ENABLE_BINDINGS_PYTHON -) - -configure_lit_site_cfg( - ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in - ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py - MAIN_CONFIG - ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py -) - -configure_lit_site_cfg( - ${CMAKE_CURRENT_SOURCE_DIR}/gc/Unit/lit.site.cfg.py.in - ${CMAKE_CURRENT_BINARY_DIR}/gc/Unit/lit.site.cfg.py - MAIN_CONFIG - ${CMAKE_CURRENT_SOURCE_DIR}/gc/Unit/lit.cfg.py -) - -set(GC_OPT_TEST_DEPENDS - FileCheck count not split-file - # mlir-gen - gc-opt - gc-cpu-runner - GCUnitTests -) - -if(GC_ENABLE_GPU) - if(GC_ENABLE_IMEX) - include(imex) - endif() - list(APPEND GC_OPT_TEST_DEPENDS gc-gpu-runner) -endif() - -if(GC_ENABLE_BINDINGS_PYTHON) - list(APPEND GC_OPT_TEST_DEPENDS GcPythonModules) -endif() - -add_lit_testsuite(gc-check "Running the regression tests" - ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS ${GC_OPT_TEST_DEPENDS} -) - -# Looks that this property is suitable for IDE -# TODO: Check is this fine for IDE -set_target_properties(gc-check PROPERTIES FOLDER "Tests") - -add_lit_testsuites(GC_OPT ${CMAKE_CURRENT_SOURCE_DIR} DEPENDS ${GC_OPT_TEST_DEPENDS}) diff --git a/test/mlir/test/gc/Conversion/GPU/XeVMToLLVM/blockdpas.mlir b/test/mlir/test/gc/Conversion/GPU/XeVMToLLVM/blockdpas.mlir deleted file mode 100644 index cf25ea2e5..000000000 --- a/test/mlir/test/gc/Conversion/GPU/XeVMToLLVM/blockdpas.mlir +++ /dev/null @@ -1,92 +0,0 @@ -// RUN: gc-opt -convert-xevm-to-llvm -split-input-file %s | FileCheck %s - -gpu.module @dpas_funcs { -llvm.func @xevm.dpas.i8(%c : vector<8xi32>, %a : vector<8xi16>, %b : vector<8xi32>) { - // CHECK: llvm.func spir_funccc @_Z36intel_sub_group_i8_i8_matrix_mad_k32Dv8_sDv8_iS0_(vector<8xi16>, vector<8xi32>, vector<8xi32>) -> vector<8xi32> attributes {convergent, memory_effects = #llvm.memory_effects, no_unwind, will_return} - // CHECK: llvm.func @xevm.dpas.i8(%arg0: vector<8xi32>, %arg1: vector<8xi16>, %arg2: vector<8xi32>) { - // CHECK-NEXT: llvm.call spir_funccc @_Z36intel_sub_group_i8_i8_matrix_mad_k32Dv8_sDv8_iS0_(%arg1, %arg2, %arg0) {{.*}} : (vector<8xi16>, vector<8xi32>, vector<8xi32>) -> vector<8xi32> - %0 = xevm.dpas %c, %a, %b {pa = i8, pb = i8, rc = 8} : (vector<8xi32>, vector<8xi16>, vector<8xi32>) -> vector<8xi32> - llvm.return -} -} - -// ----- - -gpu.module @dpas_funcs { - // CHECK: llvm.func spir_funccc @_Z36intel_sub_group_u8_u8_matrix_mad_k32Dv8_sDv8_iS0_(vector<8xi16>, vector<8xi32>, vector<8xi32>) -> vector<8xi32> attributes {convergent, memory_effects = #llvm.memory_effects, no_unwind, will_return} -llvm.func @xevm.dpas.u8(%c : vector<8xi32>, %a : vector<8xi16>, %b : vector<8xi32>) { - // CHECK: llvm.func @xevm.dpas.u8(%arg0: vector<8xi32>, %arg1: vector<8xi16>, %arg2: vector<8xi32>) { - // CHECK-NEXT: llvm.call spir_funccc @_Z36intel_sub_group_u8_u8_matrix_mad_k32Dv8_sDv8_iS0_(%arg1, %arg2, %arg0) {{.*}} : (vector<8xi16>, vector<8xi32>, vector<8xi32>) -> vector<8xi32> - %0 = xevm.dpas %c, %a, %b {pa = u8, pb = u8, rc = 8} : (vector<8xi32>, vector<8xi16>, vector<8xi32>) -> vector<8xi32> - llvm.return -} -} -// ----- - -gpu.module @dpas_funcs { - // CHECK: llvm.func spir_funccc @_Z40intel_sub_group_bf16_bf16_matrix_mad_k16Dv8_sDv8_iDv8_f(vector<8xi16>, vector<8xi32>, vector<8xf32>) -> vector<8xf32> attributes {convergent, memory_effects = #llvm.memory_effects, no_unwind, will_return} -llvm.func @xevm.dpas.bf16(%c : vector<8xf32>, %a : vector<8xi16>, %b : vector<8xi32>) { - // CHECK: llvm.func @xevm.dpas.bf16(%arg0: vector<8xf32>, %arg1: vector<8xi16>, %arg2: vector<8xi32>) { - // CHECK-NEXT: llvm.call spir_funccc @_Z40intel_sub_group_bf16_bf16_matrix_mad_k16Dv8_sDv8_iDv8_f(%arg1, %arg2, %arg0) {{.*}} : (vector<8xi16>, vector<8xi32>, vector<8xf32>) -> vector<8xf32> - %0 = xevm.dpas %c, %a, %b {pa = bf16, pb = bf16, rc = 8} : (vector<8xf32>, vector<8xi16>, vector<8xi32>) -> vector<8xf32> - llvm.return -} -} - -// ----- - -gpu.module @dpas_funcs { - // CHECK: llvm.func spir_funccc @_Z38intel_sub_group_f16_f16_matrix_mad_k16Dv8_sDv8_iDv8_f(vector<8xi16>, vector<8xi32>, vector<8xf32>) -> vector<8xf32> attributes {convergent, memory_effects = #llvm.memory_effects, no_unwind, will_return} -llvm.func @xevm.dpas.f16(%c : vector<8xf32>, %a : vector<8xi16>, %b : vector<8xi32>) { - // CHECK: llvm.func @xevm.dpas.f16(%arg0: vector<8xf32>, %arg1: vector<8xi16>, %arg2: vector<8xi32>) { - // CHECK-NEXT: llvm.call spir_funccc @_Z38intel_sub_group_f16_f16_matrix_mad_k16Dv8_sDv8_iDv8_f(%arg1, %arg2, %arg0) {{.*}} : (vector<8xi16>, vector<8xi32>, vector<8xf32>) -> vector<8xf32> - %0 = xevm.dpas %c, %a, %b {pa = f16, pb = f16, rc = 8} : (vector<8xf32>, vector<8xi16>, vector<8xi32>) -> vector<8xf32> - llvm.return -} -} - -// ----- - -gpu.module @dpas_funcs { - // CHECK: llvm.func spir_funccc @_Z39intel_sub_group_tf32_tf32_matrix_mad_k8Dv4_fDv8_fS0_(vector<4xf32>, vector<8xf32>, vector<8xf32>) -> vector<8xf32> attributes {convergent, memory_effects = #llvm.memory_effects, no_unwind, will_return} -llvm.func @xevm.dpas.f32(%c : vector<8xf32>, %a : vector<4xf32>, %b : vector<8xf32>) { - // CHECK: llvm.func @xevm.dpas.f32(%arg0: vector<8xf32>, %arg1: vector<4xf32>, %arg2: vector<8xf32>) { - // CHECK-NEXT: llvm.call spir_funccc @_Z39intel_sub_group_tf32_tf32_matrix_mad_k8Dv4_fDv8_fS0_ - // CHECK-SAME: (%arg1, %arg2, %arg0) - // CHECK-SAME: convergent - // CHECK-SAME: : (vector<4xf32>, vector<8xf32>, vector<8xf32>) -> vector<8xf32> - %0 = xevm.dpas %c, %a, %b {pa = tf32, pb = tf32, rc = 8} : (vector<8xf32>, vector<4xf32>, vector<8xf32>) -> vector<8xf32> - llvm.return -} -} - -// ----- - -gpu.module @dpas_funcs { - // CHECK: llvm.func spir_funccc @_Z38intel_sub_group_f16_f16_matrix_mad_k16Dv8_sDv8_iDv8_Dh(vector<8xi16>, vector<8xi32>, vector<8xf16>) -> vector<8xf16> attributes {convergent, memory_effects = #llvm.memory_effects, no_unwind, will_return} -llvm.func @xevm.dpas.f16_accum(%c: vector<8xf16>, %a : vector<8xi16>, %b : vector<8xi32>) { - // CHECK: llvm.func @xevm.dpas.f16_accum(%arg0: vector<8xf16>, %arg1: vector<8xi16>, %arg2: vector<8xi32>) { - // CHECK-NEXT: lvm.call spir_funccc @_Z38intel_sub_group_f16_f16_matrix_mad_k16Dv8_sDv8_iDv8_Dh - // CHECK-SAME: (%arg1, %arg2, %arg0) - // CHECK-SAME: convergent - // CHECK-SAME: : (vector<8xi16>, vector<8xi32>, vector<8xf16>) -> vector<8xf16> - %0 = xevm.dpas %c, %a, %b {pa = f16, pb = f16, rc = 8} : (vector<8xf16>, vector<8xi16>, vector<8xi32>) -> vector<8xf16> - llvm.return -} -} -// ----- - -gpu.module @dpas_funcs { - // CHECK: llvm.func spir_funccc @_Z40intel_sub_group_bf16_bf16_matrix_mad_k16Dv8_sDv8_iS_(vector<8xi16>, vector<8xi32>, vector<8xi16>) -> vector<8xi16> attributes {convergent, memory_effects = #llvm.memory_effects, no_unwind, will_return} -llvm.func @xevm.dpas.bf16_accum(%c: vector<8xbf16>, %a : vector<8xi16>, %b : vector<8xi32>) { - // CHECK: llvm.func @xevm.dpas.bf16_accum(%arg0: vector<8xbf16>, %arg1: vector<8xi16>, %arg2: vector<8xi32>) { - // CHECK-NEXT: [[CAST:%.*]] = llvm.bitcast %arg0 : vector<8xbf16> to vector<8xi16> - // CHECK-NEXT: [[RES:%.*]] = llvm.call spir_funccc @_Z40intel_sub_group_bf16_bf16_matrix_mad_k16Dv8_sDv8_iS_ - // CHECK-SAME: (%arg1, %arg2, [[CAST]]) - // CHECK-SAME: convergent - // CHECK-SAME: : (vector<8xi16>, vector<8xi32>, vector<8xi16>) -> vector<8xi16> - %0 = xevm.dpas %c, %a, %b {pa = bf16, pb = bf16, rc = 8} : (vector<8xbf16>, vector<8xi16>, vector<8xi32>) -> vector<8xbf16> - // CHECK-NEXT: {{%.*}} = llvm.bitcast [[RES]] : vector<8xi16> to vector<8xbf16> - llvm.return -} -} \ No newline at end of file diff --git a/test/mlir/test/gc/Conversion/GPU/XeVMToLLVM/blockload2d.mlir b/test/mlir/test/gc/Conversion/GPU/XeVMToLLVM/blockload2d.mlir deleted file mode 100644 index 65278adaa..000000000 --- a/test/mlir/test/gc/Conversion/GPU/XeVMToLLVM/blockload2d.mlir +++ /dev/null @@ -1,464 +0,0 @@ -// RUN: gc-opt -convert-xevm-to-llvm -split-input-file %s | FileCheck %s - - -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.func @xevm.blockload2d(%arg0: !llvm.ptr<1>, %arg1: i32, %arg2: i32, %arg3: i32, %arg4: i32, %arg5: i32) { - // CHECK-DAG: [[ZERO:%.*]] = llvm.mlir.constant(0 : i32) : i32 - // CHECK-DAG: [[ONE:%.*]] = llvm.mlir.constant(1 : i32) : i32 - // CHECK-DAG: [[UNDEF:%.*]] = llvm.mlir.undef : vector<2xi32> - // CHECK-NEXT: [[COORD0:%.*]] = llvm.insertelement %arg4, [[UNDEF]][[[ZERO]] : i32] : vector<2xi32> - // CHECK-NEXT: [[COORD1:%.*]] = llvm.insertelement %arg5, [[COORD0]][[[ONE]] : i32] : vector<2xi32> - // CHECK: [[EIGHT:%.*]] = llvm.mlir.constant(8 : i32) : i32 - // CHECK-NEXT: [[DEST:%.*]] = llvm.alloca [[EIGHT]] x i16 : (i32) -> !llvm.ptr - // CHECK-NEXT: llvm.call spir_funccc @_Z40intel_sub_group_2d_block_read_8b_8r32x1cPU3AS1viiiDv2_iPt(%arg0, %arg1, %arg2, %arg3, [[COORD1]], [[DEST]]) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>, !llvm.ptr) -> () - // CHECK-NEXT: llvm.load [[DEST]] : !llvm.ptr -> vector<8xi16> - - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=8, tile_width=32, tile_height=8, v_blocks=1, transpose=false, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<8xi16> - llvm.return -} -} - -// ----- - -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z41intel_sub_group_2d_block_read_8b_16r32x1cPU3AS1viiiDv2_iPt(%arg0, %arg1, %arg2, %arg3, {{.*}}, [[DEST:%.*]]) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>, !llvm.ptr) -> () - // CHECK-NEXT: llvm.load [[DEST]] : !llvm.ptr -> vector<16xi16> - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=8, tile_width=32, tile_height=16, v_blocks=1, transpose=false, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<16xi16> - llvm.return -} -} - -// ----- - -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z41intel_sub_group_2d_block_read_8b_32r32x1cPU3AS1viiiDv2_iPt(%arg0, %arg1, %arg2, %arg3, {{.*}}, [[DEST:%.*]]) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>, !llvm.ptr) -> () - // CHECK-NEXT: llvm.load [[DEST]] : !llvm.ptr -> vector<32xi16> - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=8, tile_width=32, tile_height=32, v_blocks=1, transpose=false, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<32xi16> - llvm.return -} -} - -// ----- - -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z40intel_sub_group_2d_block_read_8b_8r16x1cPU3AS1viiiDv2_iPh(%arg0, %arg1, %arg2, %arg3, {{.*}}, [[DEST:%.*]]) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>, !llvm.ptr) -> () - // CHECK-NEXT: llvm.load [[DEST]] : !llvm.ptr -> vector<8xi8> - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=8, tile_width=16, tile_height=8, v_blocks=1, transpose=false, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<8xi8> - llvm.return -} -} - -// ----- - -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z40intel_sub_group_2d_block_read_8b_8r32x1cPU3AS1viiiDv2_iPh(%arg0, %arg1, %arg2, %arg3, {{.*}}, [[DEST:%.*]]) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>, !llvm.ptr) -> () - // CHECK-NEXT: llvm.load [[DEST]] : !llvm.ptr -> vector<16xi8> - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=8, tile_width=32, tile_height=8, v_blocks=1, transpose=false, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<16xi8> - llvm.return -} -} - -// ----- - -// COM: This case come from the 06 tutorial of FP8 flash attention. -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z40intel_sub_group_2d_block_read_8b_8r16x4cPU3AS1viiiDv2_iPh(%arg0, %arg1, %arg2, %arg3, {{.*}}, [[DEST:%.*]]) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>, !llvm.ptr) -> () - // CHECK-NEXT: llvm.load [[DEST]] : !llvm.ptr -> vector<32xi8> - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=8, tile_width=16, tile_height=8, v_blocks=4, transpose=false, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<32xi8> - llvm.return -} -} - -// ----- - -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z41intel_sub_group_2d_block_read_16b_8r16x1cPU3AS1viiiDv2_iPt(%arg0, %arg1, %arg2, %arg3, {{.*}}, [[DEST:%.*]]) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>, !llvm.ptr) -> () - // CHECK-NEXT: llvm.load [[DEST]] : !llvm.ptr -> vector<8xi16> - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=16, tile_width=16, tile_height=8, v_blocks=1, transpose=false, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<8xi16> - llvm.return -} -} - -// ----- - -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z42intel_sub_group_2d_block_read_16b_16r16x1cPU3AS1viiiDv2_iPt(%arg0, %arg1, %arg2, %arg3, {{.*}}, [[DEST:%.*]]) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>, !llvm.ptr) -> () - // CHECK-NEXT: llvm.load [[DEST]] : !llvm.ptr -> vector<16xi16> - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=16, tile_width=16, tile_height=16, v_blocks=1, transpose=false, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<16xi16> - llvm.return -} -} - -// ----- - -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z42intel_sub_group_2d_block_read_16b_32r16x1cPU3AS1viiiDv2_iPt(%arg0, %arg1, %arg2, %arg3, {{.*}}, [[DEST:%.*]]) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>, !llvm.ptr) -> () - // CHECK-NEXT: llvm.load [[DEST]] : !llvm.ptr -> vector<32xi16> - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=16, tile_width=16, tile_height=32, v_blocks=1, transpose=false, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<32xi16> - llvm.return -} -} - -// ----- - -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z41intel_sub_group_2d_block_read_16b_8r32x1cPU3AS1viiiDv2_iPt(%arg0, %arg1, %arg2, %arg3, {{.*}}, [[DEST:%.*]]) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>, !llvm.ptr) -> () - // CHECK-NEXT: llvm.load [[DEST]] : !llvm.ptr -> vector<16xi16> - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=16, tile_width=32, tile_height=8, v_blocks=1, transpose=false, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<16xi16> - llvm.return -} -} - -// ----- - -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z40intel_sub_group_2d_block_read_32b_8r8x1cPU3AS1viiiDv2_iPj(%arg0, %arg1, %arg2, %arg3, {{.*}}, [[DEST:%.*]]) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>, !llvm.ptr) -> () - // CHECK-NEXT: llvm.load [[DEST]] : !llvm.ptr -> vector<4xi32> - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=32, tile_width=8, tile_height=8, v_blocks=1, transpose=false, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<4xi32> - llvm.return -} -} - -// ----- - -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z41intel_sub_group_2d_block_read_32b_8r16x1cPU3AS1viiiDv2_iPj(%arg0, %arg1, %arg2, %arg3, {{.*}}, [[DEST:%.*]]) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>, !llvm.ptr) -> () - // CHECK-NEXT: llvm.load [[DEST]] : !llvm.ptr -> vector<8xi32> - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=32, tile_width=16, tile_height=8, v_blocks=1, transpose=false, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<8xi32> - llvm.return -} -} - -// ----- - -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z41intel_sub_group_2d_block_read_32b_16r8x1cPU3AS1viiiDv2_iPj(%arg0, %arg1, %arg2, %arg3, {{.*}}, [[DEST:%.*]]) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>, !llvm.ptr) -> () - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=32, tile_width=8, tile_height=16, v_blocks=1, transpose=false, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<8xi32> - llvm.return -} -} - -// ----- - -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z42intel_sub_group_2d_block_read_32b_16r16x1cPU3AS1viiiDv2_iPj(%arg0, %arg1, %arg2, %arg3, {{.*}}, [[DEST:%.*]]) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>, !llvm.ptr) -> () - // CHECK-NEXT: llvm.load [[DEST]] : !llvm.ptr -> vector<16xi32> - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=32, tile_width=16, tile_height=16, v_blocks=1, transpose=false, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<16xi32> - llvm.return -} -} - -// ----- - -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z41intel_sub_group_2d_block_read_32b_32r8x1cPU3AS1viiiDv2_iPj(%arg0, %arg1, %arg2, %arg3, {{.*}}, [[DEST:%.*]]) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>, !llvm.ptr) -> () - // CHECK-NEXT: llvm.load [[DEST]] : !llvm.ptr -> vector<16xi32> - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=32, tile_width=8, tile_height=32, v_blocks=1, transpose=false, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<16xi32> - llvm.return -} -} - -// ----- - -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z40intel_sub_group_2d_block_read_32b_8r2x1cPU3AS1viiiDv2_iPj(%arg0, %arg1, %arg2, %arg3, {{.*}}, [[DEST:%.*]]) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>, !llvm.ptr) -> () - // CHECK-NEXT: llvm.load [[DEST]] : !llvm.ptr -> vector<1xi32> - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=32, tile_width=2, tile_height=8, v_blocks=1, transpose=false, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<1xi32> - llvm.return -} -} - -// ----- - -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z40intel_sub_group_2d_block_read_8b_8r32x2cPU3AS1viiiDv2_iPt(%arg0, %arg1, %arg2, %arg3, {{.*}}, [[DEST:%.*]]) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>, !llvm.ptr) -> () - // CHECK-NEXT: llvm.load [[DEST]] : !llvm.ptr -> vector<16xi16> - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=8, tile_width=32, tile_height=8, v_blocks=2, transpose=false, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<16xi16> - llvm.return -} -} - -// ----- - -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z41intel_sub_group_2d_block_read_8b_16r32x2cPU3AS1viiiDv2_iPt(%arg0, %arg1, %arg2, %arg3, {{.*}}, [[DEST:%.*]]) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>, !llvm.ptr) -> () - // CHECK-NEXT: llvm.load [[DEST]] : !llvm.ptr -> vector<32xi16> - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=8, tile_width=32, tile_height=16, v_blocks=2, transpose=false, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<32xi16> - llvm.return -} -} - -// ----- - -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z41intel_sub_group_2d_block_read_8b_32r32x2cPU3AS1viiiDv2_iPt(%arg0, %arg1, %arg2, %arg3, {{.*}}, [[DEST:%.*]]) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>, !llvm.ptr) -> () - // CHECK-NEXT: llvm.load [[DEST]] : !llvm.ptr -> vector<64xi16> - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=8, tile_width=32, tile_height=32, v_blocks=2, transpose=false, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<64xi16> - llvm.return -} -} - -// ----- - -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z41intel_sub_group_2d_block_read_16b_8r16x2cPU3AS1viiiDv2_iPt(%arg0, %arg1, %arg2, %arg3, {{.*}}, [[DEST:%.*]]) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>, !llvm.ptr) -> () - // CHECK-NEXT: llvm.load [[DEST]] : !llvm.ptr -> vector<16xi16> - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=16, tile_width=16, tile_height=8, v_blocks=2, transpose=false, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<16xi16> - llvm.return -} -} - -// ----- - -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z42intel_sub_group_2d_block_read_16b_16r16x2cPU3AS1viiiDv2_iPt(%arg0, %arg1, %arg2, %arg3, {{.*}}, [[DEST:%.*]]) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>, !llvm.ptr) -> () - // CHECK-NEXT: llvm.load [[DEST]] : !llvm.ptr -> vector<32xi16> - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=16, tile_width=16, tile_height=16, v_blocks=2, transpose=false, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<32xi16> - llvm.return -} -} - -// ----- - -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z42intel_sub_group_2d_block_read_16b_32r16x2cPU3AS1viiiDv2_iPt(%arg0, %arg1, %arg2, %arg3, {{.*}}, [[DEST:%.*]]) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>, !llvm.ptr) -> () - // CHECK-NEXT: llvm.load [[DEST]] : !llvm.ptr -> vector<64xi16> - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=16, tile_width=16, tile_height=32, v_blocks=2, transpose=false, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<64xi16> - llvm.return -} -} - -// ----- - -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z40intel_sub_group_2d_block_read_32b_8r8x2cPU3AS1viiiDv2_iPj(%arg0, %arg1, %arg2, %arg3, {{.*}}, [[DEST:%.*]]) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>, !llvm.ptr) -> () - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=32, tile_width=8, tile_height=8, v_blocks=2, transpose=false, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<8xi32> - llvm.return -} -} - -// ----- - -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z41intel_sub_group_2d_block_read_32b_16r8x2cPU3AS1viiiDv2_iPj(%arg0, %arg1, %arg2, %arg3, {{.*}}, [[DEST:%.*]]) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>, !llvm.ptr) -> () - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=32, tile_width=8, tile_height=16, v_blocks=2, transpose=false, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<16xi32> - llvm.return -} -} - -// ----- - -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z41intel_sub_group_2d_block_read_32b_32r8x2cPU3AS1viiiDv2_iPj(%arg0, %arg1, %arg2, %arg3, {{.*}}, [[DEST:%.*]]) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>, !llvm.ptr) -> () - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=32, tile_width=8, tile_height=32, v_blocks=2, transpose=false, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<32xi32> - llvm.return -} -} - -// ----- - -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z51intel_sub_group_2d_block_read_transform_8b_32r16x1cPU3AS1viiiDv2_iPj(%arg0, %arg1, %arg2, %arg3, {{.*}}, [[DEST:%.*]]) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>, !llvm.ptr) -> () - // CHECK-NEXT: llvm.load [[DEST]] : !llvm.ptr -> vector<8xi32> - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=8, tile_width=16, tile_height=32, v_blocks=1, transpose=false, vnni_transform=true, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<8xi32> - llvm.return -} -} - -// ----- - -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z51intel_sub_group_2d_block_read_transform_8b_32r16x2cPU3AS1viiiDv2_iPj(%arg0, %arg1, %arg2, %arg3, {{.*}}, [[DEST:%.*]]) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>, !llvm.ptr) -> () - // CHECK-NEXT: llvm.load [[DEST]] : !llvm.ptr -> vector<16xi32> - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=8, tile_width=16, tile_height=32, v_blocks=2, transpose=false, vnni_transform=true, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<16xi32> - llvm.return -} -} - -// ----- - -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z51intel_sub_group_2d_block_read_transform_8b_32r16x4cPU3AS1viiiDv2_iPj(%arg0, %arg1, %arg2, %arg3, {{.*}}, [[DEST:%.*]]) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>, !llvm.ptr) -> () - // CHECK-NEXT: llvm.load [[DEST]] : !llvm.ptr -> vector<32xi32> - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=8, tile_width=16, tile_height=32, v_blocks=4, transpose=false, vnni_transform=true, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<32xi32> - llvm.return -} -} - -// ----- - -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z52intel_sub_group_2d_block_read_transform_16b_16r16x1cPU3AS1viiiDv2_iPj(%arg0, %arg1, %arg2, %arg3, {{.*}}, [[DEST:%.*]]) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>, !llvm.ptr) -> () - // CHECK-NEXT: llvm.load [[DEST]] : !llvm.ptr -> vector<8xi32> - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=16, tile_width=16, tile_height=16, v_blocks=1, transpose=false, vnni_transform=true, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<8xi32> - llvm.return -} -} - -// ----- - -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z52intel_sub_group_2d_block_read_transform_16b_32r16x1cPU3AS1viiiDv2_iPj(%arg0, %arg1, %arg2, %arg3, {{.*}}, [[DEST:%.*]]) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>, !llvm.ptr) -> () - // CHECK-NEXT: llvm.load [[DEST]] : !llvm.ptr -> vector<16xi32> - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=16, tile_width=16, tile_height=32, v_blocks=1, transpose=false, vnni_transform=true, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<16xi32> - llvm.return -} -} - -// ----- - -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z52intel_sub_group_2d_block_read_transform_16b_16r16x2cPU3AS1viiiDv2_iPj(%arg0, %arg1, %arg2, %arg3, {{.*}}, [[DEST:%.*]]) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>, !llvm.ptr) -> () - // CHECK-NEXT: llvm.load [[DEST]] : !llvm.ptr -> vector<16xi32> - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=16, tile_width=16, tile_height=16, v_blocks=2, transpose=false, vnni_transform=true, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<16xi32> - llvm.return -} -} - -// ----- - -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z52intel_sub_group_2d_block_read_transform_16b_32r16x2cPU3AS1viiiDv2_iPj(%arg0, %arg1, %arg2, %arg3, {{.*}}, [[DEST:%.*]]) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>, !llvm.ptr) -> () - // CHECK-NEXT: llvm.load [[DEST]] : !llvm.ptr -> vector<32xi32> - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=16, tile_width=16, tile_height=32, v_blocks=2, transpose=false, vnni_transform=true, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<32xi32> - llvm.return -} -} - -// ----- - -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z51intel_sub_group_2d_block_read_transpose_32b_16r8x1cPU3AS1viiiDv2_iPj(%arg0, %arg1, %arg2, %arg3, {{.*}}, [[DEST:%.*]]) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>, !llvm.ptr) -> () - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=32, tile_width=8, tile_height=16, v_blocks=1, transpose=true, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<8xi32> - llvm.return -} -} - -// ----- - -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.func @xevm.blockload2d( - // CHECK: llvm.call spir_funccc @_Z40intel_sub_group_2d_block_read_8b_8r32x2cPU3AS1viiiDv2_iPt( - // CHECK: xevm.DecorationCacheControl = {{\[\[}}6442 : i32, 0 : i32, 1 : i32, 0 : i32{{\]}}, {{\[}}6442 : i32, 1 : i32, 1 : i32, 0 : i32{{\]\]}} - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=8, tile_width=32, tile_height=8, v_blocks=2, transpose=false, vnni_transform=false, l1_cache_control=L1UC, l3_cache_control=L3UC} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<16xi16> - llvm.return -} -} - -// ----- - -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.func @xevm.blockload2d( - // CHECK: llvm.call spir_funccc @_Z40intel_sub_group_2d_block_read_8b_8r32x2cPU3AS1viiiDv2_iPt( - // CHECK: xevm.DecorationCacheControl = {{\[\[}}6442 : i32, 0 : i32, 1 : i32, 0 : i32{{\]}}, {{\[}}6442 : i32, 1 : i32, 2 : i32, 0 : i32{{\]\]}} - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=8, tile_width=32, tile_height=8, v_blocks=2, transpose=false, vnni_transform=false, l1_cache_control=L1UC, l3_cache_control=L3C} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<16xi16> - llvm.return -} -} - -// ----- - -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.func @xevm.blockload2d( - // CHECK: llvm.call spir_funccc @_Z40intel_sub_group_2d_block_read_8b_8r32x2cPU3AS1viiiDv2_iPt( - // CHECK: xevm.DecorationCacheControl = {{\[\[}}6442 : i32, 0 : i32, 2 : i32, 0 : i32{{\]}}, {{\[}}6442 : i32, 1 : i32, 1 : i32, 0 : i32{{\]\]}} - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=8, tile_width=32, tile_height=8, v_blocks=2, transpose=false, vnni_transform=false, l1_cache_control=L1C, l3_cache_control=L3UC} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<16xi16> - llvm.return -} -} - -// ----- - -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.func @xevm.blockload2d( - // CHECK: llvm.call spir_funccc @_Z40intel_sub_group_2d_block_read_8b_8r32x2cPU3AS1viiiDv2_iPt( - // CHECK: xevm.DecorationCacheControl = {{\[\[}}6442 : i32, 0 : i32, 2 : i32, 0 : i32{{\]}}, {{\[}}6442 : i32, 1 : i32, 2 : i32, 0 : i32{{\]\]}} - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=8, tile_width=32, tile_height=8, v_blocks=2, transpose=false, vnni_transform=false, l1_cache_control=L1C, l3_cache_control=L3C} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<16xi16> - llvm.return -} -} - -// ----- - -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.func @xevm.blockload2d( - // CHECK: llvm.call spir_funccc @_Z40intel_sub_group_2d_block_read_8b_8r32x2cPU3AS1viiiDv2_iPt( - // CHECK: xevm.DecorationCacheControl = {{\[\[}}6442 : i32, 0 : i32, 3 : i32, 0 : i32{{\]}}, {{\[}}6442 : i32, 1 : i32, 1 : i32, 0 : i32{{\]\]}} - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=8, tile_width=32, tile_height=8, v_blocks=2, transpose=false, vnni_transform=false, l1_cache_control=L1S, l3_cache_control=L3UC} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<16xi16> - llvm.return -} -} - -// ----- - -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.func @xevm.blockload2d( - // CHECK: llvm.call spir_funccc @_Z40intel_sub_group_2d_block_read_8b_8r32x2cPU3AS1viiiDv2_iPt( - // CHECK: xevm.DecorationCacheControl = {{\[\[}}6442 : i32, 0 : i32, 3 : i32, 0 : i32{{\]}}, {{\[}}6442 : i32, 1 : i32, 2 : i32, 0 : i32{{\]\]}} - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=8, tile_width=32, tile_height=8, v_blocks=2, transpose=false, vnni_transform=false, l1_cache_control=L1S, l3_cache_control=L3C} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<16xi16> - llvm.return -} -} - -// ----- - -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.func @xevm.blockload2d( - // CHECK: llvm.call spir_funccc @_Z40intel_sub_group_2d_block_read_8b_8r32x2cPU3AS1viiiDv2_iPt( - // CHECK: xevm.DecorationCacheControl = {{\[\[}}6442 : i32, 0 : i32, 4 : i32, 0 : i32{{\]}}, {{\[}}6442 : i32, 1 : i32, 2 : i32, 0 : i32{{\]\]}} - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=8, tile_width=32, tile_height=8, v_blocks=2, transpose=false, vnni_transform=false, l1_cache_control=L1IAR, l3_cache_control=L3C} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<16xi16> - llvm.return -} -} - -// ----- - -gpu.module @load_funcs { -llvm.func @xevm.blockload2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.func @xevm.blockload2d( - // CHECK: llvm.call spir_funccc @_Z40intel_sub_group_2d_block_read_8b_8r32x2cPU3AS1viiiDv2_iPt( - // CHECK-NOT: xevm.DecorationCacheControl - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=8, tile_width=32, tile_height=8, v_blocks=2, transpose=false, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<16xi16> - llvm.return -} -} diff --git a/test/mlir/test/gc/Conversion/GPU/XeVMToLLVM/blockprefetch2d.mlir b/test/mlir/test/gc/Conversion/GPU/XeVMToLLVM/blockprefetch2d.mlir deleted file mode 100644 index 63c158bbf..000000000 --- a/test/mlir/test/gc/Conversion/GPU/XeVMToLLVM/blockprefetch2d.mlir +++ /dev/null @@ -1,248 +0,0 @@ -// RUN: gc-opt -convert-xevm-to-llvm -split-input-file %s | FileCheck %s - -gpu.module @prefetch_funcs { -llvm.func @xevm.blockprefetch2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.func @xevm.blockprefetch2d(%arg0: !llvm.ptr<1>, %arg1: i32, %arg2: i32, %arg3: i32, %arg4: i32, %arg5: i32) { - // CHECK-DAG: [[UNDEF:%.*]] = llvm.mlir.undef : vector<2xi32> - // CHECK-DAG: [[ZERO:%.*]] = llvm.mlir.constant(0 : i32) : i32 - // CHECK-DAG: [[ONE:%.*]] = llvm.mlir.constant(1 : i32) : i32 - // CHECK-NEXT: [[COORD0:%.*]] = llvm.insertelement %arg4, [[UNDEF]][[[ZERO]] : i32] : vector<2xi32> - // CHECK-NEXT: [[COORD1:%.*]] = llvm.insertelement %arg5, [[COORD0]][[[ONE]] : i32] : vector<2xi32> - // CHECK-NEXT: llvm.call spir_funccc @_Z44intel_sub_group_2d_block_prefetch_8b_8r32x1cPU3AS1viiiDv2_i(%arg0, %arg1, %arg2, %arg3, [[COORD1]]) - // CHECK: xevm.DecorationCacheControl = {{\[\[}}6442 : i32, 0 : i32, 1 : i32, 0 : i32{{\]}}, {{\[}}6442 : i32, 1 : i32, 1 : i32, 0 : i32{{\]\]}} - // CHECK: (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>) -> () - xevm.blockprefetch2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=8, tile_width=32, tile_height=8, v_blocks=1, l1_cache_control=L1UC, l3_cache_control=L3UC} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) - llvm.return -} -} - -// ----- - - -gpu.module @prefetch_funcs { -llvm.func @xevm.blockprefetch2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z45intel_sub_group_2d_block_prefetch_8b_16r32x1cPU3AS1viiiDv2_i(%arg0, %arg1, %arg2, %arg3, {{.*}}) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>) -> () - xevm.blockprefetch2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=8, tile_width=32, tile_height=16, v_blocks=1, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) - llvm.return -} -} - -// ----- - -gpu.module @prefetch_funcs { -llvm.func @xevm.blockprefetch2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z45intel_sub_group_2d_block_prefetch_8b_32r32x1cPU3AS1viiiDv2_i(%arg0, %arg1, %arg2, %arg3, {{.*}}) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>) -> () - xevm.blockprefetch2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=8, tile_width=32, tile_height=32, v_blocks=1, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) - llvm.return -} -} - -// ----- - -gpu.module @prefetch_funcs { -llvm.func @xevm.blockprefetch2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z45intel_sub_group_2d_block_prefetch_16b_8r16x1cPU3AS1viiiDv2_i(%arg0, %arg1, %arg2, %arg3, {{.*}}) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>) -> () - xevm.blockprefetch2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=16, tile_width=16, tile_height=8, v_blocks=1, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) - llvm.return -} -} - -// ----- - -gpu.module @prefetch_funcs { -llvm.func @xevm.blockprefetch2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z46intel_sub_group_2d_block_prefetch_16b_16r16x1cPU3AS1viiiDv2_i(%arg0, %arg1, %arg2, %arg3, {{.*}}) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>) -> () - xevm.blockprefetch2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=16, tile_width=16, tile_height=16, v_blocks=1, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) - llvm.return -} -} - -// ----- - -gpu.module @prefetch_funcs { -llvm.func @xevm.blockprefetch2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z46intel_sub_group_2d_block_prefetch_16b_32r16x1cPU3AS1viiiDv2_i(%arg0, %arg1, %arg2, %arg3, {{.*}}) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>) -> () - xevm.blockprefetch2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=16, tile_width=16, tile_height=32, v_blocks=1, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) - llvm.return -} -} - -// ----- - -gpu.module @prefetch_funcs { -llvm.func @xevm.blockprefetch2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z44intel_sub_group_2d_block_prefetch_32b_8r8x1cPU3AS1viiiDv2_i(%arg0, %arg1, %arg2, %arg3, {{.*}}) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>) -> () - xevm.blockprefetch2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=32, tile_width=8, tile_height=8, v_blocks=1, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) - llvm.return -} -} - -// ----- - -gpu.module @prefetch_funcs { -llvm.func @xevm.blockprefetch2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z45intel_sub_group_2d_block_prefetch_32b_8r16x1cPU3AS1viiiDv2_i(%arg0, %arg1, %arg2, %arg3, {{.*}}) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>) -> () - xevm.blockprefetch2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=32, tile_width=16, tile_height=8, v_blocks=1, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) - llvm.return -} -} - -// ----- - -gpu.module @prefetch_funcs { -llvm.func @xevm.blockprefetch2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z45intel_sub_group_2d_block_prefetch_32b_16r8x1cPU3AS1viiiDv2_i(%arg0, %arg1, %arg2, %arg3, {{.*}}) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>) -> () - xevm.blockprefetch2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=32, tile_width=8, tile_height=16, v_blocks=1, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) - llvm.return -} -} - -// ----- - -gpu.module @prefetch_funcs { -llvm.func @xevm.blockprefetch2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z46intel_sub_group_2d_block_prefetch_32b_16r16x1cPU3AS1viiiDv2_i(%arg0, %arg1, %arg2, %arg3, {{.*}}) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>) -> () - xevm.blockprefetch2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=32, tile_width=16, tile_height=16, v_blocks=1, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) - llvm.return -} -} - -// ----- - -gpu.module @prefetch_funcs { -llvm.func @xevm.blockprefetch2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z45intel_sub_group_2d_block_prefetch_32b_32r8x1cPU3AS1viiiDv2_i(%arg0, %arg1, %arg2, %arg3, {{.*}}) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>) -> () - xevm.blockprefetch2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=32, tile_width=8, tile_height=32, v_blocks=1, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) - llvm.return -} -} - -// ----- - -gpu.module @prefetch_funcs { -llvm.func @xevm.blockprefetch2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z46intel_sub_group_2d_block_prefetch_32b_32r16x1cPU3AS1viiiDv2_i(%arg0, %arg1, %arg2, %arg3, {{.*}}) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>) -> () - xevm.blockprefetch2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=32, tile_width=16, tile_height=32, v_blocks=1, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) - llvm.return -} -} - -// ----- - -gpu.module @prefetch_funcs { -llvm.func @xevm.blockprefetch2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z44intel_sub_group_2d_block_prefetch_8b_8r32x2cPU3AS1viiiDv2_i(%arg0, %arg1, %arg2, %arg3, {{.*}}) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>) -> () - xevm.blockprefetch2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=8, tile_width=32, tile_height=8, v_blocks=2, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) - llvm.return -} -} - -// ----- - -gpu.module @prefetch_funcs { -llvm.func @xevm.blockprefetch2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z45intel_sub_group_2d_block_prefetch_8b_16r32x2cPU3AS1viiiDv2_i(%arg0, %arg1, %arg2, %arg3, {{.*}}) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>) -> () - xevm.blockprefetch2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=8, tile_width=32, tile_height=16, v_blocks=2, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) - llvm.return -} -} - -// ----- - -gpu.module @prefetch_funcs { -llvm.func @xevm.blockprefetch2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z45intel_sub_group_2d_block_prefetch_8b_32r32x2cPU3AS1viiiDv2_i(%arg0, %arg1, %arg2, %arg3, {{.*}}) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>) -> () - xevm.blockprefetch2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=8, tile_width=32, tile_height=32, v_blocks=2, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) - llvm.return -} -} - -// ----- - -gpu.module @prefetch_funcs { -llvm.func @xevm.blockprefetch2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z45intel_sub_group_2d_block_prefetch_16b_8r16x2cPU3AS1viiiDv2_i(%arg0, %arg1, %arg2, %arg3, {{.*}}) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>) -> () - xevm.blockprefetch2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=16, tile_width=16, tile_height=8, v_blocks=2, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) - llvm.return -} -} - -// ----- - -gpu.module @prefetch_funcs { -llvm.func @xevm.blockprefetch2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z46intel_sub_group_2d_block_prefetch_16b_16r16x2cPU3AS1viiiDv2_i(%arg0, %arg1, %arg2, %arg3, {{.*}}) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>) -> () - xevm.blockprefetch2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=16, tile_width=16, tile_height=16, v_blocks=2, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) - llvm.return -} -} - -// ----- - -gpu.module @prefetch_funcs { -llvm.func @xevm.blockprefetch2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z46intel_sub_group_2d_block_prefetch_16b_32r16x2cPU3AS1viiiDv2_i(%arg0, %arg1, %arg2, %arg3, {{.*}}) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>) -> () - xevm.blockprefetch2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=16, tile_width=16, tile_height=32, v_blocks=2, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) - llvm.return -} -} - -// ----- - -gpu.module @prefetch_funcs { -llvm.func @xevm.blockprefetch2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z44intel_sub_group_2d_block_prefetch_32b_8r8x2cPU3AS1viiiDv2_i(%arg0, %arg1, %arg2, %arg3, {{.*}}) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>) -> () - xevm.blockprefetch2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=32, tile_width=8, tile_height=8, v_blocks=2, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) - llvm.return -} -} - -// ----- - -gpu.module @prefetch_funcs { -llvm.func @xevm.blockprefetch2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z45intel_sub_group_2d_block_prefetch_32b_16r8x2cPU3AS1viiiDv2_i(%arg0, %arg1, %arg2, %arg3, {{.*}}) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>) -> () - xevm.blockprefetch2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=32, tile_width=8, tile_height=16, v_blocks=2, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) - llvm.return -} -} - -// ----- - -gpu.module @prefetch_funcs { -llvm.func @xevm.blockprefetch2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z45intel_sub_group_2d_block_prefetch_32b_32r8x2cPU3AS1viiiDv2_i(%arg0, %arg1, %arg2, %arg3, {{.*}}) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>) -> () - xevm.blockprefetch2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=32, tile_width=8, tile_height=32, v_blocks=2, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) - llvm.return -} -} - -// ----- - -gpu.module @prefetch_funcs { -llvm.func @xevm.blockprefetch2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z45intel_sub_group_2d_block_prefetch_8b_32r16x1cPU3AS1viiiDv2_i(%arg0, %arg1, %arg2, %arg3, {{.*}}) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>) -> () - xevm.blockprefetch2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=8, tile_width=16, tile_height=32, v_blocks=1, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) - llvm.return -} -} - -// ----- - -gpu.module @prefetch_funcs { -llvm.func @xevm.blockprefetch2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z45intel_sub_group_2d_block_prefetch_8b_32r16x2cPU3AS1viiiDv2_i(%arg0, %arg1, %arg2, %arg3, {{.*}}) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>) -> () - xevm.blockprefetch2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=8, tile_width=16, tile_height=32, v_blocks=2, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) - llvm.return -} -} - -// ----- - -gpu.module @prefetch_funcs { -llvm.func @xevm.blockprefetch2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // CHECK: llvm.call spir_funccc @_Z45intel_sub_group_2d_block_prefetch_8b_32r16x4cPU3AS1viiiDv2_i(%arg0, %arg1, %arg2, %arg3, {{.*}}) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>) -> () - xevm.blockprefetch2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=8, tile_width=16, tile_height=32, v_blocks=4, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) - llvm.return -} -} diff --git a/test/mlir/test/gc/Conversion/GPU/XeVMToLLVM/blockstore2d.mlir b/test/mlir/test/gc/Conversion/GPU/XeVMToLLVM/blockstore2d.mlir deleted file mode 100644 index ccb06f029..000000000 --- a/test/mlir/test/gc/Conversion/GPU/XeVMToLLVM/blockstore2d.mlir +++ /dev/null @@ -1,51 +0,0 @@ -// RUN: gc-opt -convert-xevm-to-llvm -split-input-file %s | FileCheck %s - - -gpu.module @store_funcs { -llvm.func @xevm.blockstore2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32, %stored_val : vector<8xi8>) { - // CHECK: llvm.func @xevm.blockstore2d(%arg0: !llvm.ptr<1>, %arg1: i32, %arg2: i32, %arg3: i32, %arg4: i32, %arg5: i32, %arg6: vector<8xi8>) { - // CHECK-DAG: [[ZERO:%.*]] = llvm.mlir.constant(0 : i32) : i32 - // CHECK-DAG: [[ONE:%.*]] = llvm.mlir.constant(1 : i32) : i32 - // CHECK-DAG: [[UNDEF:%.*]] = llvm.mlir.undef : vector<2xi32> - // CHECK-NEXT: [[COORD0:%.*]] = llvm.insertelement %arg4, [[UNDEF]][[[ZERO]] : i32] : vector<2xi32> - // CHECK-NEXT: [[COORD1:%.*]] = llvm.insertelement %arg5, [[COORD0]][[[ONE]] : i32] : vector<2xi32> - // CHECK: [[C8:%.*]] = llvm.mlir.constant(8 : i32) : i32 - // CHECK-NEXT: [[STOREVALPTR:%.*]] = llvm.alloca [[C8]] x i8 : (i32) -> !llvm.ptr - // CHECK-NEXT: llvm.store %arg6, [[STOREVALPTR]] : vector<8xi8>, !llvm.ptr - // CHECK-NEXT: llvm.call spir_funccc @_Z41intel_sub_group_2d_block_write_8b_8r16x1cPU3AS1viiiDv2_iPh(%arg0, %arg1, %arg2, %arg3, [[COORD1]], [[STOREVALPTR]]) - // CHECK: xevm.DecorationCacheControl = {{\[\[}}6443 : i32, 0 : i32, 1 : i32, 0 : i32{{\]}}, {{\[}}6443 : i32, 1 : i32, 1 : i32, 0 : i32{{\]\]}} - // CHECK: : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>, !llvm.ptr) -> () - xevm.blockstore2d %ptr, %base_width, %base_height, %base_pitch, %x, %y, %stored_val {elem_size_in_bits=8, tile_width=16, tile_height=8, v_blocks=1, l1_cache_control=L1UC, l3_cache_control=L3UC} : (!llvm.ptr<1>, i32, i32, i32, i32, i32, vector<8xi8>) - llvm.return -} -} - -// ----- - -gpu.module @store_funcs { -llvm.func @xevm.blockstore2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32, %stored_val : vector<8xi16>) { - // CHECK: llvm.call spir_funccc @_Z41intel_sub_group_2d_block_write_8b_8r32x1cPU3AS1viiiDv2_iPt(%arg0, %arg1, %arg2, %arg3, {{.*}}, [[DEST:%.*]]) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>, !llvm.ptr) -> () - xevm.blockstore2d %ptr, %base_width, %base_height, %base_pitch, %x, %y, %stored_val {elem_size_in_bits=8, tile_width=32, tile_height=8, v_blocks=1, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32, vector<8xi16>) - llvm.return -} -} - -// ----- - -gpu.module @store_funcs { -llvm.func @xevm.blockstore2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32, %stored_val : vector<8xi16>) { - // CHECK: llvm.call spir_funccc @_Z42intel_sub_group_2d_block_write_16b_8r16x1cPU3AS1viiiDv2_iPt(%arg0, %arg1, %arg2, %arg3, {{.*}}, [[DEST:%.*]]) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>, !llvm.ptr) -> () - xevm.blockstore2d %ptr, %base_width, %base_height, %base_pitch, %x, %y, %stored_val {elem_size_in_bits=16, tile_width=16, tile_height=8, v_blocks=1, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32, vector<8xi16>) - llvm.return -} -} - -// ----- - -gpu.module @store_funcs { -llvm.func @xevm.blockstore2d(%ptr : !llvm.ptr<1>, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32, %stored_val : vector<8xi32>) { - // CHECK: llvm.call spir_funccc @_Z42intel_sub_group_2d_block_write_32b_8r16x1cPU3AS1viiiDv2_iPj(%arg0, %arg1, %arg2, %arg3, {{.*}}, [[DEST:%.*]]) {{.*}} : (!llvm.ptr<1>, i32, i32, i32, vector<2xi32>, !llvm.ptr) -> () - xevm.blockstore2d %ptr, %base_width, %base_height, %base_pitch, %x, %y, %stored_val {elem_size_in_bits=32, tile_width=16, tile_height=8, v_blocks=1, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32, vector<8xi32>) - llvm.return -} -} \ No newline at end of file diff --git a/test/mlir/test/gc/Dialect/CPURuntime/cpu-runner/allocators.mlir b/test/mlir/test/gc/Dialect/CPURuntime/cpu-runner/allocators.mlir deleted file mode 100644 index 399467290..000000000 --- a/test/mlir/test/gc/Dialect/CPURuntime/cpu-runner/allocators.mlir +++ /dev/null @@ -1,37 +0,0 @@ -// RUN: gc-opt %s --finalize-memref-to-llvm --convert-scf-to-cf --convert-cpuruntime-to-llvm --convert-func-to-llvm --reconcile-unrealized-casts | gc-cpu-runner -e main -entry-point-result=void -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils | FileCheck %s - -module { - func.func @doAlloc() { - %m0 = cpuruntime.alloc () : memref<13xf32> - %c0 = arith.constant 0 : index - %cst = arith.constant 42.0 : f32 - memref.store %cst, %m0[%c0] : memref<13xf32> - %val = memref.load %m0[%c0] : memref<13xf32> - cpuruntime.printf "main alloc stored val %f\n" %val : f32 - cpuruntime.dealloc %m0 : memref<13xf32> - return - } - - func.func @doThreadAlloc() { - scf.forall (%arg2) in (3) { - %m0 = cpuruntime.alloc thread_local () : memref<13xf32> - %c0 = arith.constant 0 : index - %cst = arith.constant 56.0 : f32 - memref.store %cst, %m0[%c0] : memref<13xf32> - %val = memref.load %m0[%c0] : memref<13xf32> - cpuruntime.printf "thread alloc stored val %f\n" %val : f32 - cpuruntime.dealloc thread_local %m0 : memref<13xf32> - } - return - } - - func.func @main() { - call @doAlloc() : () -> () - call @doThreadAlloc() : () -> () - return - } - // CHECK: stored val 42.000000 - // CHECK: thread alloc stored val 56.000000 - // CHECK: thread alloc stored val 56.000000 - // CHECK: thread alloc stored val 56.000000 -} diff --git a/test/mlir/test/gc/Dialect/CPURuntime/cpu-runner/printf.mlir b/test/mlir/test/gc/Dialect/CPURuntime/cpu-runner/printf.mlir deleted file mode 100644 index e95471d50..000000000 --- a/test/mlir/test/gc/Dialect/CPURuntime/cpu-runner/printf.mlir +++ /dev/null @@ -1,17 +0,0 @@ -// RUN: gc-opt %s --convert-cpuruntime-to-llvm --convert-func-to-llvm --convert-arith-to-llvm --convert-cf-to-llvm --convert-complex-to-llvm | gc-cpu-runner -e main -entry-point-result=void -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils | FileCheck %s - -module { - func.func @doprint(%t: f32, %t2: i32, %t3: i64) { - cpuruntime.printf "Hello world %f %d %lld\n" %t, %t2, %t3 : f32, i32, i64 - return - } - - func.func @main() { - %c2 = arith.constant 2.0 : f32 - %c32i = arith.constant 2000000 : i32 - %c64i = arith.constant 2000000 : i64 - call @doprint(%c2, %c32i, %c64i) : (f32, i32, i64) -> () - return - } - // CHECK: Hello world 2.000000 2000000 2000000 -} \ No newline at end of file diff --git a/test/mlir/test/gc/Dialect/CPURuntime/cpuruntime-to-llvm.mlir b/test/mlir/test/gc/Dialect/CPURuntime/cpuruntime-to-llvm.mlir deleted file mode 100644 index c480d16eb..000000000 --- a/test/mlir/test/gc/Dialect/CPURuntime/cpuruntime-to-llvm.mlir +++ /dev/null @@ -1,134 +0,0 @@ -// RUN: gc-opt %s --convert-cpuruntime-to-llvm | FileCheck %s - -module { - // CHECK: llvm.func @gcThreadAlignedFree(!llvm.ptr) - // CHECK: llvm.func @gcThreadAlignedMalloc(i64) -> !llvm.ptr - // CHECK: llvm.func @gcAlignedFree(!llvm.ptr) - // CHECK: llvm.func @gcAlignedMalloc(i64) -> !llvm.ptr - // CHECK: llvm.mlir.global internal constant @cpuprintfFormat_0("Hello world %f %d %lld\0A\00") {addr_space = 0 : i32} - // CHECK: llvm.func @printf(!llvm.ptr, - // CHECK-NEXT: func.func @doprint(%[[ARG0:.*]]: f32, %[[ARG1:.*]]: i32, %[[ARG2:.*]]: i64) - func.func @doprint(%t: f32, %t2: i32, %t3: i64) { - // CHECK-NEXT: llvm.mlir.addressof - // CHECK-DAG: %[[C1:.*]] = llvm.getelementptr - // CHECK-SAME: !llvm.ptr, !llvm.array<24 x i8> - // CHECK: %[[C2:.*]] = llvm.fpext %[[ARG0]] - // CHECK: %[[C3:.*]] = llvm.zext %[[ARG1]] - // CHECK-NOT: cpuruntime.printf - // CHECK-NEXT: llvm.call @printf(%[[C1]], %[[C2]], %[[C3]], %[[ARG2]]) - cpuruntime.printf "Hello world %f %d %lld\n" %t, %t2, %t3 : f32, i32, i64 - return - } - - func.func @doalloc() { - // CHECK: %[[c1:.*]] = llvm.mlir.constant(13 : index) : i64 - // CHECK: %[[c2:.*]] = llvm.mlir.constant(1 : index) : i64 - // CHECK: %[[null:.*]] = llvm.mlir.zero : !llvm.ptr - // CHECK: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[c1]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 - // CHECK: %[[size_bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64 - // CHECK: %[[call:.*]] = llvm.call @gcAlignedMalloc(%[[size_bytes]]) : (i64) -> !llvm.ptr - // CHECK: %[[ptr:.*]] = llvm.bitcast %[[call]] : !llvm.ptr to !llvm.ptr - // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: llvm.insertvalue %[[ptr]], %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: llvm.insertvalue %[[ptr]], %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: %[[c0:.*]] = llvm.mlir.constant(0 : index) : i64 - // CHECK: llvm.insertvalue %[[c0]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: llvm.insertvalue %[[c1]], %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: llvm.insertvalue %[[c2]], %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: %[[callfree:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: %[[ptrfree:.*]] = llvm.bitcast %[[callfree]] : !llvm.ptr to !llvm.ptr - // CHECK: llvm.call @gcAlignedFree(%[[ptrfree]]) : (!llvm.ptr) -> () - %m0 = cpuruntime.alloc () : memref<13xf32> - cpuruntime.dealloc %m0 : memref<13xf32> - return - } - - func.func @do_thread_alloc() { - // CHECK: %[[c1:.*]] = llvm.mlir.constant(13 : index) : i64 - // CHECK: %[[c2:.*]] = llvm.mlir.constant(1 : index) : i64 - // CHECK: %[[null:.*]] = llvm.mlir.zero : !llvm.ptr - // CHECK: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[c1]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 - // CHECK: %[[size_bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64 - // CHECK: %[[call:.*]] = llvm.call @gcThreadAlignedMalloc(%[[size_bytes]]) : (i64) -> !llvm.ptr - // CHECK: %[[ptr:.*]] = llvm.bitcast %[[call]] : !llvm.ptr to !llvm.ptr - // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: llvm.insertvalue %[[ptr]], %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: llvm.insertvalue %[[ptr]], %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: %[[c0:.*]] = llvm.mlir.constant(0 : index) : i64 - // CHECK: llvm.insertvalue %[[c0]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: llvm.insertvalue %[[c1]], %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: llvm.insertvalue %[[c2]], %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: %[[callfree:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: %[[ptrfree:.*]] = llvm.bitcast %[[callfree]] : !llvm.ptr to !llvm.ptr - // CHECK: llvm.call @gcThreadAlignedFree(%[[ptrfree]]) : (!llvm.ptr) -> () - %m0 = cpuruntime.alloc thread_local () : memref<13xf32> - cpuruntime.dealloc thread_local %m0 : memref<13xf32> - return - } - - // CHECK-LABEL: func @dynamic_alloc( - // CHECK: %[[Marg:.*]]: index, %[[Narg:.*]]: index) - func.func @dynamic_alloc(%arg0: index, %arg1: index) { - // CHECK-DAG: %[[M:.*]] = builtin.unrealized_conversion_cast %[[Marg]] - // CHECK-DAG: %[[N:.*]] = builtin.unrealized_conversion_cast %[[Narg]] - // CHECK-NEXT: %[[fortytwo:.*]] = llvm.mlir.constant(42 : index) : i64 - // CHECK-NEXT: %[[one:.*]] = llvm.mlir.constant(1 : index) : i64 - // CHECK-NEXT: %[[mul:.*]] = llvm.mul %[[N]], %[[fortytwo]] : i64 - // CHECK-NEXT: %[[sz:.*]] = llvm.mul %[[mul]], %[[M]] : i64 - // CHECK-NEXT: %[[null:.*]] = llvm.mlir.zero : !llvm.ptr - // CHECK-NEXT: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[sz]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 - // CHECK-NEXT: %[[sz_bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64 - // CHECK-NEXT: %[[call:.*]] = llvm.call @gcAlignedMalloc(%[[sz_bytes]]) : (i64) -> !llvm.ptr - // CHECK-NEXT: llvm.bitcast %[[call]] : !llvm.ptr to !llvm.ptr - // CHECK-NEXT: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK-NEXT: llvm.insertvalue %{{.*}}, %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK-NEXT: llvm.insertvalue %{{.*}}, %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK-NEXT: %[[off:.*]] = llvm.mlir.constant(0 : index) : i64 - // CHECK-NEXT: llvm.insertvalue %[[off]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK-NEXT: llvm.insertvalue %[[M]], %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK-NEXT: llvm.insertvalue %[[fortytwo]], %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK-NEXT: llvm.insertvalue %[[N]], %{{.*}}[3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK-NEXT: llvm.insertvalue %[[mul]], %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK-NEXT: llvm.insertvalue %[[N]], %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK-NEXT: llvm.insertvalue %[[one]], %{{.*}}[4, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK-NEXT: %[[callfree:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK-NEXT: %[[ptrfree:.*]] = llvm.bitcast %[[callfree]] : !llvm.ptr to !llvm.ptr - // CHECK-NEXT: llvm.call @gcAlignedFree(%[[ptrfree]]) : (!llvm.ptr) -> () - %m0 = cpuruntime.alloc(%arg0, %arg1) : memref - cpuruntime.dealloc %m0 : memref - return - } - - // CHECK-LABEL: func @dynamic_thread_alloc( - // CHECK: %[[Marg:.*]]: index, %[[Narg:.*]]: index) - func.func @dynamic_thread_alloc(%arg0: index, %arg1: index) { - // CHECK-DAG: %[[M:.*]] = builtin.unrealized_conversion_cast %[[Marg]] - // CHECK-DAG: %[[N:.*]] = builtin.unrealized_conversion_cast %[[Narg]] - // CHECK-NEXT: %[[fortytwo:.*]] = llvm.mlir.constant(42 : index) : i64 - // CHECK-NEXT: %[[one:.*]] = llvm.mlir.constant(1 : index) : i64 - // CHECK-NEXT: %[[mul:.*]] = llvm.mul %[[N]], %[[fortytwo]] : i64 - // CHECK-NEXT: %[[sz:.*]] = llvm.mul %[[mul]], %[[M]] : i64 - // CHECK-NEXT: %[[null:.*]] = llvm.mlir.zero : !llvm.ptr - // CHECK-NEXT: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[sz]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 - // CHECK-NEXT: %[[sz_bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64 - // CHECK-NEXT: %[[call:.*]] = llvm.call @gcThreadAlignedMalloc(%[[sz_bytes]]) : (i64) -> !llvm.ptr - // CHECK-NEXT: llvm.bitcast %[[call]] : !llvm.ptr to !llvm.ptr - // CHECK-NEXT: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK-NEXT: llvm.insertvalue %{{.*}}, %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK-NEXT: llvm.insertvalue %{{.*}}, %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK-NEXT: %[[off:.*]] = llvm.mlir.constant(0 : index) : i64 - // CHECK-NEXT: llvm.insertvalue %[[off]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK-NEXT: llvm.insertvalue %[[M]], %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK-NEXT: llvm.insertvalue %[[fortytwo]], %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK-NEXT: llvm.insertvalue %[[N]], %{{.*}}[3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK-NEXT: llvm.insertvalue %[[mul]], %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK-NEXT: llvm.insertvalue %[[N]], %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK-NEXT: llvm.insertvalue %[[one]], %{{.*}}[4, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK-NEXT: %[[callfree:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK-NEXT: %[[ptrfree:.*]] = llvm.bitcast %[[callfree]] : !llvm.ptr to !llvm.ptr - // CHECK-NEXT: llvm.call @gcThreadAlignedFree(%[[ptrfree]]) : (!llvm.ptr) -> () - %m0 = cpuruntime.alloc thread_local (%arg0, %arg1) : memref - cpuruntime.dealloc thread_local %m0 : memref - return - } -} diff --git a/test/mlir/test/gc/Dialect/CPURuntime/memref-to-cpuruntime.mlir b/test/mlir/test/gc/Dialect/CPURuntime/memref-to-cpuruntime.mlir deleted file mode 100644 index c32cb618e..000000000 --- a/test/mlir/test/gc/Dialect/CPURuntime/memref-to-cpuruntime.mlir +++ /dev/null @@ -1,89 +0,0 @@ -// RUN: gc-opt --split-input-file --convert-memref-to-cpuruntime %s -verify-diagnostics | FileCheck %s - -func.func @alloca() { - // CHECK-LABEL: func @alloca() - // CHECK: %[[m0:.*]] = cpuruntime.alloc() : memref<1024xf32> - %m0 = memref.alloca() : memref<1024xf32> - scf.forall (%i) in (32) { - } - // CHECK: cpuruntime.dealloc %[[m0]] : memref<1024xf32> - return -} - -func.func @thread_alloca() { - // CHECK-LABEL: func.func @thread_alloca() - // CHECK-NEXT: scf.forall {{.*}} { - // CHECK-NEXT: %[[m0:.*]] = cpuruntime.alloc thread_local() : memref<1024xf32> - // CHECK-NEXT: cpuruntime.dealloc thread_local %[[m0]] : memref<1024xf32> - // CHECK-NEXT: } - scf.forall (%i) in (32) { - %0 = memref.alloca() : memref<1024xf32> - } - return -} - -func.func @dynamic_ranked_alloca(%arg0: memref<*xf32>) { - // CHECK-LABEL: func @dynamic_ranked_alloca(%arg0: memref<*xf32>) - // CHECK: %[[RANK:.*]] = memref.rank %{{.*}} : memref<*xf32> - // CHECK: %[[m0:.*]] = cpuruntime.alloc(%[[RANK]]) : memref - // CHECK: cpuruntime.dealloc %[[m0]] : memref - %0 = memref.rank %arg0 : memref<*xf32> - %alloca = memref.alloca(%0) : memref - return -} - -func.func @loop_nested_if_alloca(%arg0: index, %arg1: index, %arg2: index, %arg3: memref<64xf32>) { - // CHECK-LABEL: func @loop_nested_if_alloca(%arg0: index, %arg1: index, %arg2: index, %arg3: memref<64xf32>) - // CHECK: %[[m0:.*]] = cpuruntime.alloc() : memref<64xf32> - %alloca = memref.alloca() : memref<64xf32> - %0 = scf.for %arg5 = %arg0 to %arg1 step %arg2 iter_args(%arg6 = %arg3) -> (memref<64xf32>) { - %1 = arith.cmpi eq, %arg5, %arg1 : index - %2 = scf.if %1 -> (memref<64xf32>) { - // CHECK: yield %[[m0]] : memref<64xf32> - scf.yield %alloca : memref<64xf32> - } else { - // CHECK: %[[m1:.*]] = memref.alloca() : memref<2xf32> - %alloca_0 = memref.alloca() : memref<2xf32> - scf.yield %arg6 : memref<64xf32> - } - scf.yield %2 : memref<64xf32> - } - // CHECK: cpuruntime.dealloc %[[m0]] : memref<64xf32> - return -} - -func.func @alloca_sequence() { - // CHECK-LABEL: func @alloca_sequence() - // CHECK: %[[m0:.*]] = cpuruntime.alloc() : memref<128xf32> - // CHECK: %[[m1:.*]] = cpuruntime.alloc() : memref<128xf32> - // CHECK: %[[m2:.*]] = cpuruntime.alloc() : memref<128xf32> - %alloc = memref.alloca() : memref<128xf32> - %alloc_0 = memref.alloca() : memref<128xf32> - %alloc_1 = memref.alloca() : memref<128xf32> - // CHECK: cpuruntime.dealloc %[[m2]] : memref<128xf32> - // CHECK: cpuruntime.dealloc %[[m1]] : memref<128xf32> - // CHECK: cpuruntime.dealloc %[[m0]] : memref<128xf32> - return -} - -func.func @nested_alloca() { - // CHECK-LABEL: func @nested_alloca() - // CHECK: %[[m0:.*]] = cpuruntime.alloc() : memref<512xf32> - // CHECK-NEXT: scf.forall {{.*}} { - // CHECK-NEXT: %[[m1:.*]] = cpuruntime.alloc thread_local() : memref<32xf32> - // CHECK-NEXT: cpuruntime.dealloc thread_local %[[m1]] : memref<32xf32> - // CHECK-NEXT: } - // CHECK-NEXT: scf.forall {{.*}} { - // CHECK-NEXT: %[[m2:.*]] = cpuruntime.alloc thread_local() : memref<64xf32> - // CHECK-NEXT: cpuruntime.dealloc thread_local %[[m2]] : memref<64xf32> - // CHECK-NEXT: } - // CHECK: cpuruntime.dealloc %[[m0]] : memref<512xf32> - %0 = memref.alloca() : memref<512xf32> - scf.forall (%i) in (32) { - %1 = memref.alloca() : memref<32xf32> - } - scf.forall (%i) in (32) { - %1 = memref.alloca() : memref<64xf32> - } - return -} diff --git a/test/mlir/test/gc/Dialect/Linlagx/generalize-named-ops.mlir b/test/mlir/test/gc/Dialect/Linlagx/generalize-named-ops.mlir deleted file mode 100644 index 562e020e7..000000000 --- a/test/mlir/test/gc/Dialect/Linlagx/generalize-named-ops.mlir +++ /dev/null @@ -1,24 +0,0 @@ -// RUN: gc-opt -split-input-file -linalg-generalize-named-ops -verify-diagnostics %s | FileCheck %s - -func.func @generalize_sigmoid(%arg0: tensor<4x256x64xbf16>, %arg1: tensor<4x256x64xbf16>) -> tensor<4x256x64xbf16> { - %0 = linalgx.sigmoid ins(%arg0 : tensor<4x256x64xbf16>) outs(%arg1 : tensor<4x256x64xbf16>) -> tensor<4x256x64xbf16> - return %0 : tensor<4x256x64xbf16> -} - -// CHECK: #[[MAP:.+]] = affine_map<(d0, d1, d2) -> (d0, d1, d2)> - -// CHECK: func @generalize_sigmoid -// CHECK-SAME: (%[[ARG:.+]]: tensor<4x256x64xbf16>, %[[OUT:.+]]: tensor<4x256x64xbf16>) - -// CHECK: %[[CST:.+]] = arith.constant 1 -// CHECK: linalg.generic -// CHECK-SAME: indexing_maps = [#[[MAP]], #[[MAP]]] -// CHECK-SAME: iterator_types = ["parallel", "parallel", "parallel"]} -// CHECK-SAME: ins(%[[ARG]] : tensor<4x256x64xbf16>) outs(%[[OUT]] : tensor<4x256x64xbf16>) - -// CHECK: ^{{.*}}(%[[BBARG0:.+]]: bf16, %[[BBARG1:.+]]: bf16) -// CHECK-NEXT: %[[NEG:.+]] = arith.negf %[[BBARG0]] : bf16 -// CHECK-NEXT: %[[EXP:.+]] = math.exp %[[NEG]] : bf16 -// CHECK-NEXT: %[[ADD:.+]] = arith.addf %[[EXP]], %[[CST]] : bf16 -// CHECK-NEXT: %[[DIV:.+]] = arith.divf %[[CST]], %[[ADD]] : bf16 -// CHECK-NEXT: linalg.yield %[[DIV]] : bf16 diff --git a/test/mlir/test/gc/Dialect/Linlagx/linalg-generalize.mlir b/test/mlir/test/gc/Dialect/Linlagx/linalg-generalize.mlir deleted file mode 100644 index d53c78f4c..000000000 --- a/test/mlir/test/gc/Dialect/Linlagx/linalg-generalize.mlir +++ /dev/null @@ -1,73 +0,0 @@ -// RUN: gc-opt %s --split-input-file --linalg-generalize-named-ops | FileCheck %s - - -// CHECK: #[[$MAP0:.+]] = affine_map<(d0, d1) -> (d1, d0)> -// CHECK: #[[$MAP1:.+]] = affine_map<(d0, d1) -> (d0, d1)> -// CHECK: #[[$MAP2:.+]] = affine_map<(d0, d1) -> ()> -// CHECK: #[[$MAP3:.+]] = affine_map<(d0, d1, d2) -> (d0, d2)> -// CHECK: #[[$MAP4:.+]] = affine_map<(d0, d1, d2) -> (d2, d1)> -// CHECK: #[[$MAP5:.+]] = affine_map<(d0, d1, d2) -> (d0, d1)> -// CHECK: #[[$MAP6:.+]] = affine_map<(d0, d1) -> (d1)> -// CHECK: #[[$MAP7:.+]] = affine_map<(d0, d1) -> (0, d1)> -#map = affine_map<(d0, d1) -> (d1)> -#map1 = affine_map<(d0, d1) -> (0, d1)> -#map2 = affine_map<(d0, d1) -> (d0, d1)> -// CHECK-LABEL: mlp -// CHECK-SAME: %[[ARG0:.+]]: tensor<8192x16384xf32>, %[[ARG1:.+]]: tensor<8192xf32>, %[[ARG2:.+]]: tensor<1x128x128xf32> -func.func @mlp(%arg0: tensor<8192x16384xf32>, %arg1: tensor<8192xf32>, %arg2: tensor<1x128x128xf32>) -> tensor<1x8192xf32> { - // CHECK: %[[CST:.+]] = arith.constant 0.000000e+00 : f32 - %cst = arith.constant 0.000000e+00 : f32 - - // CHECK: %[[COLLAPSE:.+]] = tensor.collapse_shape %arg2 - // CHECK-SAME{literal}: [[0], [1, 2]] : tensor<1x128x128xf32> into tensor<1x16384xf32> - %collapsed = tensor.collapse_shape %arg2 [[0], [1, 2]] : tensor<1x128x128xf32> into tensor<1x16384xf32> - - // CHECK: %[[EMPTY:.+]] = tensor.empty() : tensor<16384x8192xf32> - %0 = tensor.empty() : tensor<16384x8192xf32> - - // CHECK: %[[TRANSPOSED:.+]] = linalg.generic - // CHECK-SAME: {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel"]} - // CHECK-SAME: ins(%arg0 : tensor<8192x16384xf32>) outs(%[[EMPTY]] : tensor<16384x8192xf32>) { - // CHECK: ^bb0(%[[IN:.+]]: f32, %[[OUT:.+]]: f32): - // CHECK: linalg.yield %[[IN]] : f32 - // CHECK: } -> tensor<16384x8192xf32> - %transposed = linalg.transpose ins(%arg0 : tensor<8192x16384xf32>) outs(%0 : tensor<16384x8192xf32>) permutation = [1, 0] - - // CHECK: %[[EMPTY:.+]] = tensor.empty() : tensor<1x8192xf32> - %1 = tensor.empty() : tensor<1x8192xf32> - - // CHECK: %[[FILLED:.+]] = linalg.generic - // CHECK-SAME: {indexing_maps = [#map2, #map1], iterator_types = ["parallel", "parallel"]} - // CHECK-SAME: ins(%[[CST]] : f32) outs(%[[EMPTY]] : tensor<1x8192xf32>) - // CHECK: ^bb0(%[[IN:.+]]: f32, %[[OUT:.+]]: f32): - // CHECK: linalg.yield %[[IN]] : f32 - // CHECK: } -> tensor<1x8192xf32> - %2 = linalg.fill ins(%cst : f32) outs(%1 : tensor<1x8192xf32>) -> tensor<1x8192xf32> - - // CHECK: %[[MMRESULT:.+]] = linalg.generic - // CHECK-SAME: {indexing_maps = [#map3, #map4, #map5], iterator_types = ["parallel", "parallel", "reduction"]} - // CHECK-SAME: ins(%[[COLLAPSE]], %[[TRANSPOSED]] : tensor<1x16384xf32>, tensor<16384x8192xf32>) outs(%[[FILLED]] : tensor<1x8192xf32>) - // CHECK: ^bb0(%[[IN0:.+]]: f32, %[[IN1:.+]]: f32, %[[OUT:.+]]: f32): - // CHECK: %[[MUL:.+]] = arith.mulf %[[IN0]], %[[IN1]] : f32 - // CHECK: %[[ADD:.+]] = arith.addf %[[OUT]], %[[MUL]] : f32 - // CHECK: linalg.yield %[[ADD]] : f32 - // CHECK: } -> tensor<1x8192xf32> - %3 = linalg.matmul ins(%collapsed, %transposed : tensor<1x16384xf32>, tensor<16384x8192xf32>) outs(%2 : tensor<1x8192xf32>) -> tensor<1x8192xf32> - - // CHECK: %[[BIAS:.+]] = linalg.generic - // CHECK-SAME: {indexing_maps = [#map6, #map7, #map1], iterator_types = ["parallel", "parallel"]} - // CHECK-SAME: ins(%arg1, %[[MMRESULT]] : tensor<8192xf32>, tensor<1x8192xf32>) outs(%[[EMPTY]] : tensor<1x8192xf32>) { - // CHECK: ^bb0(%[[IN0:.+]]: f32, %[[IN1:.+]]: f32, %[[OUT:.+]]: f32): - // CHECK: %[[ADD:.+]] = arith.addf %[[IN0]], %[[IN1]] : f32 - // CHECK: linalg.yield %[[ADD]] : f32 - // CHECK: } -> tensor<1x8192xf32> - %4 = linalg.generic {indexing_maps = [#map, #map1, #map2], iterator_types = ["parallel", "parallel"]} ins(%arg1, %3 : tensor<8192xf32>, tensor<1x8192xf32>) outs(%1 : tensor<1x8192xf32>) { - ^bb0(%in: f32, %in_0: f32, %out: f32): - %5 = arith.addf %in, %in_0 : f32 - linalg.yield %5 : f32 - } -> tensor<1x8192xf32> - - // CHECK: return %[[BIAS]] : tensor<1x8192xf32> - return %4 : tensor<1x8192xf32> -} -// ----- diff --git a/test/mlir/test/gc/Dialect/Linlagx/linalgx-named-ops.mlir b/test/mlir/test/gc/Dialect/Linlagx/linalgx-named-ops.mlir deleted file mode 100644 index fc552aa0a..000000000 --- a/test/mlir/test/gc/Dialect/Linlagx/linalgx-named-ops.mlir +++ /dev/null @@ -1,8 +0,0 @@ -// RUN: gc-opt -split-input-file -verify-diagnostics %s | FileCheck %s - -// CHECK-LABEL: @sigmoid -func.func @sigmoid(%arg0: tensor<4x256x64xbf16>, %arg1: tensor<4x256x64xbf16>) -> tensor<4x256x64xbf16> { - // CHECK: linalgx.sigmoid - %0 = linalgx.sigmoid ins(%arg0 : tensor<4x256x64xbf16>) outs(%arg1 : tensor<4x256x64xbf16>) -> tensor<4x256x64xbf16> - return %0 : tensor<4x256x64xbf16> -} diff --git a/test/mlir/test/gc/Dialect/Microkernel/early-dispatch-microkernel.mlir b/test/mlir/test/gc/Dialect/Microkernel/early-dispatch-microkernel.mlir deleted file mode 100644 index 364b88dd1..000000000 --- a/test/mlir/test/gc/Dialect/Microkernel/early-dispatch-microkernel.mlir +++ /dev/null @@ -1,62 +0,0 @@ -// RUN: gc-opt %s -early-dispatch-microkernel -split-input-file | FileCheck %s - -#map = affine_map<(d0, d1) -> (d0, d1)> -module { - func.func @simple_brgemm() { - %c0_i64 = arith.constant 0 : i64 - %c16_i64 = arith.constant 16 : i64 - %cst = arith.constant 0.000000e+00 : f32 - %alloc = memref.alloc() {alignment = 64 : i64} : memref<4x16x32x32xf32> - %alloc_0 = memref.alloc() {alignment = 64 : i64} : memref<8x16x32x32xf32> - %alloc_1 = memref.alloc() {alignment = 64 : i64} : memref<4x8x32x32xf32> - %alloc_2 = memref.alloc() {alignment = 64 : i64} : memref<4x8x32x32xf32> - scf.forall (%arg0, %arg1) in (4, 8) { - %alloc_3 = memref.alloc() {alignment = 64 : i64} : memref<32x32xf32> - linalg.fill ins(%cst : f32) outs(%alloc_3 : memref<32x32xf32>) - %subview = memref.subview %alloc[%arg0, 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : memref<4x16x32x32xf32> to memref<16x32x32xf32, strided<[1024, 32, 1], offset: ?>> - %subview_4 = memref.subview %alloc_0[%arg1, 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : memref<8x16x32x32xf32> to memref<16x32x32xf32, strided<[1024, 32, 1], offset: ?>> - %0 = microkernel.brgemm.dispatch [32, 32, 32, 32, 32, 32, 1024, 1024] flags(stride) data_type(f32, f32) - microkernel.brgemm.prologue(%0) : (i64) -> () - microkernel.brgemm.execute(%0, %subview, %subview_4, %alloc_3, %c16_i64, %c0_i64) : (i64, memref<16x32x32xf32, strided<[1024, 32, 1], offset: ?>>, memref<16x32x32xf32, strided<[1024, 32, 1], offset: ?>>, memref<32x32xf32>, i64, i64) -> () - microkernel.brgemm.epilogue(%0) : (i64) -> () - %subview_5 = memref.subview %alloc_1[%arg0, %arg1, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : memref<4x8x32x32xf32> to memref<32x32xf32, strided<[32, 1], offset: ?>> - linalg.generic {indexing_maps = [#map, #map, #map], iterator_types = ["parallel", "parallel"]} ins(%alloc_3, %subview_5 : memref<32x32xf32>, memref<32x32xf32, strided<[32, 1], offset: ?>>) outs(%alloc_3 : memref<32x32xf32>) { - ^bb0(%in: f32, %in_7: f32, %out: f32): - %1 = arith.addf %in, %in_7 : f32 - linalg.yield %1 : f32 - } - %subview_6 = memref.subview %alloc_2[%arg0, %arg1, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : memref<4x8x32x32xf32> to memref<32x32xf32, strided<[32, 1], offset: ?>> - linalg.generic {indexing_maps = [#map, #map], iterator_types = ["parallel", "parallel"]} ins(%alloc_3 : memref<32x32xf32>) outs(%subview_6 : memref<32x32xf32, strided<[32, 1], offset: ?>>) { - ^bb0(%in: f32, %out: f32): - %1 = arith.maximumf %in, %cst : f32 - linalg.yield %1 : f32 - } - memref.dealloc %alloc_3 : memref<32x32xf32> - } - return - } -} - -// CHECK: llvm.mlir.global_ctors {ctors = [@[[G_CTOR_NAME:.+]]], priorities = [[[G_CTOR_PRIOR:.+]] : i32]} -// CHECK: llvm.mlir.global internal @[[G_NAME:.+]]() {addr_space = 0 : i32} : i64 - -// CHECK: llvm.func @[[G_CTOR_NAME]]() -> i64 { -// CHECK-DAG: %[[G_PTR:.+]] = llvm.mlir.addressof @[[G_NAME]] : !llvm.ptr -// CHECK-DAG: %[[KERNEL:.+]] = microkernel.brgemm.dispatch [32, 32, 32, 32, 32, 32, 1024, 1024] flags(stride) data_type(f32, f32) -// CHECK: llvm.store %[[KERNEL]], %[[G_PTR]] : i64, !llvm.ptr - -// CHECK-LABEL: simple_brgemm -// CHECK: %[[CST16:.+]] = arith.constant 16 : i64 -// CHECK: %[[CST0:.+]] = arith.constant 0 : i64 - -// CHECK: %[[G_PTR_2:.+]] = llvm.mlir.addressof @[[G_NAME]] : !llvm.ptr -// CHECK-NEXT: %[[KERNEL2:.+]] = llvm.load %[[G_PTR_2]] : !llvm.ptr -> i64 - -// CHECK: %[[memrefC:.+]] = memref.alloc() {alignment = 64 : i64} : memref<32x32xf32> - -// CHECK: %[[subviewA:.+]] = memref.subview %[[memrefA:.+]][%arg0, 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : memref<4x16x32x32xf32> to memref<16x32x32xf32, strided<[1024, 32, 1], offset: ?>> -// CHECK: %[[subviewB:.+]] = memref.subview %[[memrefB:.+]][%arg1, 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : memref<8x16x32x32xf32> to memref<16x32x32xf32, strided<[1024, 32, 1], offset: ?>> - -// CHECK: microkernel.brgemm.execute(%[[KERNEL2]], %[[subviewA]], %[[subviewB]], %[[memrefC]], %[[CST16]], %[[CST0]]) : (i64, memref<16x32x32xf32, strided<[1024, 32, 1], offset: ?>>, memref<16x32x32xf32, strided<[1024, 32, 1], offset: ?>>, memref<32x32xf32>, i64, i64) -> () - -// ----- diff --git a/test/mlir/test/gc/Dialect/Microkernel/expand-microkernel.mlir b/test/mlir/test/gc/Dialect/Microkernel/expand-microkernel.mlir deleted file mode 100644 index db6ebd286..000000000 --- a/test/mlir/test/gc/Dialect/Microkernel/expand-microkernel.mlir +++ /dev/null @@ -1,237 +0,0 @@ -// RUN: gc-opt %s -expand-microkernel -split-input-file | FileCheck %s - -#map1 = affine_map<(d0, d1) -> (d0, d1)> -func.func @basic_expand_microkernel_non_init() { - %cst = arith.constant 0.000000e+00 : f32 - %alloc_1 = memref.alloc() {alignment = 64 : i64} : memref<4x16x32x32xf32> - %alloc_4 = memref.alloc() {alignment = 64 : i64} : memref<8x16x32x32xf32> - %alloc_5 = memref.alloc() {alignment = 64 : i64} : memref<4x8x32x32xf32> - %alloc_6 = memref.alloc() {alignment = 64 : i64} : memref<4x8x32x32xf32> - scf.forall (%arg7, %arg8) in (4, 8) { - %alloc_10 = memref.alloc() {alignment = 64 : i64} : memref<32x32xf32> - %subview = memref.subview %alloc_1[%arg7, 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : memref<4x16x32x32xf32> to memref<16x32x32xf32, strided<[1024, 32, 1], offset: ?>> - %subview_11 = memref.subview %alloc_4[%arg8, 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : memref<8x16x32x32xf32> to memref<16x32x32xf32, strided<[1024, 32, 1], offset: ?>> - microkernel.brgemm ins(%subview, %subview_11 : memref<16x32x32xf32, strided<[1024, 32, 1], offset: ?>>, memref<16x32x32xf32, strided<[1024, 32, 1], offset: ?>>) outs(%alloc_10 : memref<32x32xf32>) batch_dims(0, 0) leading_dims(1, 1) flags() - %subview_12 = memref.subview %alloc_5[%arg7, %arg8, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : memref<4x8x32x32xf32> to memref<32x32xf32, strided<[32, 1], offset: ?>> - linalg.generic {indexing_maps = [#map1, #map1, #map1], iterator_types = ["parallel", "parallel"]} ins(%alloc_10, %subview_12 : memref<32x32xf32>, memref<32x32xf32, strided<[32, 1], offset: ?>>) outs(%alloc_10 : memref<32x32xf32>) { - ^bb0(%in: f32, %in_14: f32, %out: f32): - %0 = arith.addf %in, %in_14 : f32 - linalg.yield %0 : f32 - } - %subview_13 = memref.subview %alloc_6[%arg7, %arg8, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : memref<4x8x32x32xf32> to memref<32x32xf32, strided<[32, 1], offset: ?>> - linalg.generic {indexing_maps = [#map1, #map1], iterator_types = ["parallel", "parallel"]} ins(%alloc_10 : memref<32x32xf32>) outs(%subview_13 : memref<32x32xf32, strided<[32, 1], offset: ?>>) { - ^bb0(%in: f32, %out: f32): - %0 = arith.maximumf %in, %cst : f32 - linalg.yield %0 : f32 - } - memref.dealloc %alloc_10 : memref<32x32xf32> - } - return -} - -// CHECK-LABEL: basic_expand_microkernel_non_init -// CHECK: %[[CST0:.+]] = arith.constant 0 : i64 -// CHECK: %[[CST16:.+]] = arith.constant 16 : i64 -// CHECK: %[[C:.+]] = memref.alloc() {alignment = 64 : i64} : memref<32x32xf32> -// CHECK: %[[A:.+]] = memref.subview %[[TMP1:.+]][%arg0, 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : memref<4x16x32x32xf32> to memref<16x32x32xf32, strided<[1024, 32, 1], offset: ?>> -// CHECK: %[[B:.+]] = memref.subview %[[TMP2:.+]][%arg1, 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : memref<8x16x32x32xf32> to memref<16x32x32xf32, strided<[1024, 32, 1], offset: ?>> -// CHECK: %[[DIS:.+]] = microkernel.brgemm.dispatch [32, 32, 32, 32, 32, 32, 1024, 1024] flags(stride) data_type(f32, f32) -// CHECK-NEXT: microkernel.brgemm.prologue(%[[DIS]]) : (i64) -> () -// CHECK-NEXT: microkernel.brgemm.execute(%[[DIS]], %[[A]], %[[B]], %[[C]], %[[CST16]], %[[CST0]]) : (i64, memref<16x32x32xf32, strided<[1024, 32, 1], offset: ?>>, memref<16x32x32xf32, strided<[1024, 32, 1], offset: ?>>, memref<32x32xf32>, i64, i64) -> () -// CHECK-NEXT: microkernel.brgemm.epilogue(%[[DIS]]) : (i64) -> () - -// ----- - -#map1 = affine_map<(d0, d1) -> (d0, d1)> -func.func @basic_expand_microkernel_init() { - %cst = arith.constant 0.000000e+00 : f32 - %alloc_1 = memref.alloc() {alignment = 64 : i64} : memref<4x16x32x32xf32> - %alloc_4 = memref.alloc() {alignment = 64 : i64} : memref<8x16x32x32xf32> - %alloc_5 = memref.alloc() {alignment = 64 : i64} : memref<4x8x32x32xf32> - %alloc_6 = memref.alloc() {alignment = 64 : i64} : memref<4x8x32x32xf32> - scf.forall (%arg7, %arg8) in (4, 8) { - %alloc_10 = memref.alloc() {alignment = 64 : i64} : memref<32x32xf32> - %subview = memref.subview %alloc_1[%arg7, 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : memref<4x16x32x32xf32> to memref<16x32x32xf32, strided<[1024, 32, 1], offset: ?>> - %subview_11 = memref.subview %alloc_4[%arg8, 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : memref<8x16x32x32xf32> to memref<16x32x32xf32, strided<[1024, 32, 1], offset: ?>> - microkernel.brgemm ins(%subview, %subview_11 : memref<16x32x32xf32, strided<[1024, 32, 1], offset: ?>>, memref<16x32x32xf32, strided<[1024, 32, 1], offset: ?>>) outs(%alloc_10 : memref<32x32xf32>) batch_dims(0, 0) leading_dims(1, 1) flags(beta_0) - %subview_12 = memref.subview %alloc_5[%arg7, %arg8, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : memref<4x8x32x32xf32> to memref<32x32xf32, strided<[32, 1], offset: ?>> - linalg.generic {indexing_maps = [#map1, #map1, #map1], iterator_types = ["parallel", "parallel"]} ins(%alloc_10, %subview_12 : memref<32x32xf32>, memref<32x32xf32, strided<[32, 1], offset: ?>>) outs(%alloc_10 : memref<32x32xf32>) { - ^bb0(%in: f32, %in_14: f32, %out: f32): - %0 = arith.addf %in, %in_14 : f32 - linalg.yield %0 : f32 - } - %subview_13 = memref.subview %alloc_6[%arg7, %arg8, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : memref<4x8x32x32xf32> to memref<32x32xf32, strided<[32, 1], offset: ?>> - linalg.generic {indexing_maps = [#map1, #map1], iterator_types = ["parallel", "parallel"]} ins(%alloc_10 : memref<32x32xf32>) outs(%subview_13 : memref<32x32xf32, strided<[32, 1], offset: ?>>) { - ^bb0(%in: f32, %out: f32): - %0 = arith.maximumf %in, %cst : f32 - linalg.yield %0 : f32 - } - memref.dealloc %alloc_10 : memref<32x32xf32> - } - return -} - -// CHECK-LABEL: basic_expand_microkernel_init -// CHECK: %[[CST0:.+]] = arith.constant 0 : i64 -// CHECK: %[[CST16:.+]] = arith.constant 16 : i64 -// CHECK: %[[C:.+]] = memref.alloc() {alignment = 64 : i64} : memref<32x32xf32> -// CHECK: %[[A:.+]] = memref.subview %[[TMP1:.+]][%arg0, 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : memref<4x16x32x32xf32> to memref<16x32x32xf32, strided<[1024, 32, 1], offset: ?>> -// CHECK: %[[B:.+]] = memref.subview %[[TMP2:.+]][%arg1, 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : memref<8x16x32x32xf32> to memref<16x32x32xf32, strided<[1024, 32, 1], offset: ?>> -// CHECK: %[[DIS:.+]] = microkernel.brgemm.dispatch [32, 32, 32, 32, 32, 32, 1024, 1024] flags(beta_0, stride) data_type(f32, f32) -// CHECK-NEXT: microkernel.brgemm.prologue(%[[DIS]]) : (i64) -> () -// CHECK-NEXT: microkernel.brgemm.execute(%[[DIS]], %[[A]], %[[B]], %[[C]], %[[CST16]], %[[CST0]]) : (i64, memref<16x32x32xf32, strided<[1024, 32, 1], offset: ?>>, memref<16x32x32xf32, strided<[1024, 32, 1], offset: ?>>, memref<32x32xf32>, i64, i64) -> () -// CHECK-NEXT: microkernel.brgemm.epilogue(%[[DIS]]) : (i64) -> () - -// ----- - -#map1 = affine_map<(d0, d1) -> (d0, d1)> -func.func @transpose_expand_microkernel_init() { - %cst = arith.constant 0.000000e+00 : f32 - %alloc_1 = memref.alloc() {alignment = 64 : i64} : memref<4x32x16x32xf32> - %alloc_4 = memref.alloc() {alignment = 64 : i64} : memref<8x32x16x32xf32> - %alloc_5 = memref.alloc() {alignment = 64 : i64} : memref<4x8x32x32xf32> - %alloc_6 = memref.alloc() {alignment = 64 : i64} : memref<4x8x32x32xf32> - scf.forall (%arg7, %arg8) in (4, 8) { - %alloc_10 = memref.alloc() {alignment = 64 : i64} : memref<32x32xf32> - %subview = memref.subview %alloc_1[%arg7, 0, 0, 0] [1, 32, 16, 32] [1, 1, 1, 1] : memref<4x32x16x32xf32> to memref<32x16x32xf32, strided<[512, 32, 1], offset: ?>> - %subview_11 = memref.subview %alloc_4[%arg8, 0, 0, 0] [1, 32, 16, 32] [1, 1, 1, 1] : memref<8x32x16x32xf32> to memref<32x16x32xf32, strided<[512, 32, 1], offset: ?>> - microkernel.brgemm ins(%subview, %subview_11 : memref<32x16x32xf32, strided<[512, 32, 1], offset: ?>>, memref<32x16x32xf32, strided<[512, 32, 1], offset: ?>>) outs(%alloc_10 : memref<32x32xf32>) batch_dims(1, 1) leading_dims(0, 0) flags(beta_0) - %subview_12 = memref.subview %alloc_5[%arg7, %arg8, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : memref<4x8x32x32xf32> to memref<32x32xf32, strided<[32, 1], offset: ?>> - linalg.generic {indexing_maps = [#map1, #map1, #map1], iterator_types = ["parallel", "parallel"]} ins(%alloc_10, %subview_12 : memref<32x32xf32>, memref<32x32xf32, strided<[32, 1], offset: ?>>) outs(%alloc_10 : memref<32x32xf32>) { - ^bb0(%in: f32, %in_14: f32, %out: f32): - %0 = arith.addf %in, %in_14 : f32 - linalg.yield %0 : f32 - } - %subview_13 = memref.subview %alloc_6[%arg7, %arg8, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : memref<4x8x32x32xf32> to memref<32x32xf32, strided<[32, 1], offset: ?>> - linalg.generic {indexing_maps = [#map1, #map1], iterator_types = ["parallel", "parallel"]} ins(%alloc_10 : memref<32x32xf32>) outs(%subview_13 : memref<32x32xf32, strided<[32, 1], offset: ?>>) { - ^bb0(%in: f32, %out: f32): - %0 = arith.maximumf %in, %cst : f32 - linalg.yield %0 : f32 - } - memref.dealloc %alloc_10 : memref<32x32xf32> - } - return -} - -// CHECK-LABEL: transpose_expand_microkernel_init -// CHECK: %[[CST0:.+]] = arith.constant 0 : i64 -// CHECK: %[[CST16:.+]] = arith.constant 16 : i64 -// CHECK: %[[C:.+]] = memref.alloc() {alignment = 64 : i64} : memref<32x32xf32> -// CHECK: %[[A:.+]] = memref.subview %[[TMP1:.+]][%arg0, 0, 0, 0] [1, 32, 16, 32] [1, 1, 1, 1] : memref<4x32x16x32xf32> to memref<32x16x32xf32, strided<[512, 32, 1], offset: ?>> -// CHECK: %[[B:.+]] = memref.subview %[[TMP2:.+]][%arg1, 0, 0, 0] [1, 32, 16, 32] [1, 1, 1, 1] : memref<8x32x16x32xf32> to memref<32x16x32xf32, strided<[512, 32, 1], offset: ?>> -// CHECK: %[[DIS:.+]] = microkernel.brgemm.dispatch [32, 32, 32, 512, 512, 32, 32, 32] flags(beta_0, stride) data_type(f32, f32) -// CHECK-NEXT: microkernel.brgemm.prologue(%[[DIS]]) : (i64) -> () -// CHECK-NEXT: microkernel.brgemm.execute(%[[DIS]], %[[A]], %[[B]], %[[C]], %[[CST16]], %[[CST0]]) : (i64, memref<32x16x32xf32, strided<[512, 32, 1], offset: ?>>, memref<32x16x32xf32, strided<[512, 32, 1], offset: ?>>, memref<32x32xf32>, i64, i64) -> () -// CHECK-NEXT: microkernel.brgemm.epilogue(%[[DIS]]) : (i64) -> () - -// ----- - -#map1 = affine_map<(d0, d1) -> (d0, d1)> -func.func @transpose_expand_microkernel_init_vnni() { - %cst = arith.constant 0.000000e+00 : f32 - %alloc_1 = memref.alloc() {alignment = 64 : i64} : memref<4x32x16x32xbf16> - %alloc_4 = memref.alloc() {alignment = 64 : i64} : memref<8x16x16x32x2xbf16> - %alloc_5 = memref.alloc() {alignment = 64 : i64} : memref<4x8x32x32xf32> - %alloc_6 = memref.alloc() {alignment = 64 : i64} : memref<4x8x32x32xf32> - scf.forall (%arg7, %arg8) in (4, 8) { - %alloc_10 = memref.alloc() {alignment = 64 : i64} : memref<32x32xf32> - %subview = memref.subview %alloc_1[%arg7, 0, 0, 0] [1, 32, 16, 32] [1, 1, 1, 1] : memref<4x32x16x32xbf16> to memref<32x16x32xbf16, strided<[512, 32, 1], offset: ?>> - %subview_11 = memref.subview %alloc_4[%arg8, 0, 0, 0, 0] [1, 16, 16, 32, 2] [1, 1, 1, 1, 1] : memref<8x16x16x32x2xbf16> to memref<16x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>> - microkernel.brgemm ins(%subview, %subview_11 : memref<32x16x32xbf16, strided<[512, 32, 1], offset: ?>>, memref<16x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>>) outs(%alloc_10 : memref<32x32xf32>) batch_dims(1, 1) leading_dims(0, 0) flags(beta_0) - %subview_12 = memref.subview %alloc_5[%arg7, %arg8, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : memref<4x8x32x32xf32> to memref<32x32xf32, strided<[32, 1], offset: ?>> - linalg.generic {indexing_maps = [#map1, #map1, #map1], iterator_types = ["parallel", "parallel"]} ins(%alloc_10, %subview_12 : memref<32x32xf32>, memref<32x32xf32, strided<[32, 1], offset: ?>>) outs(%alloc_10 : memref<32x32xf32>) { - ^bb0(%in: f32, %in_14: f32, %out: f32): - %0 = arith.addf %in, %in_14 : f32 - linalg.yield %0 : f32 - } - %subview_13 = memref.subview %alloc_6[%arg7, %arg8, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : memref<4x8x32x32xf32> to memref<32x32xf32, strided<[32, 1], offset: ?>> - linalg.generic {indexing_maps = [#map1, #map1], iterator_types = ["parallel", "parallel"]} ins(%alloc_10 : memref<32x32xf32>) outs(%subview_13 : memref<32x32xf32, strided<[32, 1], offset: ?>>) { - ^bb0(%in: f32, %out: f32): - %0 = arith.maximumf %in, %cst : f32 - linalg.yield %0 : f32 - } - memref.dealloc %alloc_10 : memref<32x32xf32> - } - return -} - -// CHECK-LABEL: transpose_expand_microkernel_init_vnni -// CHECK: %[[CST0:.+]] = arith.constant 0 : i64 -// CHECK: %[[CST16:.+]] = arith.constant 16 : i64 -// CHECK: %[[C:.+]] = memref.alloc() {alignment = 64 : i64} : memref<32x32xf32> -// CHECK: %[[A:.+]] = memref.subview %[[TMP1:.+]][%arg0, 0, 0, 0] [1, 32, 16, 32] [1, 1, 1, 1] : memref<4x32x16x32xbf16> to memref<32x16x32xbf16, strided<[512, 32, 1], offset: ?>> -// CHECK: %[[B:.+]] = memref.subview %[[TMP2:.+]][%arg1, 0, 0, 0, 0] [1, 16, 16, 32, 2] [1, 1, 1, 1, 1] : memref<8x16x16x32x2xbf16> to memref<16x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>> -// CHECK: %[[DIS:.+]] = microkernel.brgemm.dispatch [32, 32, 32, 512, 512, 32, 32, 64] flags(beta_0, stride) data_type(bf16, bf16) -// CHECK-NEXT: microkernel.brgemm.prologue(%[[DIS]]) : (i64) -> () -// CHECK-NEXT: microkernel.brgemm.execute(%[[DIS]], %[[A]], %[[B]], %[[C]], %[[CST16]], %[[CST0]]) : (i64, memref<32x16x32xbf16, strided<[512, 32, 1], offset: ?>>, memref<16x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>>, memref<32x32xf32>, i64, i64) -> () -// CHECK-NEXT: microkernel.brgemm.epilogue(%[[DIS]]) : (i64) -> () - -// ----- - -#map = affine_map<(d0) -> (-d0 + 344, 11)> -#map1 = affine_map<(d0)[s0] -> (-d0 + s0, 8)> -#map2 = affine_map<()[s0, s1, s2] -> (s0 + s1 + s2)> -#map3 = affine_map<(d0, d1) -> (d0, d1)> -module { - func.func @expand_microkernel_with_dynamic(%arg0: memref<1x128x1x32xbf16>, %arg1: memref<344x128x16x32x2xbf16>, %arg2: memref<1x344x1x32xbf16>) attributes {llvm.emit_c_interface} { - %c1 = arith.constant 1 : index - %c64 = arith.constant 64 : index - %c128 = arith.constant 128 : index - %c8 = arith.constant 8 : index - %c0 = arith.constant 0 : index - scf.forall (%arg3) = (0) to (344) step (11) { - %0 = affine.min #map(%arg3) - %subview = memref.subview %arg2[0, %arg3, 0, 0] [1, %0, 1, 32] [1, 1, 1, 1] : memref<1x344x1x32xbf16> to memref<1x?x1x32xbf16, strided<[11008, 32, 32, 1], offset: ?>> - scf.for %arg4 = %c0 to %0 step %c8 { - %1 = affine.min #map1(%arg4)[%0] - %subview_0 = memref.subview %subview[0, %arg4, 0, 0] [1, %1, 1, 32] [1, 1, 1, 1] : memref<1x?x1x32xbf16, strided<[11008, 32, 32, 1], offset: ?>> to memref<1x?x1x32xbf16, strided<[11008, 32, 32, 1], offset: ?>> - %alloc = memref.alloc(%1) {alignment = 64 : i64} : memref<1x?x1x32xf32> - scf.for %arg5 = %c0 to %c128 step %c64 { - %subview_1 = memref.subview %alloc[0, 0, 0, 0] [1, %1, 1, 32] [1, 1, 1, 1] : memref<1x?x1x32xf32> to memref<1x?x1x32xf32, strided<[?, 32, 32, 1]>> - %subview_2 = memref.subview %arg0[0, %arg5, 0, 0] [1, 64, 1, 32] [1, 1, 1, 1] : memref<1x128x1x32xbf16> to memref<64x1x32xbf16, strided<[32, 32, 1], offset: ?>> - %2 = arith.cmpi eq, %arg5, %c0 : index - %3 = arith.addi %arg5, %c64 : index - %4 = arith.cmpi sge, %3, %c128 : index - scf.for %arg6 = %c0 to %1 step %c1 { - %5 = affine.apply #map2()[%arg3, %arg6, %arg4] - %subview_3 = memref.subview %arg1[%5, %arg5, 0, 0, 0] [1, 64, 16, 32, 2] [1, 1, 1, 1, 1] : memref<344x128x16x32x2xbf16> to memref<64x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>> - %subview_4 = memref.subview %subview_1[0, %arg6, 0, 0] [1, 1, 1, 32] [1, 1, 1, 1] : memref<1x?x1x32xf32, strided<[?, 32, 32, 1]>> to memref<1x32xf32, strided<[?, 1], offset: ?>> - %subview_5 = memref.subview %subview_0[0, %arg6, 0, 0] [1, 1, 1, 32] [1, 1, 1, 1] : memref<1x?x1x32xbf16, strided<[11008, 32, 32, 1], offset: ?>> to memref<1x32xbf16, strided<[11008, 1], offset: ?>> - scf.if %2 { - microkernel.brgemm ins(%subview_2, %subview_3 : memref<64x1x32xbf16, strided<[32, 32, 1], offset: ?>>, memref<64x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>>) outs(%subview_4 : memref<1x32xf32, strided<[?, 1], offset: ?>>) batch_dims(0, 0) leading_dims(1, 1) flags(beta_0) - } else { - microkernel.brgemm ins(%subview_2, %subview_3 : memref<64x1x32xbf16, strided<[32, 32, 1], offset: ?>>, memref<64x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>>) outs(%subview_4 : memref<1x32xf32, strided<[?, 1], offset: ?>>) batch_dims(0, 0) leading_dims(1, 1) flags() - } - scf.if %4 { - linalg.generic {indexing_maps = [#map3, #map3], iterator_types = ["parallel", "parallel"]} ins(%subview_4 : memref<1x32xf32, strided<[?, 1], offset: ?>>) outs(%subview_5 : memref<1x32xbf16, strided<[11008, 1], offset: ?>>) { - ^bb0(%in: f32, %out: bf16): - %6 = arith.truncf %in : f32 to bf16 - linalg.yield %6 : bf16 - } - } - } - } - memref.dealloc %alloc : memref<1x?x1x32xf32> - } - } - return - } -} - -// CHECK-LABEL: expand_microkernel_with_dynamic -// CHECK: scf.forall (%[[ARG:.+]]) = (0) to (344) step (11) -// CHECK: scf.for %[[ARG2:.+]] = %[[CST0:.+]] to %[[AFF:.+]] step %[[CST8:.+]] -// CHECK: scf.for %[[ARG3:.+]] = %[[CST0]] to %[[CST128:.+]] step %[[CST64:.+]] -// CHECK: scf.for %[[ARG4:.+]] = %[[CST0]] to %[[AFF1:.+]] step %[[CST1:.+]] -// CHECK: scf.if -// CHECK: %[[DIS:.+]] = microkernel.brgemm.dispatch [1, 32, 32, 32, 32, 9223372036854775807, 32, 1024] flags(beta_0, stride) data_type(bf16, bf16) -// CHECK-NEXT: microkernel.brgemm.prologue(%[[DIS]]) : (i64) -> () -// CHECK-NEXT: microkernel.brgemm.execute(%[[DIS]] -// CHECK-NEXT: microkernel.brgemm.epilogue(%[[DIS]]) : (i64) -> () -// CHECK: else -// CHECK: %[[DIS2:.+]] = microkernel.brgemm.dispatch [1, 32, 32, 32, 32, 9223372036854775807, 32, 1024] flags(stride) data_type(bf16, bf16) -// CHECK-NEXT: microkernel.brgemm.prologue(%[[DIS2]]) : (i64) -> () -// CHECK-NEXT: microkernel.brgemm.execute(%[[DIS]] -// CHECK-NEXT: microkernel.brgemm.epilogue(%[[DIS2]]) : (i64) -> () - -// ----- diff --git a/test/mlir/test/gc/Dialect/Microkernel/linalg-to-microkernel.mlir b/test/mlir/test/gc/Dialect/Microkernel/linalg-to-microkernel.mlir deleted file mode 100644 index de785c590..000000000 --- a/test/mlir/test/gc/Dialect/Microkernel/linalg-to-microkernel.mlir +++ /dev/null @@ -1,258 +0,0 @@ -// RUN: gc-opt %s -convert-linalg-to-microkernel -split-input-file | FileCheck %s - -#map1 = affine_map<(d0, d1) -> (d0, d1)> -func.func @basic_linalg_to_microkernel(%arg0: tensor<4x8x32x32xf32>) -> tensor<4x8x32x32xf32> { - %alloc_1 = tensor.empty() : tensor<4x16x32x32xf32> - %alloc_4 = tensor.empty() : tensor<8x16x32x32xf32> - %ret = scf.forall (%arg7, %arg8) in (4, 8) shared_outs(%argp = %arg0) -> (tensor<4x8x32x32xf32>) { - %alloc_10 = tensor.extract_slice %argp[%arg7, %arg8, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : tensor<4x8x32x32xf32> to tensor<32x32xf32> - %subview = tensor.extract_slice %alloc_1[%arg7, 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : tensor<4x16x32x32xf32> to tensor<16x32x32xf32> - %subview_11 = tensor.extract_slice %alloc_4[%arg8, 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : tensor<8x16x32x32xf32> to tensor<16x32x32xf32> - %res = linalg.batch_reduce_matmul ins(%subview, %subview_11 : tensor<16x32x32xf32>, tensor<16x32x32xf32>) outs(%alloc_10 : tensor<32x32xf32>) -> tensor<32x32xf32> - scf.forall.in_parallel { - tensor.parallel_insert_slice %res into %argp[%arg7, %arg8, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : tensor<32x32xf32> into tensor<4x8x32x32xf32> - } - } - return %ret : tensor<4x8x32x32xf32> -} - -// CHECK-LABEL: basic_linalg_to_microkernel -// CHECK: scf.forall -// CHECK: %[[C:.+]] = tensor.extract_slice %[[Csrc:.+]][%[[arg1:.+]], %[[arg2:.+]], 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : tensor<4x8x32x32xf32> to tensor<32x32xf32> -// CHECK: %[[A:.+]] = tensor.extract_slice %[[Asrc:.+]][%[[arg1]], 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : tensor<4x16x32x32xf32> to tensor<16x32x32xf32> -// CHECK: %[[B:.+]] = tensor.extract_slice %[[Bsrc:.+]][%[[arg2]], 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : tensor<8x16x32x32xf32> to tensor<16x32x32xf32> -// CHECK: %[[RES:.+]] = microkernel.brgemm ins(%[[A]], %[[B]] : [[TYPE:.+]]) outs(%[[C]] : [[TYPEC:.+]]) batch_dims(0, 0) leading_dims(1, 1) flags() -> tensor<32x32xf32> -// CHECK-NEXT: scf.forall.in_parallel - -// ----- - -#map1 = affine_map<(d0, d1) -> (d0, d1)> -func.func @vnni_linalg_to_microkernel(%arg0: tensor<4x8x32x32xf32>) -> tensor<4x8x32x32xf32> { - %alloc_1 = tensor.empty() : tensor<4x16x32x32xbf16> - %alloc_4 = tensor.empty() : tensor<8x16x16x32x2xbf16> - %ret = scf.forall (%arg7, %arg8) in (4, 8) shared_outs(%argp = %arg0) -> (tensor<4x8x32x32xf32>) { - %alloc_10 = tensor.extract_slice %argp[%arg7, %arg8, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : tensor<4x8x32x32xf32> to tensor<32x32xf32> - %subview = tensor.extract_slice %alloc_1[%arg7, 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : tensor<4x16x32x32xbf16> to tensor<16x32x32xbf16> - %subview_11 = tensor.extract_slice %alloc_4[%arg8, 0, 0, 0, 0] [1, 16, 16, 32, 2] [1, 1, 1, 1, 1] : tensor<8x16x16x32x2xbf16> to tensor<16x16x32x2xbf16> - %res = linalg.generic { - indexing_maps = [affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d3 * 2 + d4)>, - affine_map<(d0, d1, d2, d3, d4) -> (d0, d3, d2, d4)>, - affine_map<(d0, d1, d2, d3, d4) -> (d1, d2)>], - iterator_types = ["reduction", "parallel", "parallel", "reduction", "reduction"]} - ins(%subview, %subview_11 : tensor<16x32x32xbf16>, tensor<16x16x32x2xbf16>) - outs(%alloc_10 : tensor<32x32xf32>) { - ^bb0(%in: bf16, %in_0: bf16, %out: f32): - %1 = arith.extf %in : bf16 to f32 - %2 = arith.extf %in_0 : bf16 to f32 - %3 = arith.mulf %1, %2 : f32 - %4 = arith.addf %out, %3 : f32 - linalg.yield %4 : f32 - } -> tensor<32x32xf32> - scf.forall.in_parallel { - tensor.parallel_insert_slice %res into %argp[%arg7, %arg8, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : tensor<32x32xf32> into tensor<4x8x32x32xf32> - } - } - return %ret : tensor<4x8x32x32xf32> -} - -// CHECK-LABEL: vnni_linalg_to_microkernel -// CHECK: scf.forall -// CHECK: %[[C:.+]] = tensor.extract_slice %[[Csrc:.+]][%[[arg1:.+]], %[[arg2:.+]], 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : tensor<4x8x32x32xf32> to tensor<32x32xf32> -// CHECK: %[[A:.+]] = tensor.extract_slice %[[Asrc:.+]][%[[arg1]], 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : tensor<4x16x32x32xbf16> to tensor<16x32x32xbf16> -// CHECK: %[[B:.+]] = tensor.extract_slice %[[Bsrc:.+]][%[[arg2]], 0, 0, 0, 0] [1, 16, 16, 32, 2] [1, 1, 1, 1, 1] : tensor<8x16x16x32x2xbf16> to tensor<16x16x32x2xbf16> -// CHECK: %[[RES:.+]] = microkernel.brgemm ins(%[[A]], %[[B]] : [[TYPE:.+]]) outs(%[[C]] : [[TYPEC:.+]]) batch_dims(0, 0) leading_dims(1, 1) flags() -> tensor<32x32xf32> -// CHECK-NEXT: scf.forall.in_parallel - -// ----- - -#map1 = affine_map<(d0, d1) -> (d0, d1)> -func.func @basic_linalg_to_microkernel_fusing_fill(%arg0: tensor<4x8x32x32xf32>) -> tensor<4x8x32x32xf32> { - %cst = arith.constant 0.000000e+00 : f32 - %alloc_1 = tensor.empty() : tensor<4x16x32x32xf32> - %alloc_4 = tensor.empty() : tensor<8x16x32x32xf32> - %ret = scf.forall (%arg7, %arg8) in (4, 8) shared_outs(%argp = %arg0) -> (tensor<4x8x32x32xf32>) { - %alloc_10 = tensor.extract_slice %argp[%arg7, %arg8, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : tensor<4x8x32x32xf32> to tensor<32x32xf32> - %subview = tensor.extract_slice %alloc_1[%arg7, 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : tensor<4x16x32x32xf32> to tensor<16x32x32xf32> - %subview_11 = tensor.extract_slice %alloc_4[%arg8, 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : tensor<8x16x32x32xf32> to tensor<16x32x32xf32> - %11 = linalg.fill ins(%cst : f32) outs(%alloc_10 : tensor<32x32xf32>) -> tensor<32x32xf32> - %res = linalg.batch_reduce_matmul ins(%subview, %subview_11 : tensor<16x32x32xf32>, tensor<16x32x32xf32>) outs(%11 : tensor<32x32xf32>) -> tensor<32x32xf32> - scf.forall.in_parallel { - tensor.parallel_insert_slice %res into %argp[%arg7, %arg8, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : tensor<32x32xf32> into tensor<4x8x32x32xf32> - } - } - return %ret : tensor<4x8x32x32xf32> -} - -// CHECK-LABEL: basic_linalg_to_microkernel_fusing_fill -// CHECK: scf.forall -// CHECK: %[[C:.+]] = tensor.extract_slice %[[Csrc:.+]][%[[arg1:.+]], %[[arg2:.+]], 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : tensor<4x8x32x32xf32> to tensor<32x32xf32> -// CHECK: %[[A:.+]] = tensor.extract_slice %[[Asrc:.+]][%[[arg1]], 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : tensor<4x16x32x32xf32> to tensor<16x32x32xf32> -// CHECK: %[[B:.+]] = tensor.extract_slice %[[Bsrc:.+]][%[[arg2]], 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : tensor<8x16x32x32xf32> to tensor<16x32x32xf32> -// CHECK-NOT: linalg.fill -// CHECK: %[[RES:.+]] = microkernel.brgemm ins(%[[A]], %[[B]] : [[TYPE:.+]]) outs(%[[C]] : [[TYPE2:.+]]) batch_dims(0, 0) leading_dims(1, 1) flags(beta_0) -> tensor<32x32xf32> -// CHECK-NEXT: scf.forall.in_parallel - -// ----- - -#map1 = affine_map<(d0, d1) -> (d0, d1)> -func.func @vnni_linalg_to_microkernel_fusing_fill(%arg0: tensor<4x8x32x32xf32>) -> tensor<4x8x32x32xf32> { - %cst = arith.constant 0.000000e+00 : f32 - %alloc_1 = tensor.empty() : tensor<4x16x32x32xbf16> - %alloc_4 = tensor.empty() : tensor<8x16x16x32x2xbf16> - %ret = scf.forall (%arg7, %arg8) in (4, 8) shared_outs(%argp = %arg0) -> (tensor<4x8x32x32xf32>) { - %alloc_10 = tensor.extract_slice %argp[%arg7, %arg8, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : tensor<4x8x32x32xf32> to tensor<32x32xf32> - %subview = tensor.extract_slice %alloc_1[%arg7, 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : tensor<4x16x32x32xbf16> to tensor<16x32x32xbf16> - %subview_11 = tensor.extract_slice %alloc_4[%arg8, 0, 0, 0, 0] [1, 16, 16, 32, 2] [1, 1, 1, 1, 1] : tensor<8x16x16x32x2xbf16> to tensor<16x16x32x2xbf16> - %11 = linalg.fill ins(%cst : f32) outs(%alloc_10 : tensor<32x32xf32>) -> tensor<32x32xf32> - %res = linalg.generic { - indexing_maps = [affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d3 * 2 + d4)>, - affine_map<(d0, d1, d2, d3, d4) -> (d0, d3, d2, d4)>, - affine_map<(d0, d1, d2, d3, d4) -> (d1, d2)>], - iterator_types = ["reduction", "parallel", "parallel", "reduction", "reduction"]} - ins(%subview, %subview_11 : tensor<16x32x32xbf16>, tensor<16x16x32x2xbf16>) - outs(%11 : tensor<32x32xf32>) { - ^bb0(%in: bf16, %in_0: bf16, %out: f32): - %1 = arith.extf %in : bf16 to f32 - %2 = arith.extf %in_0 : bf16 to f32 - %3 = arith.mulf %1, %2 : f32 - %4 = arith.addf %out, %3 : f32 - linalg.yield %4 : f32 - } -> tensor<32x32xf32> - scf.forall.in_parallel { - tensor.parallel_insert_slice %res into %argp[%arg7, %arg8, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : tensor<32x32xf32> into tensor<4x8x32x32xf32> - } - } - return %ret : tensor<4x8x32x32xf32> -} - -// CHECK-LABEL: vnni_linalg_to_microkernel_fusing_fill -// CHECK: scf.forall -// CHECK: %[[C:.+]] = tensor.extract_slice %[[Csrc:.+]][%[[arg1:.+]], %[[arg2:.+]], 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : tensor<4x8x32x32xf32> to tensor<32x32xf32> -// CHECK: %[[A:.+]] = tensor.extract_slice %[[Asrc:.+]][%[[arg1]], 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : tensor<4x16x32x32xbf16> to tensor<16x32x32xbf16> -// CHECK: %[[B:.+]] = tensor.extract_slice %[[Bsrc:.+]][%[[arg2]], 0, 0, 0, 0] [1, 16, 16, 32, 2] [1, 1, 1, 1, 1] : tensor<8x16x16x32x2xbf16> to tensor<16x16x32x2xbf16> -// CHECK-NOT: linalg.fill -// CHECK: %[[RES:.+]] = microkernel.brgemm ins(%[[A]], %[[B]] : [[TYPE:.+]]) outs(%[[C]] : [[TYPE2:.+]]) batch_dims(0, 0) leading_dims(1, 1) flags(beta_0) -> tensor<32x32xf32> -// CHECK-NEXT: scf.forall.in_parallel - -// ----- - -#map1 = affine_map<(d0, d1) -> (d0, d1)> -func.func @basic_linalg_to_microkernel_fusing_transpose(%arg0: tensor<4x8x32x32xf32>) -> tensor<4x8x32x32xf32> { - %cst = arith.constant 0.000000e+00 : f32 - %alloc_1 = tensor.empty() : tensor<4x16x32x32xf32> - %alloc_4 = tensor.empty() : tensor<8x32x16x32xf32> - %trans_base = tensor.empty() : tensor<16x32x32xf32> - %ret = scf.forall (%arg7, %arg8) in (4, 8) shared_outs(%argp = %arg0) -> (tensor<4x8x32x32xf32>) { - %alloc_10 = tensor.extract_slice %argp[%arg7, %arg8, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : tensor<4x8x32x32xf32> to tensor<32x32xf32> - %subview = tensor.extract_slice %alloc_1[%arg7, 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : tensor<4x16x32x32xf32> to tensor<16x32x32xf32> - %subview_11 = tensor.extract_slice %alloc_4[%arg8, 0, 0, 0] [1, 32, 16, 32] [1, 1, 1, 1] : tensor<8x32x16x32xf32> to tensor<32x16x32xf32> - %11 = linalg.fill ins(%cst : f32) outs(%alloc_10 : tensor<32x32xf32>) -> tensor<32x32xf32> - %transposed = linalg.transpose ins(%subview_11 : tensor<32x16x32xf32>) outs(%trans_base : tensor<16x32x32xf32>) permutation = [1, 0, 2] - %res = linalg.batch_reduce_matmul ins(%subview, %transposed : tensor<16x32x32xf32>, tensor<16x32x32xf32>) outs(%11 : tensor<32x32xf32>) -> tensor<32x32xf32> - scf.forall.in_parallel { - tensor.parallel_insert_slice %res into %argp[%arg7, %arg8, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : tensor<32x32xf32> into tensor<4x8x32x32xf32> - } - } - return %ret : tensor<4x8x32x32xf32> -} - -// CHECK-LABEL: basic_linalg_to_microkernel_fusing_transpose -// CHECK: scf.forall -// CHECK: %[[C:.+]] = tensor.extract_slice %[[Csrc:.+]][%[[arg1:.+]], %[[arg2:.+]], 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : tensor<4x8x32x32xf32> to tensor<32x32xf32> -// CHECK: %[[A:.+]] = tensor.extract_slice %[[Asrc:.+]][%[[arg1]], 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : tensor<4x16x32x32xf32> to tensor<16x32x32xf32> -// CHECK: %[[B:.+]] = tensor.extract_slice %[[Bsrc:.+]][%[[arg2]], 0, 0, 0] [1, 32, 16, 32] [1, 1, 1, 1] : tensor<8x32x16x32xf32> to tensor<32x16x32xf32> -// CHECK-NOT: linalg.fill -// CHECK-NOT: linalg.transpose -// CHECK: %[[RES:.+]] = microkernel.brgemm ins(%[[A]], %[[B]] : [[TYPE:.+]]) outs(%[[C]] : [[TYPE2:.+]]) batch_dims(0, 1) leading_dims(1, 0) flags(beta_0) -> tensor<32x32xf32> -// CHECK-NEXT: scf.forall.in_parallel - -// ----- - -#map1 = affine_map<(d0, d1) -> (d0, d1)> -func.func @vnni_linalg_to_microkernel_fusing_transpose(%arg0: tensor<4x8x32x32xf32>) -> tensor<4x8x32x32xf32> { - %cst = arith.constant 0.000000e+00 : f32 - %alloc_1 = tensor.empty() : tensor<4x16x32x32xbf16> - %alloc_4 = tensor.empty() : tensor<8x16x16x32x2xbf16> - %trans_base = tensor.empty() : tensor<16x16x32x2xbf16> - %ret = scf.forall (%arg7, %arg8) in (4, 8) shared_outs(%argp = %arg0) -> (tensor<4x8x32x32xf32>) { - %alloc_10 = tensor.extract_slice %argp[%arg7, %arg8, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : tensor<4x8x32x32xf32> to tensor<32x32xf32> - %subview = tensor.extract_slice %alloc_1[%arg7, 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : tensor<4x16x32x32xbf16> to tensor<16x32x32xbf16> - %subview_11 = tensor.extract_slice %alloc_4[%arg8, 0, 0, 0, 0] [1, 16, 16, 32, 2] [1, 1, 1, 1, 1] : tensor<8x16x16x32x2xbf16> to tensor<16x16x32x2xbf16> - %11 = linalg.fill ins(%cst : f32) outs(%alloc_10 : tensor<32x32xf32>) -> tensor<32x32xf32> - %transposed = linalg.transpose ins(%subview_11 : tensor<16x16x32x2xbf16>) outs(%trans_base : tensor<16x16x32x2xbf16>) permutation = [1, 0, 2, 3] - %res = linalg.generic { - indexing_maps = [affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d3 * 2 + d4)>, - affine_map<(d0, d1, d2, d3, d4) -> (d0, d3, d2, d4)>, - affine_map<(d0, d1, d2, d3, d4) -> (d1, d2)>], - iterator_types = ["reduction", "parallel", "parallel", "reduction", "reduction"]} - ins(%subview, %transposed : tensor<16x32x32xbf16>, tensor<16x16x32x2xbf16>) - outs(%11 : tensor<32x32xf32>) { - ^bb0(%in: bf16, %in_0: bf16, %out: f32): - %1 = arith.extf %in : bf16 to f32 - %2 = arith.extf %in_0 : bf16 to f32 - %3 = arith.mulf %1, %2 : f32 - %4 = arith.addf %out, %3 : f32 - linalg.yield %4 : f32 - } -> tensor<32x32xf32> - scf.forall.in_parallel { - tensor.parallel_insert_slice %res into %argp[%arg7, %arg8, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : tensor<32x32xf32> into tensor<4x8x32x32xf32> - } - } - return %ret : tensor<4x8x32x32xf32> -} - -// CHECK-LABEL: vnni_linalg_to_microkernel_fusing_transpose -// CHECK: scf.forall -// CHECK: %[[C:.+]] = tensor.extract_slice %[[Csrc:.+]][%[[arg1:.+]], %[[arg2:.+]], 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : tensor<4x8x32x32xf32> to tensor<32x32xf32> -// CHECK: %[[A:.+]] = tensor.extract_slice %[[Asrc:.+]][%[[arg1]], 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : tensor<4x16x32x32xbf16> to tensor<16x32x32xbf16> -// CHECK: %[[B:.+]] = tensor.extract_slice %[[Bsrc:.+]][%[[arg2]], 0, 0, 0, 0] [1, 16, 16, 32, 2] [1, 1, 1, 1, 1] : tensor<8x16x16x32x2xbf16> to tensor<16x16x32x2xbf16> -// CHECK-NOT: linalg.fill -// CHECK-NOT: linalg.transpose -// CHECK: %[[RES:.+]] = microkernel.brgemm ins(%[[A]], %[[B]] : [[TYPE:.+]]) outs(%[[C]] : [[TYPE2:.+]]) batch_dims(0, 1) leading_dims(1, 0) flags(beta_0) -> tensor<32x32xf32> -// CHECK-NEXT: scf.forall.in_parallel - -// ----- - -#map1 = affine_map<(d0, d1) -> (d0, d1)> -func.func @basic_linalg_to_microkernel_fusing_with_branch(%arg0: tensor<4x8x32x32xf32>) -> tensor<4x8x32x32xf32> { - %cst = arith.constant 0.000000e+00 : f32 - %cst_0 = arith.constant 0 : index - %cst_1 = arith.constant 1 : index - %alloc_1 = tensor.empty() : tensor<4x16x32x32xf32> - %alloc_4 = tensor.empty() : tensor<8x32x16x32xf32> - %trans_base = tensor.empty() : tensor<16x32x32xf32> - %ret = scf.forall (%arg7, %arg8) in (4, 8) shared_outs(%argp = %arg0) -> (tensor<4x8x32x32xf32>) { - %alloc_10 = tensor.extract_slice %argp[%arg7, %arg8, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : tensor<4x8x32x32xf32> to tensor<32x32xf32> - %subview = tensor.extract_slice %alloc_1[%arg7, 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : tensor<4x16x32x32xf32> to tensor<16x32x32xf32> - %subview_11 = tensor.extract_slice %alloc_4[%arg8, 0, 0, 0] [1, 32, 16, 32] [1, 1, 1, 1] : tensor<8x32x16x32xf32> to tensor<32x16x32xf32> - %transposed = linalg.transpose ins(%subview_11 : tensor<32x16x32xf32>) outs(%trans_base : tensor<16x32x32xf32>) permutation = [1, 0, 2] - %6 = arith.cmpi eq, %cst_0, %cst_1 : index - %branch_res = scf.if %6 -> (tensor<32x32xf32>) { - %11 = linalg.fill ins(%cst : f32) outs(%alloc_10 : tensor<32x32xf32>) -> tensor<32x32xf32> - %res = linalg.batch_reduce_matmul ins(%subview, %transposed : tensor<16x32x32xf32>, tensor<16x32x32xf32>) outs(%11 : tensor<32x32xf32>) -> tensor<32x32xf32> - scf.yield %res : tensor<32x32xf32> - } else { - %res = linalg.batch_reduce_matmul ins(%subview, %transposed : tensor<16x32x32xf32>, tensor<16x32x32xf32>) outs(%alloc_10: tensor<32x32xf32>) -> tensor<32x32xf32> - scf.yield %res : tensor<32x32xf32> - } - scf.forall.in_parallel { - tensor.parallel_insert_slice %branch_res into %argp[%arg7, %arg8, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : tensor<32x32xf32> into tensor<4x8x32x32xf32> - } - } - return %ret : tensor<4x8x32x32xf32> -} - -// CHECK-LABEL: basic_linalg_to_microkernel_fusing_with_branch -// CHECK: scf.forall -// CHECK: %[[C:.+]] = tensor.extract_slice %[[Csrc:.+]][%[[arg1:.+]], %[[arg2:.+]], 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : tensor<4x8x32x32xf32> to tensor<32x32xf32> -// CHECK: %[[A:.+]] = tensor.extract_slice %[[Asrc:.+]][%[[arg1]], 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : tensor<4x16x32x32xf32> to tensor<16x32x32xf32> -// CHECK: %[[B:.+]] = tensor.extract_slice %[[Bsrc:.+]][%[[arg2]], 0, 0, 0] [1, 32, 16, 32] [1, 1, 1, 1] : tensor<8x32x16x32xf32> to tensor<32x16x32xf32> -// CHECK-NOT: linalg.transpose -// CHECK: scf.if -// CHECK-NOT: linalg.fill -// CHECK: %[[RES:.+]] = microkernel.brgemm ins(%[[A]], %[[B]] : [[TYPE:.+]]) outs(%[[C]] : [[TYPE2:.+]]) batch_dims(0, 1) leading_dims(1, 0) flags(beta_0) -> tensor<32x32xf32> -// CHECK: else -// CHECK: %[[RES]] = microkernel.brgemm ins(%[[A]], %[[B]] : [[TYPE:.+]]) outs(%[[C]] : [[TYPE2:.+]]) batch_dims(0, 1) leading_dims(1, 0) flags() -> tensor<32x32xf32> -// CHECK: scf.forall.in_parallel - -// ----- diff --git a/test/mlir/test/gc/Dialect/Microkernel/merge-branch-microkernel-context.mlir b/test/mlir/test/gc/Dialect/Microkernel/merge-branch-microkernel-context.mlir deleted file mode 100644 index 7429ec970..000000000 --- a/test/mlir/test/gc/Dialect/Microkernel/merge-branch-microkernel-context.mlir +++ /dev/null @@ -1,460 +0,0 @@ -// RUN: gc-opt %s -merge-branch-microkernel-context -split-input-file | FileCheck %s - -module { - llvm.mlir.global_ctors {ctors = [@g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_1024_1024_2_2_ctor, @g_dispatched_microkernel_brgemm_stride_init_32_32_32_32_32_32_1024_1024_2_2_ctor], priorities = [2147483647 : i32, 2147483647 : i32]} - llvm.mlir.global internal @g_dispatched_microkernel_brgemm_stride_init_32_32_32_32_32_32_1024_1024_2_2() {addr_space = 0 : i32} : i64 - llvm.func @g_dispatched_microkernel_brgemm_stride_init_32_32_32_32_32_32_1024_1024_2_2_ctor() -> i64 { - %0 = llvm.mlir.addressof @g_dispatched_microkernel_brgemm_stride_init_32_32_32_32_32_32_1024_1024_2_2 : !llvm.ptr - %c32_i64 = arith.constant 32 : i64 - %c1024_i64 = arith.constant 1024 : i64 - %cst = arith.constant 0.000000e+00 : f32 - %c2_i64 = arith.constant 2 : i64 - %1 = func.call @dnnl_brgemm_dispatch(%c32_i64, %c32_i64, %c32_i64, %c32_i64, %c32_i64, %c32_i64, %c1024_i64, %c1024_i64, %cst, %c2_i64, %c2_i64) : (i64, i64, i64, i64, i64, i64, i64, i64, f32, i64, i64) -> i64 - llvm.store %1, %0 : i64, !llvm.ptr - llvm.return %1 : i64 - } - llvm.mlir.global internal @g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_1024_1024_2_2() {addr_space = 0 : i32} : i64 - llvm.func @g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_1024_1024_2_2_ctor() -> i64 { - %0 = llvm.mlir.addressof @g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_1024_1024_2_2 : !llvm.ptr - %c32_i64 = arith.constant 32 : i64 - %c1024_i64 = arith.constant 1024 : i64 - %cst = arith.constant 1.000000e+00 : f32 - %c2_i64 = arith.constant 2 : i64 - %1 = func.call @dnnl_brgemm_dispatch(%c32_i64, %c32_i64, %c32_i64, %c32_i64, %c32_i64, %c32_i64, %c1024_i64, %c1024_i64, %cst, %c2_i64, %c2_i64) : (i64, i64, i64, i64, i64, i64, i64, i64, f32, i64, i64) -> i64 - llvm.store %1, %0 : i64, !llvm.ptr - llvm.return %1 : i64 - } - func.func private @dnnl_brgemm_tilerelease() - func.func private @dnnl_brgemm_execute(i64, !llvm.ptr, index, !llvm.ptr, index, !llvm.ptr, index, i64) - func.func private @dnnl_brgemm_tileconfig(i64) - func.func private @dnnl_brgemm_dispatch(i64, i64, i64, i64, i64, i64, i64, i64, f32, i64, i64) -> i64 - func.func @if_branch_context_merge() { - %cst = arith.constant 0.000000e+00 : f32 - %c16_i64 = arith.constant 16 : i64 - %c8 = arith.constant 8 : index - %c4 = arith.constant 4 : index - %c1 = arith.constant 1 : index - %c0 = arith.constant 0 : index - %0 = llvm.mlir.addressof @g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_1024_1024_2_2 : !llvm.ptr - %1 = llvm.mlir.addressof @g_dispatched_microkernel_brgemm_stride_init_32_32_32_32_32_32_1024_1024_2_2 : !llvm.ptr - %2 = llvm.load %1 : !llvm.ptr -> i64 - %3 = llvm.load %0 : !llvm.ptr -> i64 - %alloc = memref.alloc() {alignment = 64 : i64} : memref<4x16x32x32xbf16> - %alloc_0 = memref.alloc() {alignment = 64 : i64} : memref<8x16x16x32x2xbf16> - scf.for %arg0 = %c0 to %c4 step %c1 { - scf.for %arg1 = %c0 to %c8 step %c1 { - %alloc_1 = memref.alloc() {alignment = 64 : i64} : memref<32x32xf32> - %intptr = memref.extract_aligned_pointer_as_index %alloc_1 : memref<32x32xf32> -> index - %4 = arith.index_cast %intptr : index to i64 - %5 = llvm.inttoptr %4 : i64 to !llvm.ptr - linalg.fill ins(%cst : f32) outs(%alloc_1 : memref<32x32xf32>) - %subview = memref.subview %alloc[%arg0, 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : memref<4x16x32x32xbf16> to memref<16x32x32xbf16, strided<[1024, 32, 1], offset: ?>> - %base_buffer, %offset, %sizes:3, %strides:3 = memref.extract_strided_metadata %subview : memref<16x32x32xbf16, strided<[1024, 32, 1], offset: ?>> -> memref, index, index, index, index, index, index, index - %intptr_2 = memref.extract_aligned_pointer_as_index %subview : memref<16x32x32xbf16, strided<[1024, 32, 1], offset: ?>> -> index - %6 = arith.index_cast %intptr_2 : index to i64 - %7 = llvm.inttoptr %6 : i64 to !llvm.ptr - %subview_3 = memref.subview %alloc_0[%arg1, 0, 0, 0, 0] [1, 16, 16, 32, 2] [1, 1, 1, 1, 1] : memref<8x16x16x32x2xbf16> to memref<16x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>> - %base_buffer_4, %offset_5, %sizes_6:4, %strides_7:4 = memref.extract_strided_metadata %subview_3 : memref<16x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>> -> memref, index, index, index, index, index, index, index, index, index - %intptr_8 = memref.extract_aligned_pointer_as_index %subview_3 : memref<16x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>> -> index - %8 = arith.index_cast %intptr_8 : index to i64 - %9 = llvm.inttoptr %8 : i64 to !llvm.ptr - %10 = arith.cmpi eq, %arg0, %c0 : index - scf.if %10 { - func.call @dnnl_brgemm_tileconfig(%2) : (i64) -> () - func.call @dnnl_brgemm_execute(%2, %7, %offset, %9, %offset_5, %5, %c0, %c16_i64) : (i64, !llvm.ptr, index, !llvm.ptr, index, !llvm.ptr, index, i64) -> () - func.call @dnnl_brgemm_tilerelease() : () -> () - } else { - func.call @dnnl_brgemm_tileconfig(%3) : (i64) -> () - func.call @dnnl_brgemm_execute(%3, %7, %offset, %9, %offset_5, %5, %c0, %c16_i64) : (i64, !llvm.ptr, index, !llvm.ptr, index, !llvm.ptr, index, i64) -> () - func.call @dnnl_brgemm_tilerelease() : () -> () - } - memref.dealloc %alloc_1 : memref<32x32xf32> - } - } - return - } -} - -// CHECK-LABEL: if_branch_context_merge - -// CHECK: scf.for %arg0 = %c0 to %c4 step %c1 -// CHECK-NEXT: scf.for %arg1 = %c0 to %c8 step %c1 - -// CHECK: func.call @dnnl_brgemm_tileconfig -// CHECK-NEXT: scf.if -// CHECK: } else { -// CHECK: } -// CHECK-NEXT: func.call @dnnl_brgemm_tilerelease() : () -> () - -// ----- - -module { - llvm.mlir.global_ctors {ctors = [@g_dispatched_microkernel_brgemm_stride_init_32_32_32_32_32_32_1024_1024_2_2_ctor], priorities = [2147483647 : i32]} - llvm.mlir.global internal @g_dispatched_microkernel_brgemm_stride_init_32_32_32_32_32_32_1024_1024_2_2() {addr_space = 0 : i32} : i64 - llvm.func @g_dispatched_microkernel_brgemm_stride_init_32_32_32_32_32_32_1024_1024_2_2_ctor() -> i64 { - %0 = llvm.mlir.addressof @g_dispatched_microkernel_brgemm_stride_init_32_32_32_32_32_32_1024_1024_2_2 : !llvm.ptr - %c32_i64 = arith.constant 32 : i64 - %c1024_i64 = arith.constant 1024 : i64 - %cst = arith.constant 0.000000e+00 : f32 - %c2_i64 = arith.constant 2 : i64 - %1 = func.call @dnnl_brgemm_dispatch(%c32_i64, %c32_i64, %c32_i64, %c32_i64, %c32_i64, %c32_i64, %c1024_i64, %c1024_i64, %cst, %c2_i64, %c2_i64) : (i64, i64, i64, i64, i64, i64, i64, i64, f32, i64, i64) -> i64 - llvm.store %1, %0 : i64, !llvm.ptr - llvm.return %1 : i64 - } - func.func private @dnnl_brgemm_tilerelease() - func.func private @dnnl_brgemm_execute(i64, !llvm.ptr, index, !llvm.ptr, index, !llvm.ptr, index, i64) - func.func private @dnnl_brgemm_tileconfig(i64) - func.func private @dnnl_brgemm_dispatch(i64, i64, i64, i64, i64, i64, i64, i64, f32, i64, i64) -> i64 - func.func @if_only_branch_context_merge() { - %cst = arith.constant 0.000000e+00 : f32 - %c16_i64 = arith.constant 16 : i64 - %c8 = arith.constant 8 : index - %c4 = arith.constant 4 : index - %c1 = arith.constant 1 : index - %c0 = arith.constant 0 : index - %0 = llvm.mlir.addressof @g_dispatched_microkernel_brgemm_stride_init_32_32_32_32_32_32_1024_1024_2_2 : !llvm.ptr - %1 = llvm.load %0 : !llvm.ptr -> i64 - %alloc = memref.alloc() {alignment = 64 : i64} : memref<4x16x32x32xbf16> - %alloc_0 = memref.alloc() {alignment = 64 : i64} : memref<8x16x16x32x2xbf16> - scf.for %arg0 = %c0 to %c4 step %c1 { - scf.for %arg1 = %c0 to %c8 step %c1 { - %alloc_1 = memref.alloc() {alignment = 64 : i64} : memref<32x32xf32> - %intptr = memref.extract_aligned_pointer_as_index %alloc_1 : memref<32x32xf32> -> index - %2 = arith.index_cast %intptr : index to i64 - %3 = llvm.inttoptr %2 : i64 to !llvm.ptr - linalg.fill ins(%cst : f32) outs(%alloc_1 : memref<32x32xf32>) - %subview = memref.subview %alloc[%arg0, 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : memref<4x16x32x32xbf16> to memref<16x32x32xbf16, strided<[1024, 32, 1], offset: ?>> - %base_buffer, %offset, %sizes:3, %strides:3 = memref.extract_strided_metadata %subview : memref<16x32x32xbf16, strided<[1024, 32, 1], offset: ?>> -> memref, index, index, index, index, index, index, index - %intptr_2 = memref.extract_aligned_pointer_as_index %subview : memref<16x32x32xbf16, strided<[1024, 32, 1], offset: ?>> -> index - %4 = arith.index_cast %intptr_2 : index to i64 - %5 = llvm.inttoptr %4 : i64 to !llvm.ptr - %subview_3 = memref.subview %alloc_0[%arg1, 0, 0, 0, 0] [1, 16, 16, 32, 2] [1, 1, 1, 1, 1] : memref<8x16x16x32x2xbf16> to memref<16x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>> - %base_buffer_4, %offset_5, %sizes_6:4, %strides_7:4 = memref.extract_strided_metadata %subview_3 : memref<16x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>> -> memref, index, index, index, index, index, index, index, index, index - %intptr_8 = memref.extract_aligned_pointer_as_index %subview_3 : memref<16x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>> -> index - %6 = arith.index_cast %intptr_8 : index to i64 - %7 = llvm.inttoptr %6 : i64 to !llvm.ptr - %8 = arith.cmpi eq, %arg0, %c0 : index - scf.if %8 { - func.call @dnnl_brgemm_tileconfig(%1) : (i64) -> () - func.call @dnnl_brgemm_execute(%1, %5, %offset, %7, %offset_5, %3, %c0, %c16_i64) : (i64, !llvm.ptr, index, !llvm.ptr, index, !llvm.ptr, index, i64) -> () - func.call @dnnl_brgemm_tilerelease() : () -> () - } - memref.dealloc %alloc_1 : memref<32x32xf32> - } - } - return - } -} - -// CHECK-LABEL: if_only_branch_context_merge - -// CHECK: scf.for %arg0 = %c0 to %c4 step %c1 -// CHECK-NEXT: scf.for %arg1 = %c0 to %c8 step %c1 - -// CHECK: scf.if -// CHECK: func.call @dnnl_brgemm_tileconfig -// CHECK: func.call @dnnl_brgemm_tilerelease() : () -> () -// CHECK: } - -// ----- - -module { - llvm.mlir.global_ctors {ctors = [@g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_512_512_2_2_ctor, @g_dispatched_microkernel_brgemm_stride_init_32_32_32_32_32_32_1024_1024_2_2_ctor], priorities = [2147483647 : i32, 2147483647 : i32]} - llvm.mlir.global internal @g_dispatched_microkernel_brgemm_stride_init_32_32_32_32_32_32_1024_1024_2_2() {addr_space = 0 : i32} : i64 - llvm.func @g_dispatched_microkernel_brgemm_stride_init_32_32_32_32_32_32_1024_1024_2_2_ctor() -> i64 { - %0 = llvm.mlir.addressof @g_dispatched_microkernel_brgemm_stride_init_32_32_32_32_32_32_1024_1024_2_2 : !llvm.ptr - %c32_i64 = arith.constant 32 : i64 - %c1024_i64 = arith.constant 1024 : i64 - %cst = arith.constant 0.000000e+00 : f32 - %c2_i64 = arith.constant 2 : i64 - %1 = func.call @dnnl_brgemm_dispatch(%c32_i64, %c32_i64, %c32_i64, %c32_i64, %c32_i64, %c32_i64, %c1024_i64, %c1024_i64, %cst, %c2_i64, %c2_i64) : (i64, i64, i64, i64, i64, i64, i64, i64, f32, i64, i64) -> i64 - llvm.store %1, %0 : i64, !llvm.ptr - llvm.return %1 : i64 - } - llvm.mlir.global internal @g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_512_512_2_2() {addr_space = 0 : i32} : i64 - llvm.func @g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_512_512_2_2_ctor() -> i64 { - %0 = llvm.mlir.addressof @g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_512_512_2_2 : !llvm.ptr - %c32_i64 = arith.constant 32 : i64 - %c512_i64 = arith.constant 512 : i64 - %cst = arith.constant 1.000000e+00 : f32 - %c2_i64 = arith.constant 2 : i64 - %1 = func.call @dnnl_brgemm_dispatch(%c32_i64, %c32_i64, %c32_i64, %c32_i64, %c32_i64, %c32_i64, %c512_i64, %c512_i64, %cst, %c2_i64, %c2_i64) : (i64, i64, i64, i64, i64, i64, i64, i64, f32, i64, i64) -> i64 - llvm.store %1, %0 : i64, !llvm.ptr - llvm.return %1 : i64 - } - func.func private @dnnl_brgemm_tilerelease() - func.func private @dnnl_brgemm_execute(i64, !llvm.ptr, index, !llvm.ptr, index, !llvm.ptr, index, i64) - func.func private @dnnl_brgemm_tileconfig(i64) - func.func private @dnnl_brgemm_dispatch(i64, i64, i64, i64, i64, i64, i64, i64, f32, i64, i64) -> i64 - func.func @if_branch_context_no_merge() { - %cst = arith.constant 0.000000e+00 : f32 - %c16_i64 = arith.constant 16 : i64 - %c8 = arith.constant 8 : index - %c4 = arith.constant 4 : index - %c1 = arith.constant 1 : index - %c0 = arith.constant 0 : index - %0 = llvm.mlir.addressof @g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_512_512_2_2 : !llvm.ptr - %1 = llvm.mlir.addressof @g_dispatched_microkernel_brgemm_stride_init_32_32_32_32_32_32_1024_1024_2_2 : !llvm.ptr - %2 = llvm.load %1 : !llvm.ptr -> i64 - %3 = llvm.load %0 : !llvm.ptr -> i64 - %alloc = memref.alloc() {alignment = 64 : i64} : memref<4x16x32x32xbf16> - %alloc_0 = memref.alloc() {alignment = 64 : i64} : memref<8x16x16x32x2xbf16> - scf.for %arg0 = %c0 to %c4 step %c1 { - scf.for %arg1 = %c0 to %c8 step %c1 { - %alloc_1 = memref.alloc() {alignment = 64 : i64} : memref<32x32xf32> - %intptr = memref.extract_aligned_pointer_as_index %alloc_1 : memref<32x32xf32> -> index - %4 = arith.index_cast %intptr : index to i64 - %5 = llvm.inttoptr %4 : i64 to !llvm.ptr - linalg.fill ins(%cst : f32) outs(%alloc_1 : memref<32x32xf32>) - %subview = memref.subview %alloc[%arg0, 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : memref<4x16x32x32xbf16> to memref<16x32x32xbf16, strided<[1024, 32, 1], offset: ?>> - %base_buffer, %offset, %sizes:3, %strides:3 = memref.extract_strided_metadata %subview : memref<16x32x32xbf16, strided<[1024, 32, 1], offset: ?>> -> memref, index, index, index, index, index, index, index - %intptr_2 = memref.extract_aligned_pointer_as_index %subview : memref<16x32x32xbf16, strided<[1024, 32, 1], offset: ?>> -> index - %6 = arith.index_cast %intptr_2 : index to i64 - %7 = llvm.inttoptr %6 : i64 to !llvm.ptr - %subview_3 = memref.subview %alloc_0[%arg1, 0, 0, 0, 0] [1, 16, 16, 32, 2] [1, 1, 1, 1, 1] : memref<8x16x16x32x2xbf16> to memref<16x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>> - %base_buffer_4, %offset_5, %sizes_6:4, %strides_7:4 = memref.extract_strided_metadata %subview_3 : memref<16x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>> -> memref, index, index, index, index, index, index, index, index, index - %intptr_8 = memref.extract_aligned_pointer_as_index %subview_3 : memref<16x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>> -> index - %8 = arith.index_cast %intptr_8 : index to i64 - %9 = llvm.inttoptr %8 : i64 to !llvm.ptr - %10 = arith.cmpi eq, %arg0, %c0 : index - scf.if %10 { - func.call @dnnl_brgemm_tileconfig(%2) : (i64) -> () - func.call @dnnl_brgemm_execute(%2, %7, %offset, %9, %offset_5, %5, %c0, %c16_i64) : (i64, !llvm.ptr, index, !llvm.ptr, index, !llvm.ptr, index, i64) -> () - func.call @dnnl_brgemm_tilerelease() : () -> () - } else { - func.call @dnnl_brgemm_tileconfig(%3) : (i64) -> () - func.call @dnnl_brgemm_execute(%3, %7, %offset, %9, %offset_5, %5, %c0, %c16_i64) : (i64, !llvm.ptr, index, !llvm.ptr, index, !llvm.ptr, index, i64) -> () - func.call @dnnl_brgemm_tilerelease() : () -> () - } - memref.dealloc %alloc_1 : memref<32x32xf32> - } - } - return - } -} - -// CHECK-LABEL: if_branch_context_no_merge - -// CHECK: scf.for %arg0 = %c0 to %c4 step %c1 -// CHECK-NEXT: scf.for %arg1 = %c0 to %c8 step %c1 - -// CHECK: scf.if -// CHECK: func.call @dnnl_brgemm_tileconfig -// CHECK: func.call @dnnl_brgemm_tilerelease() : () -> () -// CHECK: } else { -// CHECK: func.call @dnnl_brgemm_tileconfig -// CHECK: func.call @dnnl_brgemm_tilerelease() : () -> () -// CHECK: } - -// ----- - -module { - llvm.mlir.global_ctors {ctors = [@g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_1024_1024_2_2_ctor, @g_dispatched_microkernel_brgemm_stride_init_32_32_32_32_32_32_1024_1024_2_2_ctor], priorities = [2147483647 : i32, 2147483647 : i32]} - llvm.mlir.global internal @g_dispatched_microkernel_brgemm_stride_init_32_32_32_32_32_32_1024_1024_2_2() {addr_space = 0 : i32} : i64 - llvm.func @g_dispatched_microkernel_brgemm_stride_init_32_32_32_32_32_32_1024_1024_2_2_ctor() -> i64 { - %0 = llvm.mlir.addressof @g_dispatched_microkernel_brgemm_stride_init_32_32_32_32_32_32_1024_1024_2_2 : !llvm.ptr - %c32_i64 = arith.constant 32 : i64 - %c1024_i64 = arith.constant 1024 : i64 - %cst = arith.constant 0.000000e+00 : f32 - %c2_i64 = arith.constant 2 : i64 - %1 = func.call @dnnl_brgemm_dispatch(%c32_i64, %c32_i64, %c32_i64, %c32_i64, %c32_i64, %c32_i64, %c1024_i64, %c1024_i64, %cst, %c2_i64, %c2_i64) : (i64, i64, i64, i64, i64, i64, i64, i64, f32, i64, i64) -> i64 - llvm.store %1, %0 : i64, !llvm.ptr - llvm.return %1 : i64 - } - llvm.mlir.global internal @g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_1024_1024_2_2() {addr_space = 0 : i32} : i64 - llvm.func @g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_1024_1024_2_2_ctor() -> i64 { - %0 = llvm.mlir.addressof @g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_1024_1024_2_2 : !llvm.ptr - %c32_i64 = arith.constant 32 : i64 - %c1024_i64 = arith.constant 1024 : i64 - %cst = arith.constant 1.000000e+00 : f32 - %c2_i64 = arith.constant 2 : i64 - %1 = func.call @dnnl_brgemm_dispatch(%c32_i64, %c32_i64, %c32_i64, %c32_i64, %c32_i64, %c32_i64, %c1024_i64, %c1024_i64, %cst, %c2_i64, %c2_i64) : (i64, i64, i64, i64, i64, i64, i64, i64, f32, i64, i64) -> i64 - llvm.store %1, %0 : i64, !llvm.ptr - llvm.return %1 : i64 - } - func.func private @dnnl_brgemm_tilerelease() - func.func private @dnnl_brgemm_execute(i64, !llvm.ptr, index, !llvm.ptr, index, !llvm.ptr, index, i64) - func.func private @dnnl_brgemm_tileconfig(i64) - func.func private @dnnl_brgemm_dispatch(i64, i64, i64, i64, i64, i64, i64, i64, f32, i64, i64) -> i64 - func.func @switch_branch_context_merge() { - %cst = arith.constant 0.000000e+00 : f32 - %c16_i64 = arith.constant 16 : i64 - %c8 = arith.constant 8 : index - %c4 = arith.constant 4 : index - %c1 = arith.constant 1 : index - %c0 = arith.constant 0 : index - %0 = llvm.mlir.addressof @g_dispatched_microkernel_brgemm_stride_init_32_32_32_32_32_32_1024_1024_2_2 : !llvm.ptr - %1 = llvm.mlir.addressof @g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_1024_1024_2_2 : !llvm.ptr - %2 = llvm.load %1 : !llvm.ptr -> i64 - %3 = llvm.load %0 : !llvm.ptr -> i64 - %alloc = memref.alloc() {alignment = 64 : i64} : memref<4x16x32x32xbf16> - %alloc_0 = memref.alloc() {alignment = 64 : i64} : memref<8x16x16x32x2xbf16> - scf.for %arg0 = %c0 to %c4 step %c1 { - scf.for %arg1 = %c0 to %c8 step %c1 { - %alloc_1 = memref.alloc() {alignment = 64 : i64} : memref<32x32xf32> - %intptr = memref.extract_aligned_pointer_as_index %alloc_1 : memref<32x32xf32> -> index - %4 = arith.index_cast %intptr : index to i64 - %5 = llvm.inttoptr %4 : i64 to !llvm.ptr - linalg.fill ins(%cst : f32) outs(%alloc_1 : memref<32x32xf32>) - %subview = memref.subview %alloc[%arg0, 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : memref<4x16x32x32xbf16> to memref<16x32x32xbf16, strided<[1024, 32, 1], offset: ?>> - %base_buffer, %offset, %sizes:3, %strides:3 = memref.extract_strided_metadata %subview : memref<16x32x32xbf16, strided<[1024, 32, 1], offset: ?>> -> memref, index, index, index, index, index, index, index - %intptr_2 = memref.extract_aligned_pointer_as_index %subview : memref<16x32x32xbf16, strided<[1024, 32, 1], offset: ?>> -> index - %6 = arith.index_cast %intptr_2 : index to i64 - %7 = llvm.inttoptr %6 : i64 to !llvm.ptr - %subview_3 = memref.subview %alloc_0[%arg1, 0, 0, 0, 0] [1, 16, 16, 32, 2] [1, 1, 1, 1, 1] : memref<8x16x16x32x2xbf16> to memref<16x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>> - %base_buffer_4, %offset_5, %sizes_6:4, %strides_7:4 = memref.extract_strided_metadata %subview_3 : memref<16x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>> -> memref, index, index, index, index, index, index, index, index, index - %intptr_8 = memref.extract_aligned_pointer_as_index %subview_3 : memref<16x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>> -> index - %8 = arith.index_cast %intptr_8 : index to i64 - %9 = llvm.inttoptr %8 : i64 to !llvm.ptr - scf.index_switch %arg0 - case 0 { - func.call @dnnl_brgemm_tileconfig(%3) : (i64) -> () - func.call @dnnl_brgemm_execute(%3, %7, %offset, %9, %offset_5, %5, %c0, %c16_i64) : (i64, !llvm.ptr, index, !llvm.ptr, index, !llvm.ptr, index, i64) -> () - func.call @dnnl_brgemm_tilerelease() : () -> () - scf.yield - } - case 1 { - func.call @dnnl_brgemm_tileconfig(%2) : (i64) -> () - func.call @dnnl_brgemm_execute(%2, %7, %offset, %9, %offset_5, %5, %c0, %c16_i64) : (i64, !llvm.ptr, index, !llvm.ptr, index, !llvm.ptr, index, i64) -> () - func.call @dnnl_brgemm_tilerelease() : () -> () - scf.yield - } - default { - func.call @dnnl_brgemm_tileconfig(%2) : (i64) -> () - func.call @dnnl_brgemm_execute(%2, %7, %offset, %9, %offset_5, %5, %c0, %c16_i64) : (i64, !llvm.ptr, index, !llvm.ptr, index, !llvm.ptr, index, i64) -> () - func.call @dnnl_brgemm_tilerelease() : () -> () - } - memref.dealloc %alloc_1 : memref<32x32xf32> - } - } - return - } -} - -// CHECK-LABEL: switch_branch_context_merge - -// CHECK: scf.for %arg0 = %c0 to %c4 step %c1 -// CHECK-NEXT: scf.for %arg1 = %c0 to %c8 step %c1 - -// CHECK: func.call @dnnl_brgemm_tileconfig -// CHECK-NEXT: scf.index_switch -// CHECK: case 0 { -// CHECK: case 1 { -// CHECK: default { -// CHECK: } -// CHECK-NEXT: func.call @dnnl_brgemm_tilerelease() : () -> () - -// ----- - -module { - llvm.mlir.global_ctors {ctors = [@g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_1024_1024_2_2_ctor, @g_dispatched_microkernel_brgemm_stride_init_32_32_32_32_32_32_1024_1024_2_2_ctor, @g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_512_512_2_2_ctor], priorities = [2147483647 : i32, 2147483647 : i32, 2147483647 : i32]} - llvm.mlir.global internal @g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_512_512_2_2() {addr_space = 0 : i32} : i64 - llvm.func @g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_512_512_2_2_ctor() -> i64 { - %0 = llvm.mlir.addressof @g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_512_512_2_2 : !llvm.ptr - %c32_i64 = arith.constant 32 : i64 - %c512_i64 = arith.constant 512 : i64 - %cst = arith.constant 1.000000e+00 : f32 - %c2_i64 = arith.constant 2 : i64 - %1 = func.call @dnnl_brgemm_dispatch(%c32_i64, %c32_i64, %c32_i64, %c32_i64, %c32_i64, %c32_i64, %c512_i64, %c512_i64, %cst, %c2_i64, %c2_i64) : (i64, i64, i64, i64, i64, i64, i64, i64, f32, i64, i64) -> i64 - llvm.store %1, %0 : i64, !llvm.ptr - llvm.return %1 : i64 - } - llvm.mlir.global internal @g_dispatched_microkernel_brgemm_stride_init_32_32_32_32_32_32_1024_1024_2_2() {addr_space = 0 : i32} : i64 - llvm.func @g_dispatched_microkernel_brgemm_stride_init_32_32_32_32_32_32_1024_1024_2_2_ctor() -> i64 { - %0 = llvm.mlir.addressof @g_dispatched_microkernel_brgemm_stride_init_32_32_32_32_32_32_1024_1024_2_2 : !llvm.ptr - %c32_i64 = arith.constant 32 : i64 - %c1024_i64 = arith.constant 1024 : i64 - %cst = arith.constant 0.000000e+00 : f32 - %c2_i64 = arith.constant 2 : i64 - %1 = func.call @dnnl_brgemm_dispatch(%c32_i64, %c32_i64, %c32_i64, %c32_i64, %c32_i64, %c32_i64, %c1024_i64, %c1024_i64, %cst, %c2_i64, %c2_i64) : (i64, i64, i64, i64, i64, i64, i64, i64, f32, i64, i64) -> i64 - llvm.store %1, %0 : i64, !llvm.ptr - llvm.return %1 : i64 - } - llvm.mlir.global internal @g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_1024_1024_2_2() {addr_space = 0 : i32} : i64 - llvm.func @g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_1024_1024_2_2_ctor() -> i64 { - %0 = llvm.mlir.addressof @g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_1024_1024_2_2 : !llvm.ptr - %c32_i64 = arith.constant 32 : i64 - %c1024_i64 = arith.constant 1024 : i64 - %cst = arith.constant 1.000000e+00 : f32 - %c2_i64 = arith.constant 2 : i64 - %1 = func.call @dnnl_brgemm_dispatch(%c32_i64, %c32_i64, %c32_i64, %c32_i64, %c32_i64, %c32_i64, %c1024_i64, %c1024_i64, %cst, %c2_i64, %c2_i64) : (i64, i64, i64, i64, i64, i64, i64, i64, f32, i64, i64) -> i64 - llvm.store %1, %0 : i64, !llvm.ptr - llvm.return %1 : i64 - } - func.func private @dnnl_brgemm_tilerelease() - func.func private @dnnl_brgemm_execute(i64, !llvm.ptr, index, !llvm.ptr, index, !llvm.ptr, index, i64) - func.func private @dnnl_brgemm_tileconfig(i64) - func.func private @dnnl_brgemm_dispatch(i64, i64, i64, i64, i64, i64, i64, i64, f32, i64, i64) -> i64 - func.func @switch_branch_context_no_merge() { - %cst = arith.constant 0.000000e+00 : f32 - %c16_i64 = arith.constant 16 : i64 - %c8 = arith.constant 8 : index - %c4 = arith.constant 4 : index - %c1 = arith.constant 1 : index - %c0 = arith.constant 0 : index - %0 = llvm.mlir.addressof @g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_1024_1024_2_2 : !llvm.ptr - %1 = llvm.mlir.addressof @g_dispatched_microkernel_brgemm_stride_init_32_32_32_32_32_32_1024_1024_2_2 : !llvm.ptr - %2 = llvm.mlir.addressof @g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_512_512_2_2 : !llvm.ptr - %3 = llvm.load %2 : !llvm.ptr -> i64 - %4 = llvm.load %1 : !llvm.ptr -> i64 - %5 = llvm.load %0 : !llvm.ptr -> i64 - %alloc = memref.alloc() {alignment = 64 : i64} : memref<4x16x32x32xbf16> - %alloc_0 = memref.alloc() {alignment = 64 : i64} : memref<8x16x16x32x2xbf16> - scf.for %arg0 = %c0 to %c4 step %c1 { - scf.for %arg1 = %c0 to %c8 step %c1 { - %alloc_1 = memref.alloc() {alignment = 64 : i64} : memref<32x32xf32> - %intptr = memref.extract_aligned_pointer_as_index %alloc_1 : memref<32x32xf32> -> index - %6 = arith.index_cast %intptr : index to i64 - %7 = llvm.inttoptr %6 : i64 to !llvm.ptr - linalg.fill ins(%cst : f32) outs(%alloc_1 : memref<32x32xf32>) - %subview = memref.subview %alloc[%arg0, 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : memref<4x16x32x32xbf16> to memref<16x32x32xbf16, strided<[1024, 32, 1], offset: ?>> - %base_buffer, %offset, %sizes:3, %strides:3 = memref.extract_strided_metadata %subview : memref<16x32x32xbf16, strided<[1024, 32, 1], offset: ?>> -> memref, index, index, index, index, index, index, index - %intptr_2 = memref.extract_aligned_pointer_as_index %subview : memref<16x32x32xbf16, strided<[1024, 32, 1], offset: ?>> -> index - %8 = arith.index_cast %intptr_2 : index to i64 - %9 = llvm.inttoptr %8 : i64 to !llvm.ptr - %subview_3 = memref.subview %alloc_0[%arg1, 0, 0, 0, 0] [1, 16, 16, 32, 2] [1, 1, 1, 1, 1] : memref<8x16x16x32x2xbf16> to memref<16x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>> - %base_buffer_4, %offset_5, %sizes_6:4, %strides_7:4 = memref.extract_strided_metadata %subview_3 : memref<16x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>> -> memref, index, index, index, index, index, index, index, index, index - %intptr_8 = memref.extract_aligned_pointer_as_index %subview_3 : memref<16x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>> -> index - %10 = arith.index_cast %intptr_8 : index to i64 - %11 = llvm.inttoptr %10 : i64 to !llvm.ptr - scf.index_switch %arg0 - case 0 { - func.call @dnnl_brgemm_tileconfig(%4) : (i64) -> () - func.call @dnnl_brgemm_execute(%4, %9, %offset, %11, %offset_5, %7, %c0, %c16_i64) : (i64, !llvm.ptr, index, !llvm.ptr, index, !llvm.ptr, index, i64) -> () - func.call @dnnl_brgemm_tilerelease() : () -> () - scf.yield - } - case 1 { - func.call @dnnl_brgemm_tileconfig(%5) : (i64) -> () - func.call @dnnl_brgemm_execute(%5, %9, %offset, %11, %offset_5, %7, %c0, %c16_i64) : (i64, !llvm.ptr, index, !llvm.ptr, index, !llvm.ptr, index, i64) -> () - func.call @dnnl_brgemm_tilerelease() : () -> () - scf.yield - } - default { - func.call @dnnl_brgemm_tileconfig(%3) : (i64) -> () - func.call @dnnl_brgemm_execute(%3, %9, %offset, %11, %offset_5, %7, %c0, %c16_i64) : (i64, !llvm.ptr, index, !llvm.ptr, index, !llvm.ptr, index, i64) -> () - func.call @dnnl_brgemm_tilerelease() : () -> () - } - memref.dealloc %alloc_1 : memref<32x32xf32> - } - } - return - } -} - -// CHECK-LABEL: switch_branch_context_no_merge - -// CHECK: scf.for %arg0 = %c0 to %c4 step %c1 -// CHECK-NEXT: scf.for %arg1 = %c0 to %c8 step %c1 - -// CHECK: scf.index_switch -// CHECK: case 0 { -// CHECK: func.call @dnnl_brgemm_tileconfig -// CHECK: func.call @dnnl_brgemm_tilerelease() : () -> () -// CHECK: case 1 { -// CHECK: func.call @dnnl_brgemm_tileconfig -// CHECK: func.call @dnnl_brgemm_tilerelease() : () -> () -// CHECK: default { -// CHECK: func.call @dnnl_brgemm_tileconfig -// CHECK: func.call @dnnl_brgemm_tilerelease() : () -> () -// CHECK: } diff --git a/test/mlir/test/gc/Dialect/Microkernel/microkernel-bufferize.mlir b/test/mlir/test/gc/Dialect/Microkernel/microkernel-bufferize.mlir deleted file mode 100644 index bc561790c..000000000 --- a/test/mlir/test/gc/Dialect/Microkernel/microkernel-bufferize.mlir +++ /dev/null @@ -1,53 +0,0 @@ -// RUN: gc-opt %s -one-shot-bufferize -split-input-file | FileCheck %s - -func.func @basic_microkernel_bufferize(%arg0: tensor<4x8x32x32xf32>) -> tensor<4x8x32x32xf32> { - %0 = tensor.empty() : tensor<4x16x32x32xf32> - %1 = tensor.empty() : tensor<8x16x32x32xf32> - %2 = scf.forall (%arg1, %arg2) in (4, 8) shared_outs(%arg3 = %arg0) -> (tensor<4x8x32x32xf32>) { - %extracted_slice = tensor.extract_slice %arg3[%arg1, %arg2, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : tensor<4x8x32x32xf32> to tensor<32x32xf32> - %extracted_slice_0 = tensor.extract_slice %0[%arg1, 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : tensor<4x16x32x32xf32> to tensor<16x32x32xf32> - %extracted_slice_1 = tensor.extract_slice %1[%arg2, 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : tensor<8x16x32x32xf32> to tensor<16x32x32xf32> - %3 = microkernel.brgemm ins(%extracted_slice_0, %extracted_slice_1 : tensor<16x32x32xf32>, tensor<16x32x32xf32>) outs(%extracted_slice : tensor<32x32xf32>) batch_dims(0, 0) leading_dims(1, 1) flags() -> tensor<32x32xf32> - scf.forall.in_parallel { - tensor.parallel_insert_slice %3 into %arg3[%arg1, %arg2, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : tensor<32x32xf32> into tensor<4x8x32x32xf32> - } - } - return %2 : tensor<4x8x32x32xf32> -} - -// CHECK-LABEL: basic_microkernel_bufferize -// CHECK: %[[Abuf:.+]] = memref.alloc() {alignment = 64 : i64} : memref<4x16x32x32xf32> -// CHECK: %[[Bbuf:.+]] = memref.alloc() {alignment = 64 : i64} : memref<8x16x32x32xf32> -// CHECK: %[[Cbuf:.+]] = memref.alloc() {alignment = 64 : i64} : memref<4x8x32x32xf32> -// CHECK: scf.forall -// CHECK: %[[C:.+]] = memref.subview %[[Cbuf]] -// CHECK: %[[A:.+]] = memref.subview %[[Abuf]] -// CHECK: %[[B:.+]] = memref.subview %[[Bbuf]] -// CHECK: microkernel.brgemm ins(%[[A]], %[[B]] : [[TYPE:.+]]) outs(%[[C]] : [[TYPE2:.+]]) batch_dims(0, 0) leading_dims(1, 1) flags() - -// ----- - -func.func @vnni_microkernel_bufferize(%arg0: tensor<4x8x32x32xf32>) -> tensor<4x8x32x32xf32> { - %0 = tensor.empty() : tensor<4x16x32x32xbf16> - %1 = tensor.empty() : tensor<8x16x16x32x2xbf16> - %2 = scf.forall (%arg1, %arg2) in (4, 8) shared_outs(%arg3 = %arg0) -> (tensor<4x8x32x32xf32>) { - %extracted_slice = tensor.extract_slice %arg3[%arg1, %arg2, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : tensor<4x8x32x32xf32> to tensor<32x32xf32> - %extracted_slice_0 = tensor.extract_slice %0[%arg1, 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : tensor<4x16x32x32xbf16> to tensor<16x32x32xbf16> - %extracted_slice_1 = tensor.extract_slice %1[%arg2, 0, 0, 0, 0] [1, 16, 16, 32, 2] [1, 1, 1, 1, 1] : tensor<8x16x16x32x2xbf16> to tensor<16x16x32x2xbf16> - %3 = microkernel.brgemm ins(%extracted_slice_0, %extracted_slice_1 : tensor<16x32x32xbf16>, tensor<16x16x32x2xbf16>) outs(%extracted_slice : tensor<32x32xf32>) batch_dims(0, 0) leading_dims(1, 1) flags() -> tensor<32x32xf32> - scf.forall.in_parallel { - tensor.parallel_insert_slice %3 into %arg3[%arg1, %arg2, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : tensor<32x32xf32> into tensor<4x8x32x32xf32> - } - } - return %2 : tensor<4x8x32x32xf32> -} - -// CHECK-LABEL: vnni_microkernel_bufferize -// CHECK: %[[Abuf:.+]] = memref.alloc() {alignment = 64 : i64} : memref<4x16x32x32xbf16> -// CHECK: %[[Bbuf:.+]] = memref.alloc() {alignment = 64 : i64} : memref<8x16x16x32x2xbf16> -// CHECK: %[[Cbuf:.+]] = memref.alloc() {alignment = 64 : i64} : memref<4x8x32x32xf32> -// CHECK: scf.forall -// CHECK: %[[C:.+]] = memref.subview %[[Cbuf]] -// CHECK: %[[A:.+]] = memref.subview %[[Abuf]] -// CHECK: %[[B:.+]] = memref.subview %[[Bbuf]] -// CHECK: microkernel.brgemm ins(%[[A]], %[[B]] : [[TYPE:.+]]) outs(%[[C]] : [[TYPE2:.+]]) batch_dims(0, 0) leading_dims(1, 1) flags() diff --git a/test/mlir/test/gc/Dialect/Microkernel/microkernel-invariant-code-motion.mlir b/test/mlir/test/gc/Dialect/Microkernel/microkernel-invariant-code-motion.mlir deleted file mode 100644 index 6cd0173c8..000000000 --- a/test/mlir/test/gc/Dialect/Microkernel/microkernel-invariant-code-motion.mlir +++ /dev/null @@ -1,307 +0,0 @@ -// RUN: gc-opt %s -microkernel-invariant-code-motion -split-input-file | FileCheck %s - -module { - llvm.mlir.global_ctors {ctors = [@g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_1024_1024_2_2_ctor], priorities = [2147483647 : i32]} - llvm.mlir.global internal @g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_1024_1024_2_2() {addr_space = 0 : i32} : i64 - llvm.func @g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_1024_1024_2_2_ctor() -> i64 { - %0 = llvm.mlir.addressof @g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_1024_1024_2_2 : !llvm.ptr - %c32_i64 = arith.constant 32 : i64 - %c1024_i64 = arith.constant 1024 : i64 - %cst = arith.constant 1.000000e+00 : f32 - %c2_i64 = arith.constant 2 : i64 - %1 = func.call @dnnl_brgemm_dispatch(%c32_i64, %c32_i64, %c32_i64, %c32_i64, %c32_i64, %c32_i64, %c1024_i64, %c1024_i64, %cst, %c2_i64, %c2_i64) : (i64, i64, i64, i64, i64, i64, i64, i64, f32, i64, i64) -> i64 - llvm.store %1, %0 : i64, !llvm.ptr - llvm.return %1 : i64 - } - func.func private @dnnl_brgemm_tilerelease() - func.func private @dnnl_brgemm_execute(i64, !llvm.ptr, index, !llvm.ptr, index, !llvm.ptr, index, i64) - func.func private @dnnl_brgemm_tileconfig(i64) - func.func private @dnnl_brgemm_dispatch(i64, i64, i64, i64, i64, i64, i64, i64, f32, i64, i64) -> i64 - func.func @parallel_no_hoist() { - %c0 = arith.constant 0 : index - %cst = arith.constant 0.000000e+00 : f32 - %c16_i64 = arith.constant 16 : i64 - %0 = llvm.mlir.addressof @g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_1024_1024_2_2 : !llvm.ptr - %1 = llvm.load %0 : !llvm.ptr -> i64 - %alloc = memref.alloc() {alignment = 64 : i64} : memref<4x16x32x32xbf16> - %alloc_0 = memref.alloc() {alignment = 64 : i64} : memref<8x16x16x32x2xbf16> - scf.forall (%arg0, %arg1) in (4, 8) { - %alloc_1 = memref.alloc() {alignment = 64 : i64} : memref<32x32xf32> - %intptr = memref.extract_aligned_pointer_as_index %alloc_1 : memref<32x32xf32> -> index - %2 = arith.index_cast %intptr : index to i64 - %3 = llvm.inttoptr %2 : i64 to !llvm.ptr - linalg.fill ins(%cst : f32) outs(%alloc_1 : memref<32x32xf32>) - %subview = memref.subview %alloc[%arg0, 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : memref<4x16x32x32xbf16> to memref<16x32x32xbf16, strided<[1024, 32, 1], offset: ?>> - %base_buffer, %offset, %sizes:3, %strides:3 = memref.extract_strided_metadata %subview : memref<16x32x32xbf16, strided<[1024, 32, 1], offset: ?>> -> memref, index, index, index, index, index, index, index - %intptr_2 = memref.extract_aligned_pointer_as_index %subview : memref<16x32x32xbf16, strided<[1024, 32, 1], offset: ?>> -> index - %4 = arith.index_cast %intptr_2 : index to i64 - %5 = llvm.inttoptr %4 : i64 to !llvm.ptr - %subview_3 = memref.subview %alloc_0[%arg1, 0, 0, 0, 0] [1, 16, 16, 32, 2] [1, 1, 1, 1, 1] : memref<8x16x16x32x2xbf16> to memref<16x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>> - %base_buffer_4, %offset_5, %sizes_6:4, %strides_7:4 = memref.extract_strided_metadata %subview_3 : memref<16x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>> -> memref, index, index, index, index, index, index, index, index, index - %intptr_8 = memref.extract_aligned_pointer_as_index %subview_3 : memref<16x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>> -> index - %6 = arith.index_cast %intptr_8 : index to i64 - %7 = llvm.inttoptr %6 : i64 to !llvm.ptr - func.call @dnnl_brgemm_tileconfig(%1) : (i64) -> () - func.call @dnnl_brgemm_execute(%1, %5, %offset, %7, %offset_5, %3, %c0, %c16_i64) : (i64, !llvm.ptr, index, !llvm.ptr, index, !llvm.ptr, index, i64) -> () - func.call @dnnl_brgemm_tilerelease() : () -> () - memref.dealloc %alloc_1 : memref<32x32xf32> - } - return - } -} - -// CHECK-LABEL: parallel_no_hoist - -// CHECK: scf.forall (%arg0, %arg1) in (4, 8) -// CHECK: call @dnnl_brgemm_tileconfig(%[[A:.+]]) : (i64) -> () -// CHECK: call @dnnl_brgemm_execute([[B:.+]]) : ([[C:.+]]) -> () -// CHECK-NEXT: call @dnnl_brgemm_tilerelease() : () -> () - -// ----- - -module { - llvm.mlir.global_ctors {ctors = [@g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_1024_1024_2_2_ctor], priorities = [2147483647 : i32]} - llvm.mlir.global internal @g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_1024_1024_2_2() {addr_space = 0 : i32} : i64 - llvm.func @g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_1024_1024_2_2_ctor() -> i64 { - %0 = llvm.mlir.addressof @g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_1024_1024_2_2 : !llvm.ptr - %c32_i64 = arith.constant 32 : i64 - %c1024_i64 = arith.constant 1024 : i64 - %cst = arith.constant 1.000000e+00 : f32 - %c2_i64 = arith.constant 2 : i64 - %1 = func.call @dnnl_brgemm_dispatch(%c32_i64, %c32_i64, %c32_i64, %c32_i64, %c32_i64, %c32_i64, %c1024_i64, %c1024_i64, %cst, %c2_i64, %c2_i64) : (i64, i64, i64, i64, i64, i64, i64, i64, f32, i64, i64) -> i64 - llvm.store %1, %0 : i64, !llvm.ptr - llvm.return %1 : i64 - } - func.func private @dnnl_brgemm_tilerelease() - func.func private @dnnl_brgemm_execute(i64, !llvm.ptr, index, !llvm.ptr, index, !llvm.ptr, index, i64) - func.func private @dnnl_brgemm_tileconfig(i64) - func.func private @dnnl_brgemm_dispatch(i64, i64, i64, i64, i64, i64, i64, i64, f32, i64, i64) -> i64 - func.func @multi_level_conflict() { - %cst = arith.constant 0.000000e+00 : f32 - %c16_i64 = arith.constant 16 : i64 - %c8 = arith.constant 8 : index - %c4 = arith.constant 4 : index - %c1 = arith.constant 1 : index - %c0 = arith.constant 0 : index - %c2_i64 = arith.constant 2 : i64 - %0 = llvm.mlir.addressof @g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_1024_1024_2_2 : !llvm.ptr - %1 = llvm.load %0 : !llvm.ptr -> i64 - %alloc = memref.alloc() {alignment = 64 : i64} : memref<4x16x32x32xbf16> - %alloc_0 = memref.alloc() {alignment = 64 : i64} : memref<8x16x16x32x2xbf16> - scf.for %arg0 = %c0 to %c4 step %c1 { - scf.for %arg1 = %c0 to %c8 step %c1 { - %alloc_1 = memref.alloc() {alignment = 64 : i64} : memref<32x32xf32> - %intptr = memref.extract_aligned_pointer_as_index %alloc_1 : memref<32x32xf32> -> index - %2 = arith.index_cast %intptr : index to i64 - %3 = llvm.inttoptr %2 : i64 to !llvm.ptr - linalg.fill ins(%cst : f32) outs(%alloc_1 : memref<32x32xf32>) - %subview = memref.subview %alloc[%arg0, 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : memref<4x16x32x32xbf16> to memref<16x32x32xbf16, strided<[1024, 32, 1], offset: ?>> - %base_buffer, %offset, %sizes:3, %strides:3 = memref.extract_strided_metadata %subview : memref<16x32x32xbf16, strided<[1024, 32, 1], offset: ?>> -> memref, index, index, index, index, index, index, index - %intptr_2 = memref.extract_aligned_pointer_as_index %subview : memref<16x32x32xbf16, strided<[1024, 32, 1], offset: ?>> -> index - %4 = arith.index_cast %intptr_2 : index to i64 - %5 = llvm.inttoptr %4 : i64 to !llvm.ptr - %subview_3 = memref.subview %alloc_0[%arg1, 0, 0, 0, 0] [1, 16, 16, 32, 2] [1, 1, 1, 1, 1] : memref<8x16x16x32x2xbf16> to memref<16x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>> - %base_buffer_4, %offset_5, %sizes_6:4, %strides_7:4 = memref.extract_strided_metadata %subview_3 : memref<16x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>> -> memref, index, index, index, index, index, index, index, index, index - %intptr_8 = memref.extract_aligned_pointer_as_index %subview_3 : memref<16x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>> -> index - %6 = arith.index_cast %intptr_8 : index to i64 - %7 = llvm.inttoptr %6 : i64 to !llvm.ptr - func.call @dnnl_brgemm_tileconfig(%1) : (i64) -> () - func.call @dnnl_brgemm_execute(%1, %5, %offset, %7, %offset_5, %3, %c0, %c16_i64) : (i64, !llvm.ptr, index, !llvm.ptr, index, !llvm.ptr, index, i64) -> () - func.call @dnnl_brgemm_tilerelease() : () -> () - %subview_9 = memref.subview %alloc[%arg0, 0, 0, 0] [1, 2, 32, 32] [1, 1, 1, 1] : memref<4x16x32x32xbf16> to memref<2x32x32xbf16, strided<[1024, 32, 1], offset: ?>> - %base_buffer_10, %offset_11, %sizes_12:3, %strides_13:3 = memref.extract_strided_metadata %subview_9 : memref<2x32x32xbf16, strided<[1024, 32, 1], offset: ?>> -> memref, index, index, index, index, index, index, index - %intptr_14 = memref.extract_aligned_pointer_as_index %subview_9 : memref<2x32x32xbf16, strided<[1024, 32, 1], offset: ?>> -> index - %8 = arith.index_cast %intptr_14 : index to i64 - %9 = llvm.inttoptr %8 : i64 to !llvm.ptr - %subview_15 = memref.subview %alloc_0[%arg1, 0, 0, 0, 0] [1, 2, 16, 32, 2] [1, 1, 1, 1, 1] : memref<8x16x16x32x2xbf16> to memref<2x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>> - %base_buffer_16, %offset_17, %sizes_18:4, %strides_19:4 = memref.extract_strided_metadata %subview_15 : memref<2x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>> -> memref, index, index, index, index, index, index, index, index, index - %intptr_20 = memref.extract_aligned_pointer_as_index %subview_15 : memref<2x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>> -> index - %10 = arith.index_cast %intptr_20 : index to i64 - %11 = llvm.inttoptr %10 : i64 to !llvm.ptr - func.call @dnnl_brgemm_tileconfig(%1) : (i64) -> () - func.call @dnnl_brgemm_execute(%1, %9, %offset_11, %11, %offset_17, %3, %c0, %c2_i64) : (i64, !llvm.ptr, index, !llvm.ptr, index, !llvm.ptr, index, i64) -> () - func.call @dnnl_brgemm_tilerelease() : () -> () - memref.dealloc %alloc_1 : memref<32x32xf32> - } - } - return - } -} - -// CHECK-LABEL: multi_level_conflict - -// CHECK: scf.for %arg0 = %c0 to %c4 step %c1 -// CHECK-NEXT: scf.for %arg1 = %c0 to %c8 step %c1 - -// CHECK: call @dnnl_brgemm_tileconfig(%[[A:.+]]) : (i64) -> () -// CHECK: call @dnnl_brgemm_execute([[B:.+]]) : ([[C:.+]]) -> () -// CHECK-NOT: call @dnnl_brgemm_tilerelease() : () -> () - -// CHECK: call @dnnl_brgemm_tileconfig(%[[D:.+]]) : (i64) -> () -// CHECK: call @dnnl_brgemm_execute([[E:.+]]) : ([[F:.+]]) -> () -// CHECK-NOT: call @dnnl_brgemm_tilerelease() : () -> () -// CHECK: call @dnnl_brgemm_tilerelease() : () -> () -// CHECK-NEXT: return - -// ----- - -module { - llvm.mlir.global_ctors {ctors = [@g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_1024_1024_2_2_ctor], priorities = [2147483647 : i32]} - llvm.mlir.global internal @g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_1024_1024_2_2() {addr_space = 0 : i32} : i64 - llvm.func @g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_1024_1024_2_2_ctor() -> i64 { - %0 = llvm.mlir.addressof @g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_1024_1024_2_2 : !llvm.ptr - %c32_i64 = arith.constant 32 : i64 - %c1024_i64 = arith.constant 1024 : i64 - %cst = arith.constant 1.000000e+00 : f32 - %c2_i64 = arith.constant 2 : i64 - %1 = func.call @dnnl_brgemm_dispatch(%c32_i64, %c32_i64, %c32_i64, %c32_i64, %c32_i64, %c32_i64, %c1024_i64, %c1024_i64, %cst, %c2_i64, %c2_i64) : (i64, i64, i64, i64, i64, i64, i64, i64, f32, i64, i64) -> i64 - llvm.store %1, %0 : i64, !llvm.ptr - llvm.return %1 : i64 - } - func.func private @dnnl_brgemm_tilerelease() - func.func private @dnnl_brgemm_execute(i64, !llvm.ptr, index, !llvm.ptr, index, !llvm.ptr, index, i64) - func.func private @dnnl_brgemm_tileconfig(i64) - func.func private @dnnl_brgemm_dispatch(i64, i64, i64, i64, i64, i64, i64, i64, f32, i64, i64) -> i64 - func.func @multi_level_partial_hoist() { - %cst = arith.constant 0.000000e+00 : f32 - %c16_i64 = arith.constant 16 : i64 - %c8 = arith.constant 8 : index - %c4 = arith.constant 4 : index - %c1 = arith.constant 1 : index - %c0 = arith.constant 0 : index - %c2_i64 = arith.constant 2 : i64 - %0 = llvm.mlir.addressof @g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_1024_1024_2_2 : !llvm.ptr - %1 = llvm.load %0 : !llvm.ptr -> i64 - %alloc = memref.alloc() {alignment = 64 : i64} : memref<4x16x32x32xbf16> - %alloc_0 = memref.alloc() {alignment = 64 : i64} : memref<8x16x16x32x2xbf16> - scf.for %arg0 = %c0 to %c4 step %c1 { - scf.for %arg1 = %c0 to %c8 step %c1 { - %alloc_1 = memref.alloc() {alignment = 64 : i64} : memref<32x32xf32> - %intptr = memref.extract_aligned_pointer_as_index %alloc_1 : memref<32x32xf32> -> index - %2 = arith.index_cast %intptr : index to i64 - %3 = llvm.inttoptr %2 : i64 to !llvm.ptr - linalg.fill ins(%cst : f32) outs(%alloc_1 : memref<32x32xf32>) - %subview = memref.subview %alloc[%arg0, 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : memref<4x16x32x32xbf16> to memref<16x32x32xbf16, strided<[1024, 32, 1], offset: ?>> - %base_buffer, %offset, %sizes:3, %strides:3 = memref.extract_strided_metadata %subview : memref<16x32x32xbf16, strided<[1024, 32, 1], offset: ?>> -> memref, index, index, index, index, index, index, index - %intptr_2 = memref.extract_aligned_pointer_as_index %subview : memref<16x32x32xbf16, strided<[1024, 32, 1], offset: ?>> -> index - %4 = arith.index_cast %intptr_2 : index to i64 - %5 = llvm.inttoptr %4 : i64 to !llvm.ptr - %subview_3 = memref.subview %alloc_0[%arg1, 0, 0, 0, 0] [1, 16, 16, 32, 2] [1, 1, 1, 1, 1] : memref<8x16x16x32x2xbf16> to memref<16x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>> - %base_buffer_4, %offset_5, %sizes_6:4, %strides_7:4 = memref.extract_strided_metadata %subview_3 : memref<16x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>> -> memref, index, index, index, index, index, index, index, index, index - %intptr_8 = memref.extract_aligned_pointer_as_index %subview_3 : memref<16x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>> -> index - %6 = arith.index_cast %intptr_8 : index to i64 - %7 = llvm.inttoptr %6 : i64 to !llvm.ptr - func.call @dnnl_brgemm_tileconfig(%1) : (i64) -> () - func.call @dnnl_brgemm_execute(%1, %5, %offset, %7, %offset_5, %3, %c0, %c16_i64) : (i64, !llvm.ptr, index, !llvm.ptr, index, !llvm.ptr, index, i64) -> () - func.call @dnnl_brgemm_tilerelease() : () -> () - memref.dealloc %alloc_1 : memref<32x32xf32> - } - scf.for %arg1 = %c0 to %c4 step %c1 { - %alloc_1 = memref.alloc() {alignment = 64 : i64} : memref<32x32xf32> - %intptr = memref.extract_aligned_pointer_as_index %alloc_1 : memref<32x32xf32> -> index - %2 = arith.index_cast %intptr : index to i64 - %3 = llvm.inttoptr %2 : i64 to !llvm.ptr - linalg.fill ins(%cst : f32) outs(%alloc_1 : memref<32x32xf32>) - %subview = memref.subview %alloc[%arg1, 0, 0, 0] [1, 2, 32, 32] [1, 1, 1, 1] : memref<4x16x32x32xbf16> to memref<2x32x32xbf16, strided<[1024, 32, 1], offset: ?>> - %base_buffer, %offset, %sizes:3, %strides:3 = memref.extract_strided_metadata %subview : memref<2x32x32xbf16, strided<[1024, 32, 1], offset: ?>> -> memref, index, index, index, index, index, index, index - %intptr_2 = memref.extract_aligned_pointer_as_index %subview : memref<2x32x32xbf16, strided<[1024, 32, 1], offset: ?>> -> index - %4 = arith.index_cast %intptr_2 : index to i64 - %5 = llvm.inttoptr %4 : i64 to !llvm.ptr - %subview_3 = memref.subview %alloc_0[%arg1, 0, 0, 0, 0] [1, 2, 16, 32, 2] [1, 1, 1, 1, 1] : memref<8x16x16x32x2xbf16> to memref<2x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>> - %base_buffer_4, %offset_5, %sizes_6:4, %strides_7:4 = memref.extract_strided_metadata %subview_3 : memref<2x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>> -> memref, index, index, index, index, index, index, index, index, index - %intptr_8 = memref.extract_aligned_pointer_as_index %subview_3 : memref<2x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>> -> index - %6 = arith.index_cast %intptr_8 : index to i64 - %7 = llvm.inttoptr %6 : i64 to !llvm.ptr - func.call @dnnl_brgemm_tileconfig(%1) : (i64) -> () - func.call @dnnl_brgemm_execute(%1, %5, %offset, %7, %offset_5, %3, %c0, %c2_i64) : (i64, !llvm.ptr, index, !llvm.ptr, index, !llvm.ptr, index, i64) -> () - func.call @dnnl_brgemm_tilerelease() : () -> () - } - } - return - } -} - -// CHECK-LABEL: multi_level_partial_hoist - -// CHECK: scf.for %arg0 = %c0 to %c4 step %c1 -// CHECK-NEXT: call @dnnl_brgemm_tileconfig(%[[A:.+]]) : (i64) -> () -// CHECK: scf.for %arg1 = %c0 to %c8 step %c1 -// CHECK: call @dnnl_brgemm_execute([[B:.+]]) : ([[C:.+]]) -> () - -// CHECK: } -// CHECK-NEXT: call @dnnl_brgemm_tileconfig(%[[D:.+]]) : (i64) -> () -// CHECK-NEXT: scf.for %arg1 = %c0 to %c4 step %c1 -// CHECK: call @dnnl_brgemm_execute([[E:.+]]) : ([[F:.+]]) -> () - -// CHECK: call @dnnl_brgemm_tilerelease() : () -> () -// CHECK-NEXT: return - -// ----- - -module { - llvm.mlir.global_ctors {ctors = [@g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_1024_1024_2_2_ctor], priorities = [2147483647 : i32]} - llvm.mlir.global internal @g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_1024_1024_2_2() {addr_space = 0 : i32} : i64 - llvm.func @g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_1024_1024_2_2_ctor() -> i64 { - %0 = llvm.mlir.addressof @g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_1024_1024_2_2 : !llvm.ptr - %c32_i64 = arith.constant 32 : i64 - %c1024_i64 = arith.constant 1024 : i64 - %cst = arith.constant 1.000000e+00 : f32 - %c2_i64 = arith.constant 2 : i64 - %1 = func.call @dnnl_brgemm_dispatch(%c32_i64, %c32_i64, %c32_i64, %c32_i64, %c32_i64, %c32_i64, %c1024_i64, %c1024_i64, %cst, %c2_i64, %c2_i64) : (i64, i64, i64, i64, i64, i64, i64, i64, f32, i64, i64) -> i64 - llvm.store %1, %0 : i64, !llvm.ptr - llvm.return %1 : i64 - } - func.func private @dnnl_brgemm_tilerelease() - func.func private @dnnl_brgemm_execute(i64, !llvm.ptr, index, !llvm.ptr, index, !llvm.ptr, index, i64) - func.func private @dnnl_brgemm_tileconfig(i64) - func.func private @dnnl_brgemm_dispatch(i64, i64, i64, i64, i64, i64, i64, i64, f32, i64, i64) -> i64 - func.func @multi_level_full_hoist() { - %cst = arith.constant 0.000000e+00 : f32 - %c16_i64 = arith.constant 16 : i64 - %c8 = arith.constant 8 : index - %c4 = arith.constant 4 : index - %c1 = arith.constant 1 : index - %c0 = arith.constant 0 : index - %0 = llvm.mlir.addressof @g_dispatched_microkernel_brgemm_stride_32_32_32_32_32_32_1024_1024_2_2 : !llvm.ptr - %1 = llvm.load %0 : !llvm.ptr -> i64 - %alloc = memref.alloc() {alignment = 64 : i64} : memref<4x16x32x32xbf16> - %alloc_0 = memref.alloc() {alignment = 64 : i64} : memref<8x16x16x32x2xbf16> - scf.for %arg0 = %c0 to %c4 step %c1 { - scf.for %arg1 = %c0 to %c8 step %c1 { - %alloc_1 = memref.alloc() {alignment = 64 : i64} : memref<32x32xf32> - %intptr = memref.extract_aligned_pointer_as_index %alloc_1 : memref<32x32xf32> -> index - %2 = arith.index_cast %intptr : index to i64 - %3 = llvm.inttoptr %2 : i64 to !llvm.ptr - linalg.fill ins(%cst : f32) outs(%alloc_1 : memref<32x32xf32>) - %subview = memref.subview %alloc[%arg0, 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : memref<4x16x32x32xbf16> to memref<16x32x32xbf16, strided<[1024, 32, 1], offset: ?>> - %base_buffer, %offset, %sizes:3, %strides:3 = memref.extract_strided_metadata %subview : memref<16x32x32xbf16, strided<[1024, 32, 1], offset: ?>> -> memref, index, index, index, index, index, index, index - %intptr_2 = memref.extract_aligned_pointer_as_index %subview : memref<16x32x32xbf16, strided<[1024, 32, 1], offset: ?>> -> index - %4 = arith.index_cast %intptr_2 : index to i64 - %5 = llvm.inttoptr %4 : i64 to !llvm.ptr - %subview_3 = memref.subview %alloc_0[%arg1, 0, 0, 0, 0] [1, 16, 16, 32, 2] [1, 1, 1, 1, 1] : memref<8x16x16x32x2xbf16> to memref<16x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>> - %base_buffer_4, %offset_5, %sizes_6:4, %strides_7:4 = memref.extract_strided_metadata %subview_3 : memref<16x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>> -> memref, index, index, index, index, index, index, index, index, index - %intptr_8 = memref.extract_aligned_pointer_as_index %subview_3 : memref<16x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>> -> index - %6 = arith.index_cast %intptr_8 : index to i64 - %7 = llvm.inttoptr %6 : i64 to !llvm.ptr - func.call @dnnl_brgemm_tileconfig(%1) : (i64) -> () - func.call @dnnl_brgemm_execute(%1, %5, %offset, %7, %offset_5, %3, %c0, %c16_i64) : (i64, !llvm.ptr, index, !llvm.ptr, index, !llvm.ptr, index, i64) -> () - func.call @dnnl_brgemm_tilerelease() : () -> () - memref.dealloc %alloc_1 : memref<32x32xf32> - } - } - return - } -} - -// CHECK-LABEL: multi_level_full_hoist - -// CHECK: call @dnnl_brgemm_tileconfig(%[[A:.+]]) : (i64) -> () -// CHECK-NEXT: scf.for %arg0 = %c0 to %c4 step %c1 -// CHECK-NEXT: scf.for %arg1 = %c0 to %c8 step %c1 - -// CHECK: call @dnnl_brgemm_execute([[B:.+]]) : ([[C:.+]]) -> () - -// CHECK: call @dnnl_brgemm_tilerelease() : () -> () -// CHECK-NEXT: return - -// ----- diff --git a/test/mlir/test/gc/Dialect/Microkernel/microkernel-to-dnnl-func.mlir b/test/mlir/test/gc/Dialect/Microkernel/microkernel-to-dnnl-func.mlir deleted file mode 100644 index ad5c3b84c..000000000 --- a/test/mlir/test/gc/Dialect/Microkernel/microkernel-to-dnnl-func.mlir +++ /dev/null @@ -1,70 +0,0 @@ -// RUN: gc-opt %s -convert-microkernel-to-dnnl-func -cse -split-input-file | FileCheck %s - -#map = affine_map<(d0, d1) -> (d0, d1)> -module { - func.func @basic_convert() { - %c0_i64 = arith.constant 0 : i64 - %c16_i64 = arith.constant 16 : i64 - %cst = arith.constant 0.000000e+00 : f32 - %alloc = memref.alloc() {alignment = 64 : i64} : memref<4x16x32x32xf32> - %alloc_0 = memref.alloc() {alignment = 64 : i64} : memref<8x16x32x32xf32> - %alloc_1 = memref.alloc() {alignment = 64 : i64} : memref<4x8x32x32xf32> - %alloc_2 = memref.alloc() {alignment = 64 : i64} : memref<4x8x32x32xf32> - scf.forall (%arg0, %arg1) in (4, 8) { - %alloc_3 = memref.alloc() {alignment = 64 : i64} : memref<32x32xf32> - linalg.fill ins(%cst : f32) outs(%alloc_3 : memref<32x32xf32>) - %subview = memref.subview %alloc[%arg0, 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : memref<4x16x32x32xf32> to memref<16x32x32xf32, strided<[1024, 32, 1], offset: ?>> - %subview_4 = memref.subview %alloc_0[%arg1, 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : memref<8x16x32x32xf32> to memref<16x32x32xf32, strided<[1024, 32, 1], offset: ?>> - %0 = microkernel.brgemm.dispatch [32, 32, 32, 32, 32, 32, 1024, 1024] flags(stride) data_type(f32, f32) - microkernel.brgemm.prologue(%0) : (i64) -> () - microkernel.brgemm.execute(%0, %subview, %subview_4, %alloc_3, %c16_i64, %c0_i64) : (i64, memref<16x32x32xf32, strided<[1024, 32, 1], offset: ?>>, memref<16x32x32xf32, strided<[1024, 32, 1], offset: ?>>, memref<32x32xf32>, i64, i64) -> () - microkernel.brgemm.epilogue(%0) : (i64) -> () - %subview_5 = memref.subview %alloc_1[%arg0, %arg1, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : memref<4x8x32x32xf32> to memref<32x32xf32, strided<[32, 1], offset: ?>> - linalg.generic {indexing_maps = [#map, #map, #map], iterator_types = ["parallel", "parallel"]} ins(%alloc_3, %subview_5 : memref<32x32xf32>, memref<32x32xf32, strided<[32, 1], offset: ?>>) outs(%alloc_3 : memref<32x32xf32>) { - ^bb0(%in: f32, %in_7: f32, %out: f32): - %1 = arith.addf %in, %in_7 : f32 - linalg.yield %1 : f32 - } - %subview_6 = memref.subview %alloc_2[%arg0, %arg1, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : memref<4x8x32x32xf32> to memref<32x32xf32, strided<[32, 1], offset: ?>> - linalg.generic {indexing_maps = [#map, #map], iterator_types = ["parallel", "parallel"]} ins(%alloc_3 : memref<32x32xf32>) outs(%subview_6 : memref<32x32xf32, strided<[32, 1], offset: ?>>) { - ^bb0(%in: f32, %out: f32): - %1 = arith.maximumf %in, %cst : f32 - linalg.yield %1 : f32 - } - memref.dealloc %alloc_3 : memref<32x32xf32> - } - return - } -} - -// CHECK-LABEL: dnnl_brgemm_execute -// CHECK-LABEL: dnnl_brgemm_dispatch -// CHECK-LABEL: basic_convert -// CHECK: %[[CST3:.+]] = arith.constant 3 : i64 -// CHECK: %[[CST1F:.+]] = arith.constant 1.000000e+00 : f32 -// CHECK: %[[CST1024:.+]] = arith.constant 1024 : i64 -// CHECK: %[[CST32:.+]] = arith.constant 32 : i64 -// CHECK: %[[CST0:.+]] = arith.constant 0 : index -// CHECK: %[[CST16:.+]] = arith.constant 16 : i64 - -// CHECK: %[[ptrC:.+]] = memref.extract_aligned_pointer_as_index %[[memrefC:.+]] : memref<32x32xf32> -> index -// CHECK-NEXT: %[[idxC:.+]] = arith.index_cast %[[ptrC]] : index to i64 -// CHECK-NEXT: %[[llvmptrC:.+]] = llvm.inttoptr %[[idxC]] : i64 to !llvm.ptr - -// CHECK: %[[bbA:.+]], %[[offA:.+]], %[[szA:.+]]:3, %[[strdA:.+]]:3 = memref.extract_strided_metadata %[[memrefA:.+]] : memref<16x32x32xf32, strided<[1024, 32, 1], offset: ?>> -> memref, index, index, index, index, index, index, index -// CHECK-NEXT: %[[ptrA:.+]] = memref.extract_aligned_pointer_as_index %[[memrefA]] : memref<16x32x32xf32, strided<[1024, 32, 1], offset: ?>> -> index -// CHECK-NEXT: %[[idxA:.+]] = arith.index_cast %[[ptrA]] : index to i64 -// CHECK-NEXT: %[[llvmptrA:.+]] = llvm.inttoptr %[[idxA]] : i64 to !llvm.ptr - -// CHECK: %[[bbB:.+]], %[[offB:.+]], %[[szB:.+]]:3, %[[strdB:.+]]:3 = memref.extract_strided_metadata %[[memrefB:.+]] : memref<16x32x32xf32, strided<[1024, 32, 1], offset: ?>> -> memref, index, index, index, index, index, index, index -// CHECK-NEXT: %[[ptrB:.+]] = memref.extract_aligned_pointer_as_index %[[memrefB]] : memref<16x32x32xf32, strided<[1024, 32, 1], offset: ?>> -> index -// CHECK-NEXT: %[[idxB:.+]] = arith.index_cast %[[ptrB]] : index to i64 -// CHECK-NEXT: %[[llvmptrB:.+]] = llvm.inttoptr %[[idxB]] : i64 to !llvm.ptr - -// CHECK: %[[KERNEL:.+]] = func.call @dnnl_brgemm_dispatch(%[[CST32]], %[[CST32]], %[[CST32]], %[[CST32]], %[[CST32]], %[[CST32]], %[[CST1024]], %[[CST1024]], %[[CST1F]], %[[CST3]], %[[CST3]]) : (i64, i64, i64, i64, i64, i64, i64, i64, f32, i64, i64) -> i64 -// CHECK-NOT: microkernel.brgemm.prologue(%[[TMP:.+]]) : (i64) -> () - -// CHECK: func.call @dnnl_brgemm_execute(%[[KERNEL]], %[[llvmptrA]], %[[offA]], %[[llvmptrB]], %[[offB]], %[[llvmptrC]], %[[CST0]], %[[CST16]]) : (i64, !llvm.ptr, index, !llvm.ptr, index, !llvm.ptr, index, i64) -> () -// CHECK-NOT: microkernel.brgemm.epilogue(%[[KERNEL]]) : (i64) -> () - -// ----- diff --git a/test/mlir/test/gc/Dialect/OneDNNGraph/lit.local.cfg b/test/mlir/test/gc/Dialect/OneDNNGraph/lit.local.cfg deleted file mode 100644 index c56b0351c..000000000 --- a/test/mlir/test/gc/Dialect/OneDNNGraph/lit.local.cfg +++ /dev/null @@ -1,2 +0,0 @@ -if not config.gc_use_dnnl: - config.unsupported = True \ No newline at end of file diff --git a/test/mlir/test/gc/Dialect/OneDNNGraph/onednn-graph-llama2.mlir b/test/mlir/test/gc/Dialect/OneDNNGraph/onednn-graph-llama2.mlir deleted file mode 100644 index 732a92f69..000000000 --- a/test/mlir/test/gc/Dialect/OneDNNGraph/onednn-graph-llama2.mlir +++ /dev/null @@ -1,85 +0,0 @@ -// RUN: gc-opt %s -o=/dev/null 2>&1 -// gc-opt %s --gc-cpu-pipeline | gc-cpu-runner -e main -entry-point-result=void | FileCheck --allow-empty %s -// Disabled for now because CI does not have bf16 lib support, once we add our bf16 legalizer, we can enable - -func.func @llama2_mlp(%2: tensor<1x32x4096xbf16>, %3: tensor<4096x4096xbf16>, %1 : tensor<1x32x4096xbf16>, %00: tensor<1xf32>, - %26: tensor<4096xbf16>, %28: tensor<11008x4096xbf16>, %33: tensor<11008x4096xbf16>, %37: tensor<4096x11008xbf16>, - %0: tensor<1xf32>, %60: tensor<4096xbf16>) -> (tensor<1x32x4096xbf16>, tensor<1x32x4096xbf16>) { - %5 = onednn_graph.matmul %2, %3 {transpose_b = true} - : (tensor<1x32x4096xbf16>, tensor<4096x4096xbf16>) -> tensor<1x32x4096xbf16> - %7 = onednn_graph.add %1, %5 : (tensor<1x32x4096xbf16>, tensor<1x32x4096xbf16>) -> tensor<1x32x4096xbf16> - - %11 = onednn_graph.type_cast %7 : (tensor<1x32x4096xbf16>) -> tensor<1x32x4096xf32> - %13 = onednn_graph.pow %11 {beta = 2.0 : f32} : (tensor<1x32x4096xf32>) -> tensor<1x32x4096xf32> - %17 = onednn_graph.reduce_mean %13 {axes = array, keep_dims = true} - : (tensor<1x32x4096xf32>) -> tensor<1x32x1xf32> - %19 = onednn_graph.add %17, %0 : (tensor<1x32x1xf32>, tensor<1xf32>) -> tensor<1x32x1xf32> - %20 = onednn_graph.pow %19 {beta = -0.5 : f32} : (tensor<1x32x1xf32>) -> tensor<1x32x1xf32> - %21 = onednn_graph.mul %11, %20 : (tensor<1x32x4096xf32>, tensor<1x32x1xf32>) -> tensor<1x32x4096xf32> - %25 = onednn_graph.type_cast %21 : (tensor<1x32x4096xf32>) -> tensor<1x32x4096xbf16> - %27 = onednn_graph.mul %26, %25 : (tensor<4096xbf16>, tensor<1x32x4096xbf16>) -> tensor<1x32x4096xbf16> - - %30 = onednn_graph.matmul %27, %28 {transpose_b = true} - : (tensor<1x32x4096xbf16>, tensor<11008x4096xbf16>) -> tensor<1x32x11008xbf16> - %31 = onednn_graph.sigmoid %30 : (tensor<1x32x11008xbf16>) -> tensor<1x32x11008xbf16> - %32 = onednn_graph.mul %31, %30 : (tensor<1x32x11008xbf16>, tensor<1x32x11008xbf16>) -> tensor<1x32x11008xbf16> - %35 = onednn_graph.matmul %27, %33 {transpose_b = true} - : (tensor<1x32x4096xbf16>, tensor<11008x4096xbf16>) -> tensor<1x32x11008xbf16> - %36 = onednn_graph.mul %32, %35 : (tensor<1x32x11008xbf16>, tensor<1x32x11008xbf16>) -> tensor<1x32x11008xbf16> - %39 = onednn_graph.matmul %36, %37 {transpose_b = true} - : (tensor<1x32x11008xbf16>, tensor<4096x11008xbf16>) -> tensor<1x32x4096xbf16> - %41 = onednn_graph.add %7, %39 : (tensor<1x32x4096xbf16>, tensor<1x32x4096xbf16>) -> tensor<1x32x4096xbf16> - %45 = onednn_graph.type_cast %41 : (tensor<1x32x4096xbf16>) -> tensor<1x32x4096xf32> - - %47 = onednn_graph.pow %45 {beta = 2.0 : f32} : (tensor<1x32x4096xf32>) -> tensor<1x32x4096xf32> - %51 = onednn_graph.reduce_mean %47 {axes = array, keep_dims = true} - : (tensor<1x32x4096xf32>) -> tensor<1x32x1xf32> - %53 = onednn_graph.add %51, %0 : (tensor<1x32x1xf32>, tensor<1xf32>) -> tensor<1x32x1xf32> - %54 = onednn_graph.pow %53 {beta = -0.5 : f32} : (tensor<1x32x1xf32>) -> tensor<1x32x1xf32> - - %55 = onednn_graph.mul %45, %54 : (tensor<1x32x4096xf32>, tensor<1x32x1xf32>) -> tensor<1x32x4096xf32> - %59 = onednn_graph.type_cast %55 : (tensor<1x32x4096xf32>) -> tensor<1x32x4096xbf16> - %61 = onednn_graph.mul %60, %59 : (tensor<4096xbf16>, tensor<1x32x4096xbf16>) -> tensor<1x32x4096xbf16> - - return %61, %41 : tensor<1x32x4096xbf16>, tensor<1x32x4096xbf16> -} - -func.func @main() { - %cst0 = arith.constant 1.000000e+00 : bf16 - %cst1 = arith.constant 1.000000e+00 : f32 - - %e2 = tensor.empty() : tensor<1x32x4096xbf16> - %e3 = tensor.empty() : tensor<4096x4096xbf16> - %e1 = tensor.empty() : tensor<1x32x4096xbf16> - %e00 = tensor.empty() : tensor<1xf32> - %e26 = tensor.empty() : tensor<4096xbf16> - %e28 = tensor.empty() : tensor<11008x4096xbf16> - %e33 = tensor.empty() : tensor<11008x4096xbf16> - %e37 = tensor.empty() : tensor<4096x11008xbf16> - %e0 = tensor.empty() : tensor<1xf32> - %e60 = tensor.empty() : tensor<4096xbf16> - - %2 = linalg.fill ins(%cst0 : bf16) outs(%e2 : tensor<1x32x4096xbf16> ) -> tensor<1x32x4096xbf16> - %3 = linalg.fill ins(%cst0 : bf16) outs(%e3 : tensor<4096x4096xbf16> ) -> tensor<4096x4096xbf16> - %1 = linalg.fill ins(%cst0 : bf16) outs(%e1 : tensor<1x32x4096xbf16> ) -> tensor<1x32x4096xbf16> - %00 = linalg.fill ins(%cst1 : f32) outs(%e00 : tensor<1xf32> ) -> tensor<1xf32> - %26 = linalg.fill ins(%cst0 : bf16) outs(%e26 : tensor<4096xbf16> ) -> tensor<4096xbf16> - %28 = linalg.fill ins(%cst0 : bf16) outs(%e28 : tensor<11008x4096xbf16> ) -> tensor<11008x4096xbf16> - %33 = linalg.fill ins(%cst0 : bf16) outs(%e33 : tensor<11008x4096xbf16> ) -> tensor<11008x4096xbf16> - %37 = linalg.fill ins(%cst0 : bf16) outs(%e37 : tensor<4096x11008xbf16> ) -> tensor<4096x11008xbf16> - %0 = linalg.fill ins(%cst1 : f32) outs(%e0 : tensor<1xf32> ) -> tensor<1xf32> - %60 = linalg.fill ins(%cst0 : bf16) outs(%e60 : tensor<4096xbf16> ) -> tensor<4096xbf16> - - %61, %41 = func.call @llama2_mlp(%2, %3, %1, %00, %26, %28, %33, %37, %0, %60) : - (tensor<1x32x4096xbf16>, tensor<4096x4096xbf16>, tensor<1x32x4096xbf16>, tensor<1xf32>, tensor<4096xbf16>, - tensor<11008x4096xbf16>, tensor<11008x4096xbf16>, tensor<4096x11008xbf16>, tensor<1xf32>, tensor<4096xbf16>) - -> (tensor<1x32x4096xbf16>, tensor<1x32x4096xbf16>) - - %idx = arith.constant 0 : index - %ex = tensor.extract %61[%idx, %idx, %idx] : tensor<1x32x4096xbf16> - %ext = arith.extf %ex : bf16 to f32 - cpuruntime.printf "output[0, 0, 0]: %f\n" %ext : f32 - - return -} -// CHECK: output[0, 0, 0]: 1.0 diff --git a/test/mlir/test/gc/Dialect/OneDNNGraph/onednn-graph-mlp.mlir b/test/mlir/test/gc/Dialect/OneDNNGraph/onednn-graph-mlp.mlir deleted file mode 100644 index d10146d28..000000000 --- a/test/mlir/test/gc/Dialect/OneDNNGraph/onednn-graph-mlp.mlir +++ /dev/null @@ -1,60 +0,0 @@ -// RUN: gc-opt -split-input-file -verify-diagnostics %s | FileCheck %s - -// CHECK-LABEL: @mlp -func.func @mlp(%in: tensor<128x512xbf16>, - %weight0: tensor<512x64xbf16>, %bias0: tensor<64xbf16>, - %weight1: tensor<64x256xbf16>, %bias1: tensor<256xbf16>) -> tensor<128x256xbf16> { - // CHECK: [[MM1:%.+]] = onednn_graph.matmul - // CHECK: [[RL1:%.+]] = onednn_graph.relu [[MM1]] - // CHECK: [[MM2:%.+]] = onednn_graph.matmul - // CHECK: [[AD2:%.+]] = onednn_graph.add [[MM2]] - // CHECK: [[RL2:%.+]] = onednn_graph.relu [[AD2]] - // CHECK: return [[RL2]] - %0 = onednn_graph.matmul %in, %weight0, %bias0 - : (tensor<128x512xbf16>, tensor<512x64xbf16>, tensor<64xbf16>) -> tensor<128x64xbf16> - %1 = onednn_graph.relu %0 : (tensor<128x64xbf16>) -> tensor<128x64xbf16> - %2 = onednn_graph.matmul %1, %weight1 - : (tensor<128x64xbf16>, tensor<64x256xbf16>) -> tensor<128x256xbf16> - %3 = onednn_graph.add %2, %bias1 : (tensor<128x256xbf16>, tensor<256xbf16>) -> tensor<128x256xbf16> - %4 = onednn_graph.relu %3 : (tensor<128x256xbf16>) -> tensor<128x256xbf16> - return %4 : tensor<128x256xbf16> -} - -// CHECK-LABEL: @mlp_transpose_a -func.func @mlp_transpose_a(%in: tensor<512x128xbf16>, - %weight0: tensor<512x256xbf16>, %bias0: tensor<256xbf16>) -> tensor<128x256xbf16> { - // CHECK: [[MM1:%.+]] = onednn_graph.matmul - // CHECK: {transpose_a = true} - // CHECK-NEXT: [[RL1:%.+]] = onednn_graph.relu [[MM1]] - // CHECK-NEXT: return [[RL1]] - %0 = onednn_graph.matmul %in, %weight0, %bias0 {transpose_a = true} - : (tensor<512x128xbf16>, tensor<512x256xbf16>, tensor<256xbf16>) -> tensor<128x256xbf16> - %1 = onednn_graph.relu %0 : (tensor<128x256xbf16>) -> tensor<128x256xbf16> - return %1 : tensor<128x256xbf16> -} - -// CHECK-LABEL: @mlp_transpose_b -func.func @mlp_transpose_b(%in: tensor<128x512xbf16>, - %weight0: tensor<256x512xbf16>, %bias0: tensor<256xbf16>) -> tensor<128x256xbf16> { - // CHECK: [[MM1:%.+]] = onednn_graph.matmul - // CHECK: {transpose_b = true} - // CHECK-NEXT: [[RL1:%.+]] = onednn_graph.relu [[MM1]] - // CHECK-NEXT: return [[RL1]] - %0 = onednn_graph.matmul %in, %weight0, %bias0 {transpose_b = true} - : (tensor<128x512xbf16>, tensor<256x512xbf16>, tensor<256xbf16>) -> tensor<128x256xbf16> - %1 = onednn_graph.relu %0 : (tensor<128x256xbf16>) -> tensor<128x256xbf16> - return %1 : tensor<128x256xbf16> -} - -// CHECK-LABEL: @mlp_transpose_a_b -func.func @mlp_transpose_a_b(%in: tensor<512x128xbf16>, - %weight0: tensor<256x512xbf16>, %bias0: tensor<256xbf16>) -> tensor<128x256xbf16> { - // CHECK: [[MM1:%.+]] = onednn_graph.matmul - // CHECK: {transpose_a = true, transpose_b = true} - // CHECK-NEXT: [[RL1:%.+]] = onednn_graph.relu [[MM1]] - // CHECK-NEXT: return [[RL1]] - %0 = onednn_graph.matmul %in, %weight0, %bias0 {transpose_a = true, transpose_b = true} - : (tensor<512x128xbf16>, tensor<256x512xbf16>, tensor<256xbf16>) -> tensor<128x256xbf16> - %1 = onednn_graph.relu %0 : (tensor<128x256xbf16>) -> tensor<128x256xbf16> - return %1 : tensor<128x256xbf16> -} diff --git a/test/mlir/test/gc/Dialect/OneDNNGraph/onednn-graph-ops.mlir b/test/mlir/test/gc/Dialect/OneDNNGraph/onednn-graph-ops.mlir deleted file mode 100644 index 32ab25e0a..000000000 --- a/test/mlir/test/gc/Dialect/OneDNNGraph/onednn-graph-ops.mlir +++ /dev/null @@ -1,108 +0,0 @@ -// RUN: gc-opt -split-input-file -canonicalize -verify-diagnostics %s | FileCheck %s - -//// Matmul - -// CHECK-LABEL: @matmul -func.func @matmul(%arg0: tensor<128x512xbf16>, %arg1: tensor<512x64xbf16>, - %arg2: tensor<64xbf16>) -> tensor<128x64xbf16> { - // CHECK: onednn_graph.matmul - %0 = onednn_graph.matmul %arg0, %arg1, %arg2 - : (tensor<128x512xbf16>, tensor<512x64xbf16>, tensor<64xbf16>) -> tensor<128x64xbf16> - return %0 : tensor<128x64xbf16> -} - -// CHECK-LABEL: @matmul_broadcast_batch -func.func @matmul_broadcast_batch(%arg0: tensor<6x128x512xbf16>, %arg1: tensor<4x1x512x64xbf16>, - %arg2: tensor<64xbf16>) -> tensor<4x6x128x64xbf16> { - // CHECK: onednn_graph.matmul - %0 = onednn_graph.matmul %arg0, %arg1, %arg2 - : (tensor<6x128x512xbf16>, tensor<4x1x512x64xbf16>, tensor<64xbf16>) -> tensor<4x6x128x64xbf16> - return %0 : tensor<4x6x128x64xbf16> -} - -//// Unary - -// CHECK-LABEL: @relu -func.func @relu(%arg0: tensor<128x512xbf16>) -> tensor<128x512xbf16> { - // CHECK: onednn_graph.relu - %0 = onednn_graph.relu %arg0 : (tensor<128x512xbf16>) -> tensor<128x512xbf16> - return %0 : tensor<128x512xbf16> -} - -// CHECK-LABEL: @sigmoid -func.func @sigmoid(%arg0: tensor<128x512xbf16>) -> tensor<128x512xbf16> { - // CHECK: onednn_graph.sigmoid - %0 = onednn_graph.sigmoid %arg0 : (tensor<128x512xbf16>) -> tensor<128x512xbf16> - return %0 : tensor<128x512xbf16> -} - -// CHECK-LABEL: @type_cast -func.func @type_cast(%arg0: tensor<128x512xbf16>) -> tensor<128x512xf32> { - // CHECK: onednn_graph.type_cast - %0 = onednn_graph.type_cast %arg0 : (tensor<128x512xbf16>) -> tensor<128x512xf32> - return %0 : tensor<128x512xf32> -} - -// CHECK-LABEL: @pow -func.func @pow(%arg0: tensor<128x512xbf16>) -> tensor<128x512xbf16> { - // CHECK: onednn_graph.pow - // CHECK-SAME: beta = 2 - %0 = onednn_graph.pow %arg0 {beta = 2.0 : f32} : (tensor<128x512xbf16>) -> tensor<128x512xbf16> - return %0 : tensor<128x512xbf16> -} - -//// Binary - -// CHECK-LABEL: @add -func.func @add(%arg0: tensor<512x64xbf16>, %arg1: tensor<512x64xbf16>) -> tensor<512x64xbf16> { - // CHECK: onednn_graph.add - %0 = onednn_graph.add %arg0, %arg1 : (tensor<512x64xbf16>, tensor<512x64xbf16>) -> tensor<512x64xbf16> - return %0 : tensor<512x64xbf16> -} - -// CHECK-LABEL: @mul -func.func @mul(%arg0: tensor<512x64xbf16>, %arg1: tensor<64xbf16>) -> tensor<512x64xbf16> { - // CHECK: onednn_graph.mul - %0 = onednn_graph.mul %arg0, %arg1 : (tensor<512x64xbf16>, tensor<64xbf16>) -> tensor<512x64xbf16> - return %0 : tensor<512x64xbf16> -} - -// CHECK-LABEL: @sub -func.func @sub(%arg0: tensor<512x64xbf16>, %arg1: tensor<512x64xbf16>) -> tensor<512x64xbf16> { - // CHECK: onednn_graph.sub - %0 = onednn_graph.sub %arg0, %arg1 : (tensor<512x64xbf16>, tensor<512x64xbf16>) -> tensor<512x64xbf16> - return %0 : tensor<512x64xbf16> -} - -// CHECK-LABEL: @div -func.func @div(%arg0: tensor<512x64xbf16>, %arg1: tensor<64xbf16>) -> tensor<512x64xbf16> { - // CHECK: onednn_graph.div - %0 = onednn_graph.div %arg0, %arg1 : (tensor<512x64xbf16>, tensor<64xbf16>) -> tensor<512x64xbf16> - return %0 : tensor<512x64xbf16> -} - -//// Reduce - -// CHECK-LABEL: @reduce_sum_keep_dims -func.func @reduce_sum_keep_dims(%arg0: tensor<64x128x512xbf16>) -> tensor<64x1x512xbf16> { - // CHECK: onednn_graph.reduce_sum - // CHECK-SAME: axes = array - // CHECK-SAME: keep_dims = true - %0 = onednn_graph.reduce_sum %arg0 {axes = array, keep_dims = true} : (tensor<64x128x512xbf16>) -> tensor<64x1x512xbf16> - return %0 : tensor<64x1x512xbf16> -} - -// CHECK-LABEL: @reduce_sum_no_axis -func.func @reduce_sum_no_axis(%arg0: tensor<64x128x512xbf16>) -> tensor<64x128x512xbf16> { - // CHECK: onednn_graph.reduce_sum - %0 = onednn_graph.reduce_sum %arg0 : (tensor<64x128x512xbf16>) -> tensor<64x128x512xbf16> - return %0 : tensor<64x128x512xbf16> -} - -// CHECK-LABEL: @reduce_mean_no_keep_dims -func.func @reduce_mean_no_keep_dims(%arg0: tensor<64x128x512xbf16>) -> tensor<128xbf16> { - // CHECK: onednn_graph.reduce_mean - // CHECK-SAME: axes = array - %0 = onednn_graph.reduce_mean %arg0 {axes = array} : (tensor<64x128x512xbf16>) -> tensor<128xbf16> - return %0 : tensor<128xbf16> -} diff --git a/test/mlir/test/gc/Dialect/OneDNNGraph/onednn-graph-to-linalg.mlir b/test/mlir/test/gc/Dialect/OneDNNGraph/onednn-graph-to-linalg.mlir deleted file mode 100644 index b6f796cdb..000000000 --- a/test/mlir/test/gc/Dialect/OneDNNGraph/onednn-graph-to-linalg.mlir +++ /dev/null @@ -1,240 +0,0 @@ -// RUN: gc-opt --split-input-file --convert-onednn-graph-to-linalg %s -verify-diagnostics -o -| FileCheck %s - -// CHECK-LABEL: @matmul -func.func @matmul(%arg0: tensor<128x512xbf16>, %arg1: tensor<512x256xbf16>) -> tensor<128x256xbf16> { - // CHECK: [[C0:%.+]] = arith.constant 0 - // CHECK: [[INIT:%.+]] = tensor.empty() - // CHECK: [[FILLED:%.+]] = linalg.fill ins([[C0]] : bf16) outs([[INIT]] : tensor<128x256xbf16>) -> tensor<128x256xbf16> - // CHECK: linalg.matmul ins(%arg0, %arg1 : tensor<128x512xbf16>, tensor<512x256xbf16>) outs([[FILLED]] : tensor<128x256xbf16>) -> tensor<128x256xbf16> - %0 = onednn_graph.matmul %arg0, %arg1 : (tensor<128x512xbf16>, tensor<512x256xbf16>) -> tensor<128x256xbf16> - return %0 : tensor<128x256xbf16> -} - -// CHECK-LABEL: @matmul_bias -func.func @matmul_bias(%arg0: tensor<128x512xbf16>, %arg1: tensor<512x256xbf16>, %arg3: tensor<256xbf16>) -> tensor<128x256xbf16> { - // CHECK: [[C0:%.+]] = arith.constant 0 - // CHECK: [[INIT:%.+]] = tensor.empty() - // CHECK: [[FILLED:%.+]] = linalg.fill ins([[C0]] : bf16) outs([[INIT]] : tensor<128x256xbf16>) -> tensor<128x256xbf16> - // CHECK: linalg.matmul ins(%arg0, %arg1 : tensor<128x512xbf16>, tensor<512x256xbf16>) outs([[FILLED]] : tensor<128x256xbf16>) -> tensor<128x256xbf16> - // CHECK: tensor.empty() - // CHECK: linalg.broadcast - // CHECK: tensor.empty() - // CHECK: linalg.add - %0 = onednn_graph.matmul %arg0, %arg1, %arg3 : (tensor<128x512xbf16>, tensor<512x256xbf16>, tensor<256xbf16>) -> tensor<128x256xbf16> - return %0 : tensor<128x256xbf16> -} - -// CHECK-LABEL: @matmul_batch_flatten -func.func @matmul_batch_flatten(%arg0: tensor<6x128x512xbf16>, %arg1: tensor<512x256xbf16>) -> tensor<6x128x256xbf16> { - // CHECK: tensor.collapse_shape - // CHECK-SAME: tensor<6x128x512xbf16> into tensor<768x512xbf16> - // CHECK: arith.constant 0 - // CHECK: tensor.empty() - // CHECK: linalg.fill - // CHECK: linalg.matmul - // CHECK-SAME: tensor<768x512xbf16>, tensor<512x256xbf16> - // CHECK-SAME: tensor<768x256xbf16> - // CHECK: tensor.expand_shape - // CHECK-SAME: tensor<768x256xbf16> into tensor<6x128x256xbf16> - %0 = onednn_graph.matmul %arg0, %arg1 : (tensor<6x128x512xbf16>, tensor<512x256xbf16>) -> tensor<6x128x256xbf16> - return %0 : tensor<6x128x256xbf16> -} - -// CHECK-LABEL: @matmul_multi_batch_flatten -func.func @matmul_multi_batch_flatten(%arg0: tensor<1x2x3x128x512xbf16>, %arg1: tensor<512x256xbf16>) -> tensor<1x2x3x128x256xbf16> { - // CHECK: tensor.collapse_shape - // CHECK-SAME: tensor<1x2x3x128x512xbf16> into tensor<768x512xbf16> - // CHECK: arith.constant 0 - // CHECK: tensor.empty() - // CHECK: linalg.fill - // CHECK: linalg.matmul - // CHECK-SAME: tensor<768x512xbf16>, tensor<512x256xbf16> - // CHECK-SAME: tensor<768x256xbf16> - // CHECK: tensor.expand_shape - // CHECK-SAME: tensor<768x256xbf16> into tensor<1x2x3x128x256xbf16> - %0 = onednn_graph.matmul %arg0, %arg1 : (tensor<1x2x3x128x512xbf16>, tensor<512x256xbf16>) -> tensor<1x2x3x128x256xbf16> - return %0 : tensor<1x2x3x128x256xbf16> -} - -// CHECK-LABEL: @add -func.func @add(%arg0: tensor<128x256xf32>, %arg1: tensor<128x256xf32>) -> tensor<128x256xf32> { - // CHECK: tensor.empty() - // CHECK: linalg.add - %0 = onednn_graph.add %arg0, %arg1 : (tensor<128x256xf32>, tensor<128x256xf32>) -> tensor<128x256xf32> - return %0 : tensor<128x256xf32> -} - -// CHECK-LABEL: @mul -func.func @mul(%arg0: tensor<512x64xbf16>, %arg1: tensor<64xbf16>) -> tensor<512x64xbf16> { - // CHECK: tensor.empty() - // CHECK: linalg.mul - %0 = onednn_graph.mul %arg0, %arg1 : (tensor<512x64xbf16>, tensor<64xbf16>) -> tensor<512x64xbf16> - return %0 : tensor<512x64xbf16> -} - -// CHECK-LABEL: @sub -func.func @sub(%arg0: tensor<512x64xbf16>, %arg1: tensor<512x64xbf16>) -> tensor<512x64xbf16> { - // CHECK: tensor.empty() - // CHECK: linalg.sub - %0 = onednn_graph.sub %arg0, %arg1 : (tensor<512x64xbf16>, tensor<512x64xbf16>) -> tensor<512x64xbf16> - return %0 : tensor<512x64xbf16> -} - -// CHECK-LABEL: @div -func.func @div(%arg0: tensor<512x64xbf16>, %arg1: tensor<64xbf16>) -> tensor<512x64xbf16> { - // CHECK: tensor.empty() - // CHECK: linalg.div - %0 = onednn_graph.div %arg0, %arg1 : (tensor<512x64xbf16>, tensor<64xbf16>) -> tensor<512x64xbf16> - return %0 : tensor<512x64xbf16> -} - -// CHECK-LABEL: @add_bcast -func.func @add_bcast(%arg0: tensor<128x256xf32>, %arg1: tensor<256xf32>) -> tensor<128x256xf32> { - // CHECK: tensor.empty() - // CHECK: linalg.broadcast - // CHECK: tensor.empty() - // CHECK: linalg.add - %0 = onednn_graph.add %arg0, %arg1 : (tensor<128x256xf32>, tensor<256xf32>) -> tensor<128x256xf32> - return %0 : tensor<128x256xf32> -} - -// CHECK-LABEL: @mul_bacst -func.func @mul_bacst(%arg0: tensor<512x64xbf16>, %arg1: tensor<1x64xbf16>) -> tensor<512x64xbf16> { - // CHECK: tensor.collapse_shape - // CHECK-SAME: tensor<1x64xbf16> into tensor<64xbf16> - // CHECK: tensor.empty() - // CHECK: linalg.broadcast - // CHECK: tensor.empty() - // CHECK: linalg.mul - %0 = onednn_graph.mul %arg0, %arg1 : (tensor<512x64xbf16>, tensor<1x64xbf16>) -> tensor<512x64xbf16> - return %0 : tensor<512x64xbf16> -} - -// CHECK-LABEL: @relu -func.func @relu(%arg0: tensor<128x256xbf16>) -> tensor<128x256xbf16> { - // CHECK: arith.constant dense<0.0{{.*}}> - // CHECK: tensor.empty() - // CHECK: linalg.max - %0 = onednn_graph.relu %arg0 : (tensor<128x256xbf16>) -> tensor<128x256xbf16> - return %0 : tensor<128x256xbf16> -} - -// CHECK-LABEL: @sigmoid -func.func @sigmoid(%arg0: tensor<128x512xbf16>) -> tensor<128x512xbf16> { - // CHECK: tensor.empty() - // CHECK: linalgx.sigmoid - %0 = onednn_graph.sigmoid %arg0 : (tensor<128x512xbf16>) -> tensor<128x512xbf16> - return %0 : tensor<128x512xbf16> -} - -// CHECK-LABEL: @type_cast -func.func @type_cast(%arg0: tensor<128x512xbf16>) -> tensor<128x512xf32> { - // CHECK: tensor.empty() - // CHECK: linalg.copy - %0 = onednn_graph.type_cast %arg0 : (tensor<128x512xbf16>) -> tensor<128x512xf32> - return %0 : tensor<128x512xf32> -} - -// CHECK-LABEL: @pow -func.func @pow(%arg0: tensor<128x512xbf16>) -> tensor<128x512xbf16> { - // CHECK: arith.constant dense<2.0{{.*}}> - // CHECK: tensor.empty() - // CHECK: linalg.powf - %0 = onednn_graph.pow %arg0 {beta = 2.0 : f32} : (tensor<128x512xbf16>) -> tensor<128x512xbf16> - return %0 : tensor<128x512xbf16> -} - - -// CHECK-LABEL: @reduce_sum_keep_dims -func.func @reduce_sum_keep_dims(%arg0: tensor<64x128x512xbf16>) -> tensor<64x1x512xbf16> { - // CHECK: arith.constant 0 - // CHECK: tensor.empty() - // CHECK: linalg.fill - // CHECK: linalg.reduce - // CHECK: arith.addf - // CHECK: tensor.expand_shape - %0 = onednn_graph.reduce_sum %arg0 {axes = array, keep_dims = true} : (tensor<64x128x512xbf16>) -> tensor<64x1x512xbf16> - return %0 : tensor<64x1x512xbf16> -} - -// CHECK-LABEL: @reduce_sum_no_axis -func.func @reduce_sum_no_axis(%arg0: tensor<64x128x512xbf16>) -> tensor<64x128x512xbf16> { - // CHECK: return %arg0 - %0 = onednn_graph.reduce_sum %arg0 : (tensor<64x128x512xbf16>) -> tensor<64x128x512xbf16> - return %0 : tensor<64x128x512xbf16> -} - -// CHECK-LABEL: @reduce_mean_no_keep_dims -func.func @reduce_mean_no_keep_dims(%arg0: tensor<64x128x512xbf16>) -> tensor<128xbf16> { - // CHECK: arith.constant 0 - // CHECK: tensor.empty() - // CHECK: linalg.fill - // CHECK: linalg.reduce - // CHECK: arith.addf - // CHECK: arith.constant dense - // CHECK: tensor.empty() - // CHECK: linalg.div - %0 = onednn_graph.reduce_mean %arg0 {axes = array} : (tensor<64x128x512xbf16>) -> tensor<128xbf16> - return %0 : tensor<128xbf16> -} - -// CHECK-LABEL: @mlp_transpose_a -func.func @mlp_transpose_a(%arg0: tensor<512x128xbf16>, %arg1: tensor<512x256xbf16>, %arg3: tensor<256xbf16>) -> tensor<128x256xbf16> { - // CHECK: [[C0:%.+]] = arith.constant 0 - // CHECK: [[INIT:%.+]] = tensor.empty() - // CHECK: [[FILLED:%.+]] = linalg.fill ins([[C0]] : bf16) outs([[INIT]] : tensor<128x256xbf16>) -> tensor<128x256xbf16> - // CHECK: linalg.matmul_transpose_a ins(%arg0, %arg1 : tensor<512x128xbf16>, tensor<512x256xbf16>) outs([[FILLED]] : tensor<128x256xbf16>) -> tensor<128x256xbf16> - // CHECK: tensor.empty() - // CHECK: linalg.broadcast - // CHECK: tensor.empty() - // CHECK: linalg.add - // CHECK: arith.constant dense<0.0{{.*}}> - // CHECK: tensor.empty() - // CHECK: linalg.max - %0 = onednn_graph.matmul %arg0, %arg1, %arg3 {transpose_a = true} - : (tensor<512x128xbf16>, tensor<512x256xbf16>, tensor<256xbf16>) -> tensor<128x256xbf16> - %1 = onednn_graph.relu %0 : (tensor<128x256xbf16>) -> tensor<128x256xbf16> - return %1 : tensor<128x256xbf16> -} - -// CHECK-LABEL: @mlp_transpose_b -func.func @mlp_transpose_b(%arg0: tensor<128x512xbf16>, %arg1: tensor<256x512xbf16>, %arg3: tensor<256xbf16>) -> tensor<128x256xbf16> { - // CHECK: [[C0:%.+]] = arith.constant 0 - // CHECK: [[INIT:%.+]] = tensor.empty() - // CHECK: [[FILLED:%.+]] = linalg.fill ins([[C0]] : bf16) outs([[INIT]] : tensor<128x256xbf16>) -> tensor<128x256xbf16> - // CHECK: linalg.matmul_transpose_b ins(%arg0, %arg1 : tensor<128x512xbf16>, tensor<256x512xbf16>) outs([[FILLED]] : tensor<128x256xbf16>) -> tensor<128x256xbf16> - // CHECK: tensor.empty() - // CHECK: linalg.broadcast - // CHECK: tensor.empty() - // CHECK: linalg.add - // CHECK: arith.constant dense<0.0{{.*}}> - // CHECK: tensor.empty() - // CHECK: linalg.max - %0 = onednn_graph.matmul %arg0, %arg1, %arg3 {transpose_b = true} - : (tensor<128x512xbf16>, tensor<256x512xbf16>, tensor<256xbf16>) -> tensor<128x256xbf16> - %1 = onednn_graph.relu %0 : (tensor<128x256xbf16>) -> tensor<128x256xbf16> - return %1 : tensor<128x256xbf16> -} - -// CHECK-LABEL: @mlp_transpose_a_b -func.func @mlp_transpose_a_b(%arg0: tensor<512x128xbf16>, %arg1: tensor<256x512xbf16>, %arg3: tensor<256xbf16>) -> tensor<128x256xbf16> { - // CHECK: [[C0:%.+]] = arith.constant 0 - // CHECK: [[INIT0:%.+]] = tensor.empty() - // CHECK: [[FILLED0:%.+]] = linalg.fill ins([[C0]] : bf16) outs([[INIT0]] : tensor<256x128xbf16>) -> tensor<256x128xbf16> - // CHECK: [[MMT:%.+]] = linalg.matmul ins(%arg1, %arg0 : tensor<256x512xbf16>, tensor<512x128xbf16>) outs([[FILLED0]] : tensor<256x128xbf16>) -> tensor<256x128xbf16> - // CHECK: [[C1:%.+]] = arith.constant 0 - // CHECK: [[INIT1:%.+]] = tensor.empty() - // CHECK: [[FILLED1:%.+]] = linalg.fill ins([[C1]] : bf16) outs([[INIT1]] : tensor<128x256xbf16>) -> tensor<128x256xbf16> - // CHECK: linalg.transpose ins([[MMT]] : tensor<256x128xbf16>) outs([[FILLED1]] : tensor<128x256xbf16>) - // CHECK: tensor.empty() - // CHECK: linalg.broadcast - // CHECK: tensor.empty() - // CHECK: linalg.add - // CHECK: arith.constant dense<0.0{{.*}}> - // CHECK: tensor.empty() - // CHECK: linalg.max - %0 = onednn_graph.matmul %arg0, %arg1, %arg3 {transpose_a = true, transpose_b = true} - : (tensor<512x128xbf16>, tensor<256x512xbf16>, tensor<256xbf16>) -> tensor<128x256xbf16> - %1 = onednn_graph.relu %0 : (tensor<128x256xbf16>) -> tensor<128x256xbf16> - return %1 : tensor<128x256xbf16> -} diff --git a/test/mlir/test/gc/Dialect/XeVM/xevm-invalid.mlir b/test/mlir/test/gc/Dialect/XeVM/xevm-invalid.mlir deleted file mode 100644 index 86d00167f..000000000 --- a/test/mlir/test/gc/Dialect/XeVM/xevm-invalid.mlir +++ /dev/null @@ -1,171 +0,0 @@ -// RUN: gc-opt -split-input-file -verify-diagnostics %s - -llvm.func @blockload2d(%ptr : !llvm.ptr, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // expected-error @+1 {{'xevm.blockload2d' op transpose and vnni_transform are mutually exclusive}} - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=8, tile_width=16, tile_height=16, v_blocks=1, transpose=true, vnni_transform=true, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr, i32, i32, i32, i32, i32) -> vector<4xi32> - llvm.return -} - -// ----- - -llvm.func @blockload2d(%ptr : !llvm.ptr, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // expected-error @+1 {{'xevm.blockload2d' op expecting tile_height to be between 1 and 32}} - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=8, tile_width=32, tile_height=64, v_blocks=1, transpose=false, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr, i32, i32, i32, i32, i32) -> vector<64xi16> - llvm.return -} - -// ----- - -llvm.func @blockload2d(%ptr : !llvm.ptr, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // expected-error @+1 {{'xevm.blockload2d' op expecting tile_width to be between 4 and 64}} - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=8, tile_width=128, tile_height=8, v_blocks=1, transpose=false, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr, i32, i32, i32, i32, i32) -> vector<16xi32> - llvm.return -} - -// ----- - -llvm.func @blockload2d(%ptr : !llvm.ptr, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // expected-error @+1 {{'xevm.blockload2d' op expecting v_blocks to be 1, 2, or 4}} - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=8, tile_width=32, tile_height=8, v_blocks=6, transpose=false, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr, i32, i32, i32, i32, i32) -> vector<48xi16> - llvm.return -} - -// ----- - -llvm.func @blockload2d(%ptr : !llvm.ptr, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // expected-error @+1 {{'xevm.blockload2d' op tile_width * v_blocks should be less than or equal to 64 for 8 bit elements}} - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=8, tile_width=32, tile_height=8, v_blocks=4, transpose=false, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr, i32, i32, i32, i32, i32) -> vector<32xi16> - llvm.return -} - -// ----- - -llvm.func @blockload2d(%ptr : !llvm.ptr, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // expected-error @+1 {{'xevm.blockload2d' op transpose is only supported for 32 and 64 bit elements}} - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=8, tile_width=32, tile_height=8, v_blocks=1, transpose=true, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr, i32, i32, i32, i32, i32) -> vector<8xi16> - llvm.return -} - -// ----- - -llvm.func @blockload2d(%ptr : !llvm.ptr, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // expected-error @+1 {{'xevm.blockload2d' op vnni_transform is only supported for 8 and 16 bit elements}} - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=32, tile_width=8, tile_height=8, v_blocks=1, transpose=false, vnni_transform=true, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr, i32, i32, i32, i32, i32) -> vector<4xi32> - llvm.return -} - -// ----- - -llvm.func @blockload2d(%ptr : !llvm.ptr, %base_height : i32, %x : i32, %y : i32) { - %base_width = llvm.mlir.constant(4 : i32) : i32 - %base_pitch = llvm.mlir.constant(2 : i32) : i32 - // expected-error @+1 {{'xevm.blockload2d' op 4th operand (base pitch) should be >= 2nd operand (base width)}} - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=8, tile_width=32, tile_height=8, v_blocks=1, transpose=false, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr, i32, i32, i32, i32, i32) -> vector<8xi16> - llvm.return -} - -// ----- - -llvm.func @blockload2d(%ptr : !llvm.ptr, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // expected-error @+1 {{'xevm.blockload2d' op expecting 'elem_size_in_bits' to be 8, 16, or 32}} - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=64, tile_width=4, tile_height=8, v_blocks=1, transpose=false, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr, i32, i32, i32, i32, i32) -> vector<8xi16> - llvm.return -} - -// ----- - -llvm.func @blockload2d(%ptr : !llvm.ptr, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // expected-error @+1 {{'xevm.blockload2d' op expecting tile_height to be 1, 2, 4, 8, 16, or 32}} - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=8, tile_width=32, tile_height=24, v_blocks=1, transpose=false, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr, i32, i32, i32, i32, i32) -> vector<24xi16> - llvm.return -} - -// ----- - -llvm.func @blockload2d(%ptr : !llvm.ptr, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // expected-error @+1 {{'xevm.blockload2d' op tile_width when vnni_transform is true should be equal to subgroup size (16 elements)}} - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=8, tile_width=8, tile_height=8, v_blocks=1, transpose=false, vnni_transform=true, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr, i32, i32, i32, i32, i32) -> vector<1xi32> - llvm.return -} - -// ----- - -llvm.func @blockload2d(%ptr : !llvm.ptr, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32) { - // expected-error @+1 {{'xevm.blockload2d' op expecting result element type to be 32 bits}} - %0 = xevm.blockload2d %ptr, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=32, tile_width=8, tile_height=8, v_blocks=1, transpose=false, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr, i32, i32, i32, i32, i32) -> vector<8xi16> - llvm.return -} - -// ----- - -llvm.func @blockstore2d(%ptr : !llvm.ptr, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32, %stored_val : vector<64xi8>) { - // expected-error @+1 {{'xevm.blockstore2d' op expecting tile_height to be between 1 and 8}} - xevm.blockstore2d %ptr, %base_width, %base_height, %base_pitch, %x, %y, %stored_val {elem_size_in_bits=8, tile_width=32, tile_height=64, v_blocks=1, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr, i32, i32, i32, i32, i32, vector<64xi8>) - llvm.return -} - -// ----- - -llvm.func @blockstore2d(%ptr : !llvm.ptr, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32, %stored_val : vector<4xi8>) { - // expected-error @+1 {{'xevm.blockstore2d' op expecting tile_width to be between 4 and 64}} - xevm.blockstore2d %ptr, %base_width, %base_height, %base_pitch, %x, %y, %stored_val {elem_size_in_bits=8, tile_width=2, tile_height=8, v_blocks=1, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr, i32, i32, i32, i32, i32, vector<4xi8>) - llvm.return -} - -// ----- - -llvm.func @blockstore2d(%ptr : !llvm.ptr, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32, %stored_val : vector<8xi8>) { - // expected-error @+1 {{'xevm.blockstore2d' op expecting v_blocks to be 1}} - xevm.blockstore2d %ptr, %base_width, %base_height, %base_pitch, %x, %y, %stored_val {elem_size_in_bits=8, tile_width=32, tile_height=8, v_blocks=2, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr, i32, i32, i32, i32, i32, vector<8xi8>) - llvm.return -} - -// ----- - -llvm.func @blockstore2d(%ptr : !llvm.ptr, %base_height : i32, %x : i32, %y : i32, %stored_val : vector<8xi8>) { - %base_width = llvm.mlir.constant(4 : i32) : i32 - %base_pitch = llvm.mlir.constant(2 : i32) : i32 - // expected-error @+1 {{'xevm.blockstore2d' op 4th operand (base pitch) should be >= 2nd operand (base width)}} - xevm.blockstore2d %ptr, %base_width, %base_height, %base_pitch, %x, %y, %stored_val {elem_size_in_bits=8, tile_width=32, tile_height=8, v_blocks=1, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr, i32, i32, i32, i32, i32, vector<8xi8>) - llvm.return -} - -// ----- - -llvm.func @blockstore2d(%ptr : !llvm.ptr, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32, %stored_val : vector<32xi16>) { - // expected-error @+1 {{'xevm.blockstore2d' op expecting 'elem_size_in_bits' to be 8, 16, or 32}} - xevm.blockstore2d %ptr, %base_width, %base_height, %base_pitch, %x, %y, %stored_val {elem_size_in_bits=64, tile_width=4, tile_height=8, v_blocks=1, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr, i32, i32, i32, i32, i32, vector<32xi16>) - llvm.return -} - -// ----- - -llvm.func @blockstore2d(%ptr : !llvm.ptr, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32, %stored_val : vector<64xi8>) { - // expected-error @+1 {{'xevm.blockstore2d' op expecting tile_height to be 1, 2, 4, 8, 16, or 32}} - xevm.blockstore2d %ptr, %base_width, %base_height, %base_pitch, %x, %y, %stored_val {elem_size_in_bits=8, tile_width=32, tile_height=6, v_blocks=1, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr, i32, i32, i32, i32, i32, vector<64xi8>) - llvm.return -} - -// ----- - -llvm.func @blockstore2d(%ptr : !llvm.ptr, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32, %stored_val : vector<4xi8>) { - // expected-error @+1 {{'xevm.blockstore2d' op tile_width for 8 bit elements should be equal to 16 or 32}} - xevm.blockstore2d %ptr, %base_width, %base_height, %base_pitch, %x, %y, %stored_val {elem_size_in_bits=8, tile_width=8, tile_height=8, v_blocks=1, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr, i32, i32, i32, i32, i32, vector<4xi8>) - llvm.return -} - -// ----- - -llvm.func @blockstore2d(%ptr : !llvm.ptr, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32, %stored_val : vector<8xi16>) { - // expected-error @+1 {{'xevm.blockstore2d' op tile_width for 16 bit elements should be equal to 16}} - xevm.blockstore2d %ptr, %base_width, %base_height, %base_pitch, %x, %y, %stored_val {elem_size_in_bits=16, tile_width=32, tile_height=8, v_blocks=1, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr, i32, i32, i32, i32, i32, vector<8xi16>) - llvm.return -} - -// ----- - -llvm.func @blockstore2d(%ptr : !llvm.ptr, %base_width : i32, %base_height : i32, %base_pitch : i32, %x : i32, %y : i32, %stored_val : vector<4xi32>) { - // expected-error @+1 {{'xevm.blockstore2d' op tile_width for 32 bit elements should be equal to 16}} - xevm.blockstore2d %ptr, %base_width, %base_height, %base_pitch, %x, %y, %stored_val {elem_size_in_bits=32, tile_width=8, tile_height=8, v_blocks=1, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr, i32, i32, i32, i32, i32, vector<4xi32>) - llvm.return -} diff --git a/test/mlir/test/gc/Transforms/DecomposeAggregatedOps.mlir b/test/mlir/test/gc/Transforms/DecomposeAggregatedOps.mlir deleted file mode 100644 index 9b178b3e7..000000000 --- a/test/mlir/test/gc/Transforms/DecomposeAggregatedOps.mlir +++ /dev/null @@ -1,11 +0,0 @@ -// RUN: gc-opt %s -decompose-aggregated-ops | FileCheck %s - -// CHECK-LABEL: softmax -func.func @softmax(%arg0: tensor<2x2x2x2xf32>) -> tensor<2x2x2x2xf32> { - %0 = tensor.empty() : tensor<2x2x2x2xf32> - // CHECK-NOT: linalg.softmax - // CHECK-COUNT-4: linalg.generic - %1 = linalg.softmax dimension(3) - ins(%arg0 : tensor<2x2x2x2xf32>) outs(%0 : tensor<2x2x2x2xf32>) -> tensor<2x2x2x2xf32> - return %1 : tensor<2x2x2x2xf32> -} diff --git a/test/mlir/test/gc/Transforms/DecomposeTensorOperation.mlir b/test/mlir/test/gc/Transforms/DecomposeTensorOperation.mlir deleted file mode 100644 index dd2aa9aed..000000000 --- a/test/mlir/test/gc/Transforms/DecomposeTensorOperation.mlir +++ /dev/null @@ -1,72 +0,0 @@ -// RUN: gc-opt %s -decompose-tensor-operation --split-input-file | FileCheck %s - -/// CHECK-LABEL: @gather_single_gather_dim -func.func @gather_single_gather_dim(%arg0: tensor<2x2x2x2xf32>, %arg1: tensor<2x3x1xindex>) -> tensor<2x3x2x2x2xf32> { - /// CHECK: %[[EMPTY:.*]] = tensor.empty() : tensor<2x3x2x2x2xf32> - /// CHECK: linalg.generic {{.*}} iterator_types = ["parallel", "parallel", "parallel", "parallel", "parallel"]} ins(%[[ARG1:.*]] : tensor<2x3x1xindex>) outs(%[[EMPTY:.*]] : tensor<2x3x2x2x2xf32>) - %1 = tensor.gather %arg0[%arg1] gather_dims([1]) : (tensor<2x2x2x2xf32>, tensor<2x3x1xindex>) -> tensor<2x3x2x2x2xf32> - return %1 : tensor<2x3x2x2x2xf32> -} - -// ----- - -/// CHECK-LABEL: @gather_single_gather_dim_no_shrink -func.func @gather_single_gather_dim_no_shrink(%arg0: tensor<2x2x2x2xf32>, %arg1: tensor<2x3x1xindex>) -> tensor<2x3x2x1x2x2xf32> { - /// CHECK: %[[EMPTY:.*]] = tensor.empty() : tensor<2x3x2x1x2x2xf32> - /// CHECK: linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel", "parallel", "parallel", "parallel"]} ins(%[[ARG1:.*]] : tensor<2x3x1xindex>) outs(%[[EMPTY:.*]] : tensor<2x3x2x1x2x2xf32>) - %1 = tensor.gather %arg0[%arg1] gather_dims([1]) : (tensor<2x2x2x2xf32>, tensor<2x3x1xindex>) -> tensor<2x3x2x1x2x2xf32> - return %1 : tensor<2x3x2x1x2x2xf32> -} - -// ----- - -/// CHECK-LABEL: @gather_multiple_gather_dim -func.func @gather_multiple_gather_dim(%arg0: tensor<2x2x2x2xf32>, %arg1: tensor<2x3x2xindex>) -> tensor<2x3x2x2xf32> { - /// CHECK: %[[EMPTY:.*]] = tensor.empty() : tensor<2x3x2x2xf32> - /// CHECK: %[[EXTRACTSLICE1:.*]] = tensor.extract_slice %[[ARG1:.*]][0, 0, 0] [2, 3, 1] [1, 1, 1] : tensor<2x3x2xindex> to tensor<2x3x1xindex> - /// CHECK: %[[EXTRACTSLICE2:.*]] = tensor.extract_slice %[[ARG1:.*]][0, 0, 1] [2, 3, 1] [1, 1, 1] : tensor<2x3x2xindex> to tensor<2x3x1xindex> - /// CHECK: linalg.generic {indexing_maps = [#map, #map, #map1], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins(%[[EXTRACTSLICE1:.*]], %[[EXTRACTSLICE2:.*]] : tensor<2x3x1xindex>, tensor<2x3x1xindex>) outs(%[[EMPTY:.*]] : tensor<2x3x2x2xf32>) - %1 = tensor.gather %arg0[%arg1] gather_dims([1, 2]) : (tensor<2x2x2x2xf32>, tensor<2x3x2xindex>) -> tensor<2x3x2x2xf32> - return %1 : tensor<2x3x2x2xf32> -} - -// ----- - -/// CHECK-LABEL: @gather_multiple_gather_dim_no_shrink -func.func @gather_multiple_gather_dim_no_shrink(%arg0: tensor<2x2x2x2xf32>, %arg1: tensor<2x3x2xindex>) -> tensor<2x3x2x1x1x2xf32> { - /// CHECK: %[[EMPTY:.*]] = tensor.empty() : tensor<2x3x2x1x1x2xf32> - /// CHECK: %[[EXTRACTSLICE1:.*]] = tensor.extract_slice %[[ARG1:.*]][0, 0, 0] [2, 3, 1] [1, 1, 1] : tensor<2x3x2xindex> to tensor<2x3x1xindex> - /// CHECK: %[[EXTRACTSLICE2:.*]] = tensor.extract_slice %[[ARG1:.*]][0, 0, 1] [2, 3, 1] [1, 1, 1] : tensor<2x3x2xindex> to tensor<2x3x1xindex> - /// CHECK: linalg.generic {indexing_maps = [#map, #map, #map1], iterator_types = ["parallel", "parallel", "parallel", "parallel", "parallel", "parallel"]} ins(%[[EXTRACTSLICE1:.*]], %[[EXTRACTSLICE2:.*]] : tensor<2x3x1xindex>, tensor<2x3x1xindex>) outs(%[[EMPTY:.*]] : tensor<2x3x2x1x1x2xf32>) - %1 = tensor.gather %arg0[%arg1] gather_dims([1, 2]) : (tensor<2x2x2x2xf32>, tensor<2x3x2xindex>) -> tensor<2x3x2x1x1x2xf32> - return %1 : tensor<2x3x2x1x1x2xf32> -} - -// ----- - -/// CHECK-LABEL: @gather_single_gather_dim_dynamic -func.func @gather_single_gather_dim_dynamic(%arg0: tensor, %arg1: tensor<2x3x1xindex>) -> tensor<2x3x?x?x?xf32> { - /// CHECK: %[[DIM1:.*]] = tensor.dim - /// CHECK: %[[DIM2:.*]] = tensor.dim - /// CHECK: %[[DIM3:.*]] = tensor.dim - /// CHECK: %[[EMPTY:.*]] = tensor.empty(%[[DIM1:.*]], %[[DIM2:.*]], %[[DIM3:.*]]) : tensor<2x3x?x?x?xf32> - /// CHECK: linalg.generic {{.*}} iterator_types = ["parallel", "parallel", "parallel", "parallel", "parallel"]} ins(%[[ARG0:.*]] : tensor<2x3x1xindex>) outs(%[[EMPTY:.*]] : tensor<2x3x?x?x?xf32>) - %1 = tensor.gather %arg0[%arg1] gather_dims([1]) : (tensor, tensor<2x3x1xindex>) -> tensor<2x3x?x?x?xf32> - return %1 : tensor<2x3x?x?x?xf32> -} - -// ----- - -/// CHECK-LABEL: @gather_multiple_gather_dim_no_shrink_dynamic -func.func @gather_multiple_gather_dim_no_shrink_dynamic(%arg0: tensor<2x2x2x2xf32>, %arg1: tensor) -> tensor { - /// CHECK: %[[DIM1:.*]] = tensor.dim - /// CHECK: %[[DIM2:.*]] = tensor.dim - /// CHECK: %[[EMPTY:.*]] = tensor.empty(%[[DIM1:.*]], %[[DIM2:.*]]) : tensor - /// CHECK: %[[DIM3:.*]] = tensor.dim - /// CHECK: %[[DIM4:.*]] = tensor.dim - /// CHECK: %[[EXTRACTSLICE1:.*]] = tensor.extract_slice %[[ARG1:.*]][0, 0, 0] [%[[DIM3:.*]], %[[DIM4:.*]], 1] [1, 1, 1] : tensor to tensor - /// CHECK: %[[EXTRACTSLICE2:.*]] = tensor.extract_slice %[[ARG1:.*]][0, 0, 1] [%[[DIM3:.*]], %[[DIM4:.*]], 1] [1, 1, 1] : tensor to tensor - /// CHECK: linalg.generic {indexing_maps = [#map, #map, #map1], iterator_types = ["parallel", "parallel", "parallel", "parallel", "parallel", "parallel"]} ins(%[[EXTRACTSLICE1:.*]], %[[EXTRACTSLICE2:.*]] : tensor, tensor) outs(%[[EMPTY:.*]] : tensor) - %1 = tensor.gather %arg0[%arg1] gather_dims([1, 2]) : (tensor<2x2x2x2xf32>, tensor) -> tensor - return %1 : tensor -} diff --git a/test/mlir/test/gc/Transforms/GPU/IMEX/gpu-pipeline.mlir b/test/mlir/test/gc/Transforms/GPU/IMEX/gpu-pipeline.mlir deleted file mode 100644 index 1c94b11a7..000000000 --- a/test/mlir/test/gc/Transforms/GPU/IMEX/gpu-pipeline.mlir +++ /dev/null @@ -1,18 +0,0 @@ -// RUN: gc-opt %s --gc-imex-pipeline -split-input-file | FileCheck %s - -// Regression check for infinite loop in the linalg-to-xegpu pass -// CHECK-DAG: @gcGpuOclKernel_corner_shape_matmul_f16_kernel_SPIRV -// CHECK-DAG: @createGcGpuOclKernel_corner_shape_matmul_f16_kernel -// CHECK-DAG: @corner_shape_matmul_f16 -module @fragment_name attributes {} { - func.func @corner_shape_matmul_f16(%arg0: memref<521x521xf16>, %arg1: memref<521x521xf16>, %arg2: memref<521x521xf16>) { - %0 = bufferization.to_tensor %arg0 restrict : memref<521x521xf16> - %1 = bufferization.to_tensor %arg1 restrict : memref<521x521xf16> - %2 = tensor.empty() : tensor<521x521xf16> - %cst = arith.constant 0.000000e+00 : f16 - %3 = linalg.fill ins(%cst : f16) outs(%2 : tensor<521x521xf16>) -> tensor<521x521xf16> - %4 = linalg.matmul_transpose_b ins(%0, %1 : tensor<521x521xf16>, tensor<521x521xf16>) outs(%3 : tensor<521x521xf16>) -> tensor<521x521xf16> - bufferization.materialize_in_destination %4 in restrict writable %arg2 : (tensor<521x521xf16>, memref<521x521xf16>) -> () - return - } -} diff --git a/test/mlir/test/gc/Transforms/GPU/IMEX/gpu-to-gpuocl.mlir b/test/mlir/test/gc/Transforms/GPU/IMEX/gpu-to-gpuocl.mlir deleted file mode 100644 index 1a8933f12..000000000 --- a/test/mlir/test/gc/Transforms/GPU/IMEX/gpu-to-gpuocl.mlir +++ /dev/null @@ -1,98 +0,0 @@ -// RUN: gc-opt %s --gpu-to-gpuocl | FileCheck %s - -module @test attributes {gpu.container_module} { - llvm.func @entry(%arg0: !llvm.ptr, %arg1: !llvm.ptr, %arg2: i64, %arg3: i64, %arg4: i64, %arg5: i64, %arg6: i64, %arg7: !llvm.ptr, %arg8: !llvm.ptr, %arg9: i64) attributes {llvm.emit_c_interface} { - %0 = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - %1 = llvm.insertvalue %arg0, %0[0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - %2 = llvm.insertvalue %arg1, %1[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - %3 = llvm.insertvalue %arg2, %2[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - %4 = llvm.insertvalue %arg3, %3[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - %5 = llvm.insertvalue %arg4, %4[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - %6 = llvm.insertvalue %arg5, %5[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - %7 = llvm.insertvalue %arg6, %6[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - %8 = builtin.unrealized_conversion_cast %7 : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> to memref<64x64xf32> - %gpu_mem = gpu.alloc host_shared () : memref<64x64xf32> - gpu.memcpy %gpu_mem, %8 : memref<64x64xf32>, memref<64x64xf32> - %9 = llvm.mlir.constant(32 : index) : i64 - %10 = builtin.unrealized_conversion_cast %9 : i64 to index - %11 = llvm.mlir.constant(2 : index) : i64 - %12 = builtin.unrealized_conversion_cast %11 : i64 to index - %13 = llvm.mlir.constant(1 : index) : i64 - %14 = builtin.unrealized_conversion_cast %13 : i64 to index - gpu.launch_func @entry_kernel::@entry_kernel blocks in (%12, %12, %14) threads in (%14, %14, %14) args(%10 : index, %gpu_mem : memref<64x64xf32>) - gpu.memcpy %8, %gpu_mem : memref<64x64xf32>, memref<64x64xf32> - gpu.dealloc %gpu_mem : memref<64x64xf32> - llvm.return - } - - gpu.module @entry_kernel attributes {gpu.binary = "Some SPIRV here \00"} { - gpu.func @entry_kernel(%arg0: index, %arg1: memref<64x64xf32>) kernel attributes {} { - gpu.return - } - } -} - -// CHECK: llvm.mlir.global internal constant @gcGpuOclKernel_entry_kernel_SPIRV -// CHECK: llvm.mlir.global internal constant @gcGpuOclKernel_entry_kernel_Name -// CHECK: llvm.mlir.global internal @gcGpuOclKernel_entry_kernel_Ptr - -// CHECK: llvm.func @createGcGpuOclKernel_entry_kernel([[CTX:%.+]]: !llvm.ptr) -> !llvm.ptr -// CHECK: [[NEW_PTR:%.+]] = llvm.call @gcGpuOclKernelCreate([[CTX]] -// CHECK: [[ZERO:%.+]] = llvm.mlir.zero -// CHECK: [[PTR_ADDR:%.+]] = llvm.mlir.addressof @gcGpuOclKernel_entry_kernel_Ptr -// CHECK: [[CMPXCHG:%.+]] = llvm.cmpxchg [[PTR_ADDR]], [[ZERO]], [[NEW_PTR]] -// CHECK: [[FLAG:%.+]] = llvm.extractvalue [[CMPXCHG]][1] -// CHECK: llvm.cond_br [[FLAG]], [[BB1:\^.+]], [[BB2:\^.+]] -// CHECK: [[BB1]]: -// CHECK: llvm.return [[NEW_PTR]] -// CHECK: [[BB2]]: -// CHECK: [[ONE:%.+]] = llvm.mlir.constant(1 : i64) : i64 -// CHECK: [[ARRAY:%.+]] = llvm.alloca [[ONE]] -// CHECK: [[ADDR:%.+]] = llvm.getelementptr [[ARRAY]] -// CHECK: llvm.store [[NEW_PTR]], [[ADDR]] -// CHECK: llvm.call @gcGpuOclKernelDestroy([[ONE]], [[ARRAY]]) -// CHECK: [[OLD_PTR:%.+]] = llvm.extractvalue [[CMPXCHG]][0] -// CHECK: llvm.return [[OLD_PTR]] - -// CHECK: llvm.func internal @getGcGpuOclKernel_entry_kernel([[CTX:%.+]]: !llvm.ptr) -> !llvm.ptr attributes {always_inline} -// CHECK: [[ZERO:%.+]] = llvm.mlir.zero -// CHECK: [[PTR_ADDR:%.+]] = llvm.mlir.addressof @gcGpuOclKernel_entry_kernel_Ptr -// CHECK: [[PTR:%.+]] = llvm.load [[PTR_ADDR]] -// CHECK: [[ICMP:%.+]] = llvm.icmp "eq" [[PTR]], [[ZERO]] -// CHECK: llvm.cond_br [[ICMP]], [[BB1:\^.+]], [[BB2:\^.+]] -// CHECK: [[BB1]]: -// CHECK: [[NEW_PTR:%.+]] = llvm.call @createGcGpuOclKernel_entry_kernel([[CTX]]) -// CHECK: llvm.return [[NEW_PTR]] -// CHECK: [[BB2]]: -// CHECK: llvm.return [[PTR]] - -// CHECK: llvm.func @entry(%arg0: !llvm.ptr, %arg1: !llvm.ptr, %arg2: i64, %arg3: i64, %arg4: i64, %arg5: i64, %arg6: i64, [[CTX:%.+]]: !llvm.ptr, %arg8: !llvm.ptr, %arg9: i64) -// CHECK: [[SIZE:%.+]] = llvm.mlir.constant(16384 : i64) : i64 -// CHECK: llvm.call @gcGpuOclMallocShared([[CTX]], [[SIZE]]) -// CHECK: [[SIZE:%.+]] = llvm.mlir.constant(16384 : i64) : i64 -// CHECK: [[SRC:%.+]] = llvm.extractvalue -// CHECK: [[DST:%.+]] = llvm.extractvalue [[GPU_MEMREF:%.+]][1] -// CHECK: llvm.call @gcGpuOclMemcpy([[CTX]], [[SRC]], [[DST]], [[SIZE]]) -// CHECK: [[KERNEL:%.+]] = llvm.call @getGcGpuOclKernel_entry_kernel([[CTX:%.+]]) : (!llvm.ptr) -> !llvm.ptr -// CHECK: llvm.call @gcGpuOclKernelLaunch([[CTX]], [[KERNEL]], -// CHECK: [[SIZE:%.+]] = llvm.mlir.constant(16384 : i64) : i64 -// CHECK: [[SRC:%.+]] = llvm.extractvalue [[GPU_MEMREF:%.+]][1] -// CHECK: [[DST:%.+]] = llvm.extractvalue -// CHECK: llvm.call @gcGpuOclMemcpy([[CTX]], [[SRC]], [[DST]], [[SIZE]]) -// CHECK: [[GPU_PTR:%.+]] = llvm.extractvalue [[GPU_MEMREF:%.+]][0] -// CHECK: llvm.call @gcGpuOclDealloc([[CTX]], [[GPU_PTR]]) - -// CHECK: llvm.func @gcGpuOclKernelCreate -// CHECK: llvm.func @gcGpuOclKernelDestroy -// CHECK: llvm.func @gcGpuOclKernelLaunch - - -// CHECK: llvm.func @gcGpuOclModuleDestructor() -// CHECK: llvm.fence acquire -// CHECK: [[PTR_ADDR:%.+]] = llvm.mlir.addressof @gcGpuOclKernel_entry_kernel_Ptr -// CHECK: [[PTR:%.+]] = llvm.load [[PTR_ADDR]] -// CHECK: [[ONE:%.+]] = llvm.mlir.constant(1 : i64) : i64 -// CHECK: [[ARRAY:%.+]] = llvm.alloca [[ONE]] -// CHECK: [[ADDR:%.+]] = llvm.getelementptr [[ARRAY]] -// CHECK: llvm.store [[PTR]], [[ADDR]] -// CHECK: llvm.call @gcGpuOclKernelDestroy([[ONE]], [[ARRAY]]) diff --git a/test/mlir/test/gc/Transforms/GPU/IMEX/linalg-to-xegpu-broadcast-fold.mlir b/test/mlir/test/gc/Transforms/GPU/IMEX/linalg-to-xegpu-broadcast-fold.mlir deleted file mode 100644 index bb0091cea..000000000 --- a/test/mlir/test/gc/Transforms/GPU/IMEX/linalg-to-xegpu-broadcast-fold.mlir +++ /dev/null @@ -1,109 +0,0 @@ -// RUN: gc-opt %s -linalg-to-xegpu="dpas-tile=8,16,16 k-tile=16" -canonicalize -split-input-file | FileCheck %s - -// CHECK-LABEL: func.func @broadcast_eliminate_2d -func.func @broadcast_eliminate_2d() { - %c1 = arith.constant 1 : index - %c2 = arith.constant 2 : index - %c4 = arith.constant 4 : index - - // CHECK: %[[MEMREF_0:.*]] = memref.alloc() : memref<7x128xf16> - %0 = memref.alloc() : memref<7x128xf16> - // CHECK: %[[MEMREF_2:.*]] = memref.alloc() : memref<1x1x7x128xf16> - %2 = memref.alloc() : memref<1x1x7x128xf16> - // CHECK: %[[MEMREF_3:.*]] = memref.alloc() : memref<1x1x7x128xf16> - %3 = memref.alloc() : memref<1x1x7x128xf16> - - gpu.launch blocks(%arg3, %arg4, %arg5) in (%arg11 = %c2, %arg12 = %c4, %arg13 = %c1) threads(%arg6, %arg7, %arg8) in (%arg14 = %c4, %arg15 = %c1, %arg16 = %c1) { - // CHECK-NOT: memref.alloc() : memref<4x1x7x128xf16, 3> - %slm_base = memref.alloc() : memref<4x1x7x128xf16, 3> - %1 = memref.subview %slm_base[%arg6, 0, 0, 0] [1, 1, 7, 128] [1, 1, 1, 1] : memref<4x1x7x128xf16, 3> to memref<1x1x7x128xf16, strided<[896, 896, 128, 1], offset: ?>, 3> - - // CHECK-NOT: linalg.broadcast - linalg.broadcast ins(%0 : memref<7x128xf16>) outs(%1 : memref<1x1x7x128xf16, strided<[896, 896, 128, 1], offset: ?>, 3>) dimensions = [0, 1] - // CHECK: xegpu.create_nd_tdesc %[[MEMREF_0]] - linalg.add ins(%1, %2 : memref<1x1x7x128xf16, strided<[896, 896, 128, 1], offset: ?>, 3>, memref<1x1x7x128xf16>) outs(%3 : memref<1x1x7x128xf16>) - gpu.terminator - } - return -} - -// ----- - -// CHECK-LABEL: func.func @broadcast_eliminate -func.func @broadcast_eliminate_3d() { - %c1 = arith.constant 1 : index - %c2 = arith.constant 2 : index - %c4 = arith.constant 4 : index - - // CHECK: %[[MEMREF_0:.*]] = memref.alloc() : memref<1x7x128xf16> - %0 = memref.alloc() : memref<1x7x128xf16> - // CHECK: %[[MEMREF_2:.*]] = memref.alloc() : memref<1x1x7x128xf16> - %2 = memref.alloc() : memref<1x1x7x128xf16> - // CHECK: %[[MEMREF_3:.*]] = memref.alloc() : memref<1x1x7x128xf16> - %3 = memref.alloc() : memref<1x1x7x128xf16> - - gpu.launch blocks(%arg3, %arg4, %arg5) in (%arg11 = %c2, %arg12 = %c4, %arg13 = %c1) threads(%arg6, %arg7, %arg8) in (%arg14 = %c4, %arg15 = %c1, %arg16 = %c1) { - // CHECK-NOT: memref.alloc() : memref<4x1x7x128xf16, 3> - %slm_base = memref.alloc() : memref<4x1x7x128xf16, 3> - %1 = memref.subview %slm_base[%arg6, 0, 0, 0] [1, 1, 7, 128] [1, 1, 1, 1] : memref<4x1x7x128xf16, 3> to memref<1x1x7x128xf16, strided<[896, 896, 128, 1], offset: ?>, 3> - - // CHECK-NOT: linalg.broadcast - linalg.broadcast ins(%0 : memref<1x7x128xf16>) outs(%1 : memref<1x1x7x128xf16, strided<[896, 896, 128, 1], offset: ?>, 3>) dimensions = [0] - // Squeezing the %0 before passing to 'linalg.add' - // CHECK: %[[MEMREF0_SQUEEZ:.+]] = memref.subview %[[MEMREF_0]][0, 0, 0] [1, 7, 128] [1, 1, 1] : - // CHECK-SAME: memref<1x7x128xf16> to memref<7x128xf16, strided<[128, 1]>> - // CHECK: xegpu.create_nd_tdesc %[[MEMREF0_SQUEEZ]] - linalg.add ins(%1, %2 : memref<1x1x7x128xf16, strided<[896, 896, 128, 1], offset: ?>, 3>, memref<1x1x7x128xf16>) outs(%3 : memref<1x1x7x128xf16>) - gpu.terminator - } - return -} - -// ----- - -// CHECK-LABEL: func.func @complex_broadcast -func.func @complex_broadcast_3d() { - %c1 = arith.constant 1 : index - %c2 = arith.constant 2 : index - %c4 = arith.constant 4 : index - - // CHECK: %[[MEMREF_0:.*]] = memref.alloc() : memref<7x128xf16> - %0 = memref.alloc() : memref<7x128xf16> - // CHECK: %[[MEMREF_1:.*]] = memref.alloc() : memref<7x7x128xf16> - %1 = memref.alloc() : memref<7x7x128xf16> - // CHECK: %[[MEMREF_2:.*]] = memref.alloc() : memref<7x7x128xf16> - %2 = memref.alloc() : memref<7x7x128xf16> - // CHECK: %[[MEMREF_3:.*]] = memref.alloc() : memref<7x7x128xf16> - %3 = memref.alloc() : memref<7x7x128xf16> - - gpu.launch blocks(%arg3, %arg4, %arg5) in (%arg11 = %c2, %arg12 = %c4, %arg13 = %c1) threads(%arg6, %arg7, %arg8) in (%arg14 = %c4, %arg15 = %c1, %arg16 = %c1) { - // This broadcast can't be replaced by a single memref.subview. Can't remove it - // CHECK: linalg.broadcast - linalg.broadcast ins(%0 : memref<7x128xf16>) outs(%1 : memref<7x7x128xf16>) dimensions = [0] - linalg.add ins(%1, %2 : memref<7x7x128xf16>, memref<7x7x128xf16>) outs(%3 : memref<7x7x128xf16>) - gpu.terminator - } - return -} - -// ----- - -// CHECK-LABEL: func.func @single_broadcast -func.func @single_broadcast() { - %c1 = arith.constant 1 : index - %c2 = arith.constant 2 : index - %c4 = arith.constant 4 : index - - // CHECK: %[[MEMREF_0:.*]] = memref.alloc() : memref<7x128xf16> - %0 = memref.alloc() : memref<7x128xf16> - // CHECK: %[[MEMREF_1:.*]] = memref.alloc() : memref<1x1x7x128xf16> - %1 = memref.alloc() : memref<1x1x7x128xf16> - - gpu.launch blocks(%arg3, %arg4, %arg5) in (%arg11 = %c2, %arg12 = %c4, %arg13 = %c1) threads(%arg6, %arg7, %arg8) in (%arg14 = %c4, %arg15 = %c1, %arg16 = %c1) { - // broadcast result is not an input of any xegpu operation, we can't lower it - // CHECK: linalg.broadcast - linalg.broadcast ins(%0 : memref<7x128xf16>) outs(%1 : memref<1x1x7x128xf16>) dimensions = [0, 1] - gpu.terminator - } - return -} diff --git a/test/mlir/test/gc/Transforms/GPU/IMEX/linalg-to-xegpu-dpas-transpose-sep-alloc.mlir b/test/mlir/test/gc/Transforms/GPU/IMEX/linalg-to-xegpu-dpas-transpose-sep-alloc.mlir deleted file mode 100644 index 6e08efe46..000000000 --- a/test/mlir/test/gc/Transforms/GPU/IMEX/linalg-to-xegpu-dpas-transpose-sep-alloc.mlir +++ /dev/null @@ -1,86 +0,0 @@ -// RUN: gc-opt %s -linalg-to-xegpu="dpas-tile=8,16,16 k-tile=16" -canonicalize -split-input-file | FileCheck %s - -module { - func.func @matmul_transpose_b_sep(%arg0: memref<1024x1024xf16>, %arg1: memref<1024x1024xf16>, %arg2: memref<1024x1024xf16>) { - %c0 = arith.constant 0 : index - %c32 = arith.constant 32 : index - %c1024 = arith.constant 1024 : index - %alloc = memref.alloc() {alignment = 64 : i64} : memref<1024x1024xf16> - scf.parallel (%arg3, %arg4) = (%c0, %c0) to (%c1024, %c1024) step (%c32, %c32) { - %subview_0 = memref.subview %arg2[%arg3, %arg4] [32, 32] [1, 1] : memref<1024x1024xf16> to memref<32x32xf16, strided<[1024, 1], offset: ?>> - %subview_1 = memref.subview %arg0[%arg3, 0] [32, 1024] [1, 1] : memref<1024x1024xf16> to memref<32x1024xf16, strided<[1024, 1], offset: ?>> - %subview_2 = memref.subview %arg1[%arg4, 0] [32, 1024] [1, 1] : memref<1024x1024xf16> to memref<32x1024xf16, strided<[1024, 1], offset: ?>> - %subview_3 = memref.subview %alloc[0, %arg4] [1024, 32] [1, 1] : memref<1024x1024xf16> to memref<1024x32xf16, strided<[1024, 1], offset: ?>> - linalg.transpose ins(%subview_2 : memref<32x1024xf16, strided<[1024, 1], offset: ?>>) outs(%subview_3 : memref<1024x32xf16, strided<[1024, 1], offset: ?>>) permutation = [1, 0] - linalg.matmul ins(%subview_1, %subview_3 : memref<32x1024xf16, strided<[1024, 1], offset: ?>>, memref<1024x32xf16, strided<[1024, 1], offset: ?>>) outs(%subview_0 : memref<32x32xf16, strided<[1024, 1], offset: ?>>) - scf.reduce - } - memref.dealloc %alloc : memref<1024x1024xf16> - return - } -} - -// CHECK-LABEL: func.func @matmul_transpose_b_sep -// CHECK-SAME: %[[Ap:.+]]: memref<1024x1024xf16>, %[[Bp:.+]]: memref<1024x1024xf16>, %[[Cp:.+]]: memref<1024x1024xf16> - -// CHECK-NOT: memref.alloc() - -// CHECK: scf.parallel (%[[iter1:.+]], %[[iter2:.+]]) = (%c0, %c0) to (%c1024, %c1024) step (%c32, %c32) { -// CHECK: %[[C:.+]] = memref.subview %[[Cp]][%[[iter1]], %[[iter2]]] {{.*}} -// CHECK: %[[A:.+]] = memref.subview %[[Ap]][%[[iter1]], 0] {{.*}} -// CHECK: %[[B:.+]] = memref.subview %[[Bp]][%[[iter2]], 0] {{.*}} - -// CHECK-NOT: linalg.transpose - -// Create output initial value load tiles. -// CHECK-DAG: %[[rootC:.+]] = xegpu.create_nd_tdesc %[[C]] -// CHECK: %[[tC:.+]] = xegpu.update_nd_offset %[[rootC]], [%c0, %c0] -// CHECK-COUNT-7: xegpu.update_nd_offset - -// Load initial accumulator values. -// CHECK-DAG: %[[vC:.+]] = xegpu.load_nd %[[tC]] -// CHECK-COUNT-7: xegpu.load_nd - -// Extend the type to match DPAS output precision. -// CHECK: %[[vC_f32:.+]] = arith.extf %[[vC]] -// CHECK-COUNT-7: arith.extf - -// Create input load tiles. -// CHECK: %[[rootA:.+]] = xegpu.create_nd_tdesc %[[A]] -// CHECK: %[[tA:.+]] = xegpu.update_nd_offset %[[rootA]], [%c0, %c0] -// CHECK: %[[rootB:.+]] = xegpu.create_nd_tdesc %[[B]] -// CHECK: %[[tB:.+]] = xegpu.update_nd_offset %[[rootB]], [%c0, %c0] -// CHECK: %[[tB1:.+]] = xegpu.update_nd_offset %[[rootB]], [%c16, %c0] - -// Create DPAS computation loop over tiled reduction dimension. -// CHECK: %[[res:.+]]:11 = scf.for{{.*}}%c0 to %c1024 step %c16 -// CHECK-SAME: iter_args(%[[acc:.+]] = %[[vC_f32]],{{.*}}%[[iterA:.+]] = %[[tA]],{{.*}}%[[iterB:.+]] = %[[tB]],{{.*}}%[[iterB1:.+]] = %[[tB1]] -// CHECK-SAME: { - -// Load input values and update the load tile position. -// CHECK: %[[vA:.+]] = xegpu.load_nd %[[iterA]] -// CHECK: %[[vB:.+]] = xegpu.load_nd %[[iterB]] {{.*}}transpose = array{{.*}}transpose_bit_width = 32 : i32{{.*}} -// CHECK: %[[vB1:.+]] = xegpu.load_nd %[[iterB1]] {{.*}}transpose = array, transpose_bit_width = 32 : i32{{.*}} - -// CHECK: %[[new_tA:.+]] = xegpu.update_nd_offset %[[iterA]], [%c0, %c16] -// CHECK: %[[new_tB:.+]] = xegpu.update_nd_offset %[[iterB]], [%c0, %c16] -// CHECK: %[[new_tB1:.+]] = xegpu.update_nd_offset %[[iterB1]], [%c0, %c16] - -// Apply simple prefetching scheme - start loading the next set of input -// tiles before computation is started. -// CHECK: xegpu.prefetch_nd %[[new_tA]] -// CHECK: xegpu.prefetch_nd %[[new_tB]] -// CHECK: xegpu.prefetch_nd %[[new_tB1]] - -// Extract DPAS-sized chunks from larger loaded tile A. -// Tile B is already in the correct shape. -// CHECK: %[[vA_flat:.+]] = vector.shape_cast %[[vA]] : vector<32x16xf16> to vector<512xf16> -// CHECK: %[[vA_dpas_flat:.+]] = vector.extract_strided_slice{{.*}}: vector<512xf16> to vector<128xf16> -// CHECK: %[[vA_dpas:.+]] = vector.shape_cast %[[vA_dpas_flat]] : vector<128xf16> to vector<8x16xf16> -// CHECK-COUNT-3: vector.extract_strided_slice - -// Perform DPAS computation. -// CHECK: %[[dpas:.+]] = xegpu.dpas %[[vA_dpas]], %[[vB]], %[[acc]] -// CHECK-COUNT-7: xegpu.dpas - -// CHECK-NOT: memref.dealloc() diff --git a/test/mlir/test/gc/Transforms/GPU/IMEX/linalg-to-xegpu-dpas-transpose-sep.mlir b/test/mlir/test/gc/Transforms/GPU/IMEX/linalg-to-xegpu-dpas-transpose-sep.mlir deleted file mode 100644 index f7ec70ac9..000000000 --- a/test/mlir/test/gc/Transforms/GPU/IMEX/linalg-to-xegpu-dpas-transpose-sep.mlir +++ /dev/null @@ -1,83 +0,0 @@ -// RUN: gc-opt %s -linalg-to-xegpu="dpas-tile=8,16,16 k-tile=16" -canonicalize -split-input-file | FileCheck %s - -module { - func.func @matmul_transpose_b_sep(%arg0: memref<1024x1024xf16>, %arg1: memref<1024x1024xf16>, %arg2: memref<1024x1024xf16>) { - %c0 = arith.constant 0 : index - %c32 = arith.constant 32 : index - %c1024 = arith.constant 1024 : index - scf.parallel (%arg3, %arg4) = (%c0, %c0) to (%c1024, %c1024) step (%c32, %c32) { - %subview_0 = memref.subview %arg2[%arg3, %arg4] [32, 32] [1, 1] : memref<1024x1024xf16> to memref<32x32xf16, strided<[1024, 1], offset: ?>> - %subview_1 = memref.subview %arg0[%arg3, 0] [32, 1024] [1, 1] : memref<1024x1024xf16> to memref<32x1024xf16, strided<[1024, 1], offset: ?>> - %subview_2 = memref.subview %arg1[%arg4, 0] [32, 1024] [1, 1] : memref<1024x1024xf16> to memref<32x1024xf16, strided<[1024, 1], offset: ?>> - %alloc = memref.alloc() {alignment = 64 : i64} : memref<1024x32xf16> - linalg.transpose ins(%subview_2 : memref<32x1024xf16, strided<[1024, 1], offset: ?>>) outs(%alloc : memref<1024x32xf16>) permutation = [1, 0] - linalg.matmul ins(%subview_1, %alloc : memref<32x1024xf16, strided<[1024, 1], offset: ?>>, memref<1024x32xf16>) outs(%subview_0 : memref<32x32xf16, strided<[1024, 1], offset: ?>>) - memref.dealloc %alloc : memref<1024x32xf16> - scf.reduce - } - return - } -} - -// CHECK-LABEL: func.func @matmul_transpose_b_sep -// CHECK-SAME: %[[Ap:.+]]: memref<1024x1024xf16>, %[[Bp:.+]]: memref<1024x1024xf16>, %[[Cp:.+]]: memref<1024x1024xf16> - -// CHECK: scf.parallel (%[[iter1:.+]], %[[iter2:.+]]) = (%c0, %c0) to (%c1024, %c1024) step (%c32, %c32) { -// CHECK: %[[C:.+]] = memref.subview %[[Cp]][%[[iter1]], %[[iter2]]] {{.*}} -// CHECK: %[[A:.+]] = memref.subview %[[Ap]][%[[iter1]], 0] {{.*}} -// CHECK: %[[B:.+]] = memref.subview %[[Bp]][%[[iter2]], 0] {{.*}} - -// CHECK-NOT: memref.alloc() -// CHECK-NOT: linalg.transpose -// CHECK-NOT: memref.dealloc() - -// Create output initial value load tiles. -// CHECK-DAG: %[[rootC:.+]] = xegpu.create_nd_tdesc %[[C]] -// CHECK: %[[tC:.+]] = xegpu.update_nd_offset %[[rootC]], [%c0, %c0] -// CHECK-COUNT-7: xegpu.update_nd_offset - -// Load initial accumulator values. -// CHECK-DAG: %[[vC:.+]] = xegpu.load_nd %[[tC]] -// CHECK-COUNT-7: xegpu.load_nd - -// Extend the type to match DPAS output precision. -// CHECK: %[[vC_f32:.+]] = arith.extf %[[vC]] -// CHECK-COUNT-7: arith.extf - -// Create input load tiles. -// CHECK: %[[rootA:.+]] = xegpu.create_nd_tdesc %[[A]] -// CHECK: %[[tA:.+]] = xegpu.update_nd_offset %[[rootA]], [%c0, %c0] -// CHECK: %[[rootB:.+]] = xegpu.create_nd_tdesc %[[B]] -// CHECK: %[[tB:.+]] = xegpu.update_nd_offset %[[rootB]], [%c0, %c0] -// CHECK: %[[tB1:.+]] = xegpu.update_nd_offset %[[rootB]], [%c16, %c0] - -// Create DPAS computation loop over tiled reduction dimension. -// CHECK: %[[res:.+]]:11 = scf.for{{.*}}%c0 to %c1024 step %c16 -// CHECK-SAME: iter_args(%[[acc:.+]] = %[[vC_f32]],{{.*}}%[[iterA:.+]] = %[[tA]],{{.*}}%[[iterB:.+]] = %[[tB]],{{.*}}%[[iterB1:.+]] = %[[tB1]] -// CHECK-SAME: { - -// Load input values and update the load tile position. -// CHECK: %[[vA:.+]] = xegpu.load_nd %[[iterA]] -// CHECK: %[[vB:.+]] = xegpu.load_nd %[[iterB]] {{.*}}transpose = array{{.*}}transpose_bit_width = 32 : i32{{.*}} -// CHECK: %[[vB1:.+]] = xegpu.load_nd %[[iterB1]] {{.*}}transpose = array, transpose_bit_width = 32 : i32{{.*}} - -// CHECK: %[[new_tA:.+]] = xegpu.update_nd_offset %[[iterA]], [%c0, %c16] -// CHECK: %[[new_tB:.+]] = xegpu.update_nd_offset %[[iterB]], [%c0, %c16] -// CHECK: %[[new_tB1:.+]] = xegpu.update_nd_offset %[[iterB1]], [%c0, %c16] - -// Apply simple prefetching scheme - start loading the next set of input -// tiles before computation is started. -// CHECK: xegpu.prefetch_nd %[[new_tA]] -// CHECK: xegpu.prefetch_nd %[[new_tB]] -// CHECK: xegpu.prefetch_nd %[[new_tB1]] - -// Extract DPAS-sized chunks from larger loaded tile A. -// Tile B is already in the correct shape. -// CHECK: %[[vA_flat:.+]] = vector.shape_cast %[[vA]] : vector<32x16xf16> to vector<512xf16> -// CHECK: %[[vA_dpas_flat:.+]] = vector.extract_strided_slice{{.*}}: vector<512xf16> to vector<128xf16> -// CHECK: %[[vA_dpas:.+]] = vector.shape_cast %[[vA_dpas_flat]] : vector<128xf16> to vector<8x16xf16> -// CHECK-COUNT-3: vector.extract_strided_slice - -// Perform DPAS computation. -// CHECK: %[[dpas:.+]] = xegpu.dpas %[[vA_dpas]], %[[vB]], %[[acc]] -// CHECK-COUNT-7: xegpu.dpas diff --git a/test/mlir/test/gc/Transforms/GPU/IMEX/linalg-to-xegpu-dpas-transpose.mlir b/test/mlir/test/gc/Transforms/GPU/IMEX/linalg-to-xegpu-dpas-transpose.mlir deleted file mode 100644 index fa7ce0252..000000000 --- a/test/mlir/test/gc/Transforms/GPU/IMEX/linalg-to-xegpu-dpas-transpose.mlir +++ /dev/null @@ -1,79 +0,0 @@ -// RUN: gc-opt %s -linalg-to-xegpu="dpas-tile=8,16,16 k-tile=16" -canonicalize -split-input-file | FileCheck %s - -module { - func.func @matmul_transpose_b(%arg0: memref<1024x1024xf16>, %arg1: memref<1024x1024xf16>, %arg2: memref<1024x1024xf16>) { - %c0 = arith.constant 0 : index - %c16 = arith.constant 16 : index - %c64 = arith.constant 64 : index - %c1024 = arith.constant 1024 : index - scf.parallel (%arg3, %arg4) = (%c0, %c0) to (%c1024, %c1024) step (%c16, %c64) { - %subview_0 = memref.subview %arg2[%arg3, %arg4] [16, 64] [1, 1] : memref<1024x1024xf16> to memref<16x64xf16, strided<[1024, 1], offset: ?>> - %subview_1 = memref.subview %arg0[%arg3, 0] [16, 1024] [1, 1] : memref<1024x1024xf16> to memref<16x1024xf16, strided<[1024, 1], offset: ?>> - %subview_2 = memref.subview %arg1[%arg4, 0] [64, 1024] [1, 1] : memref<1024x1024xf16> to memref<64x1024xf16, strided<[1024, 1], offset: ?>> - linalg.matmul_transpose_b ins(%subview_1, %subview_2 : memref<16x1024xf16, strided<[1024, 1], offset: ?>>, memref<64x1024xf16, strided<[1024, 1], offset: ?>>) outs(%subview_0 : memref<16x64xf16, strided<[1024, 1], offset: ?>>) - scf.reduce - } - return - } -} - -// CHECK-LABEL: func.func @matmul_transpose_b -// CHECK-SAME: %[[Ap:.+]]: memref<1024x1024xf16>, %[[Bp:.+]]: memref<1024x1024xf16>, %[[Cp:.+]]: memref<1024x1024xf16> - -// CHECK: scf.parallel (%[[iter1:.+]], %[[iter2:.+]]) = (%c0, %c0) to (%c1024, %c1024) step (%c16, %c64) { -// CHECK: %[[C:.+]] = memref.subview %[[Cp]][%[[iter1]], %[[iter2]]] {{.*}} -// CHECK: %[[A:.+]] = memref.subview %[[Ap]][%[[iter1]], 0] {{.*}} -// CHECK: %[[B:.+]] = memref.subview %[[Bp]][%[[iter2]], 0] {{.*}} - -// Create output initial value load tiles. -// CHECK-DAG: %[[rootC:.+]] = xegpu.create_nd_tdesc %[[C]] -// CHECK: %[[tC:.+]] = xegpu.update_nd_offset %[[rootC]], [%c0, %c0] -// CHECK-COUNT-7: xegpu.update_nd_offset - -// Load initial accumulator values. -// CHECK-DAG: %[[vC:.+]] = xegpu.load_nd %[[tC]] -// CHECK-COUNT-7: xegpu.load_nd - -// Extend the type to match DPAS output precision. -// CHECK: %[[vC_f32:.+]] = arith.extf %[[vC]] -// CHECK-COUNT-7: arith.extf - -// Create input load tiles. -// CHECK: %[[rootA:.+]] = xegpu.create_nd_tdesc %[[A]] -// CHECK: %[[tA:.+]] = xegpu.update_nd_offset %[[rootA]], [%c0, %c0] -// CHECK: %[[rootB:.+]] = xegpu.create_nd_tdesc %[[B]] -// CHECK: %[[tB:.+]] = xegpu.update_nd_offset %[[rootB]], [%c0, %c0] -// CHECK: %[[tB1:.+]] = xegpu.update_nd_offset %[[rootB]], [%c16, %c0] -// CHECK: %[[tB2:.+]] = xegpu.update_nd_offset %[[rootB]], [%c32, %c0] -// CHECK: %[[tB3:.+]] = xegpu.update_nd_offset %[[rootB]], [%c48, %c0] - -// Create DPAS computation loop over tiled reduction dimension. -// CHECK: %[[res:.+]]:13 = scf.for{{.*}}%c0 to %c1024 step %c16 -// CHECK-SAME: iter_args(%[[acc:.+]] = %[[vC_f32]],{{.*}}%[[iterA:.+]] = %[[tA]],{{.*}}%[[iterB:.+]] = %[[tB]],{{.*}}%[[iterB1:.+]] = %[[tB1]] -// CHECK-SAME: { - -// Load input values and update the load tile position. -// CHECK: %[[vA:.+]] = xegpu.load_nd %[[iterA]] -// CHECK: %[[vB:.+]] = xegpu.load_nd %[[iterB]] {{.*}}transpose = array{{.*}}transpose_bit_width = 32 : i32{{.*}} -// CHECK: %[[vB1:.+]] = xegpu.load_nd %[[iterB1]] {{.*}}transpose = array, transpose_bit_width = 32 : i32{{.*}} - -// CHECK: %[[new_tA:.+]] = xegpu.update_nd_offset %[[iterA]], [%c0, %c16] -// CHECK: %[[new_tB:.+]] = xegpu.update_nd_offset %[[iterB]], [%c0, %c16] -// CHECK: %[[new_tB1:.+]] = xegpu.update_nd_offset %[[iterB1]], [%c0, %c16] - -// Apply simple prefetching scheme - start loading the next set of input -// tiles before computation is started. -// CHECK: xegpu.prefetch_nd %[[new_tA]] -// CHECK: xegpu.prefetch_nd %[[new_tB]] -// CHECK: xegpu.prefetch_nd %[[new_tB1]] - -// Extract DPAS-sized chunks from larger loaded tile A. -// Tile B is already in the correct shape. -// CHECK: %[[vA_flat:.+]] = vector.shape_cast %[[vA]] : vector<16x16xf16> to vector<256xf16> -// CHECK: %[[vA_dpas_flat:.+]] = vector.extract_strided_slice{{.*}}: vector<256xf16> to vector<128xf16> -// CHECK: %[[vA_dpas:.+]] = vector.shape_cast %[[vA_dpas_flat]] : vector<128xf16> to vector<8x16xf16> -// CHECK-COUNT-1: vector.extract_strided_slice - -// Perform DPAS computation. -// CHECK: %[[dpas:.+]] = xegpu.dpas %[[vA_dpas]], %[[vB]], %[[acc]] -// CHECK-COUNT-7: xegpu.dpas diff --git a/test/mlir/test/gc/Transforms/GPU/IMEX/linalg-to-xegpu-dpas.mlir b/test/mlir/test/gc/Transforms/GPU/IMEX/linalg-to-xegpu-dpas.mlir deleted file mode 100644 index 0564fb27d..000000000 --- a/test/mlir/test/gc/Transforms/GPU/IMEX/linalg-to-xegpu-dpas.mlir +++ /dev/null @@ -1,87 +0,0 @@ -// RUN: gc-opt %s -linalg-to-xegpu="dpas-tile=8,16,16 k-tile=16" -canonicalize -split-input-file | FileCheck %s - -func.func @matmul(%arg0: memref<32x32xf16>, %arg1: memref<32x32xf16>, %arg2: memref<32x32xf16>) { - %c1 = arith.constant 1 : index - gpu.launch blocks(%arg3, %arg4, %arg5) in (%arg9 = %c1, %arg10 = %c1, %arg11 = %c1) threads(%arg6, %arg7, %arg8) in (%arg12 = %c1, %arg13 = %c1, %arg14 = %c1) { - linalg.matmul ins(%arg0, %arg1 : memref<32x32xf16>, memref<32x32xf16>) - outs(%arg2 : memref<32x32xf16>) - gpu.terminator - } - return -} - -// CHECK-LABEL: func.func @matmul -// CHECK-SAME: %[[A:.+]]: memref<32x32xf16>, %[[B:.+]]: memref<32x32xf16>, %[[C:.+]]: memref<32x32xf16> -// CHECK-DAG: %[[c0:.+]] = arith.constant 0 -// CHECK-DAG: %[[c16:.+]] = arith.constant 16 -// CHECK-DAG: %[[c32:.+]] = arith.constant 32 - -// Create output initial value load tiles. -// CHECK: %[[rootC:.+]] = xegpu.create_nd_tdesc %[[C]] -// CHECK: %[[tC:.+]] = xegpu.update_nd_offset %[[rootC]], [%c0, %c0] -// CHECK-COUNT-7: xegpu.update_nd_offset %[[rootC]] - -// Load initial accumulator values. -// CHECK: %[[vC:.+]] = xegpu.load_nd %[[tC]] -// CHECK-COUNT-7: xegpu.load_nd - -// Extend the type to match DPAS output precision. -// CHECK: %[[vC_f32:.+]] = arith.extf %[[vC]] -// CHECK-COUNT-7: arith.extf - -// Create input load tiles. -// CHECK: %[[rootA:.+]] = xegpu.create_nd_tdesc %[[A]] -// CHECK: %[[tA:.+]] = xegpu.update_nd_offset %[[rootA]], [%c0, %c0] -// CHECK: %[[rootB:.+]] = xegpu.create_nd_tdesc %[[B]] -// CHECK: %[[tB:.+]] = xegpu.update_nd_offset %[[rootB]], [%c0, %c0] -// CHECK-COUNT-1: xegpu.update_nd_offset %[[rootB]] - -// Create DPAS computation loop over tiled reduction dimension. -// CHECK: %[[res:.+]]:11 = scf.for{{.*}}%[[c0]] to %[[c32]] step %[[c16]] -// CHECK-SAME: iter_args(%[[acc:.+]] = %[[vC_f32]],{{.*}}%[[iterA:.+]] = %[[tA]],{{.*}}%[[iterB:.+]] = %[[tB]] -// CHECK-SAME: { - -// Periodically synchronize the workgroup. -// CHECK: scf.if -// CHECK-SAME: { -// CHECK: gpu.barrier -// CHECK: } - -// Load input values and update the load tile position. -// CHECK: %[[vA:.+]] = xegpu.load_nd %[[iterA]] -// CHECK: %[[vB:.+]] = xegpu.load_nd %[[iterB]] -// CHECK-COUNT-1: xegpu.load_nd -// CHECK: %[[new_tA:.+]] = xegpu.update_nd_offset %[[iterA]] -// CHECK: %[[new_tB:.+]] = xegpu.update_nd_offset %[[iterB]] -// CHECK-COUNT-1: xegpu.update_nd_offset - -// Apply simple prefetching scheme - start loading the next set of input -// tiles before computation is started. -// CHECK: xegpu.prefetch_nd %[[new_tA]] -// CHECK: xegpu.prefetch_nd %[[new_tB]] -// CHECK-COUNT-1: xegpu.prefetch_nd - -// Extract DPAS-sized chunks from larger loaded tile A. -// Tile B is already in the correct shape. -// CHECK: %[[vA_flat:.+]] = vector.shape_cast %[[vA]] : vector<32x16xf16> to vector<512xf16> -// CHECK: %[[vA_dpas_flat:.+]] = vector.extract_strided_slice{{.*}}: vector<512xf16> to vector<128xf16> -// CHECK: %[[vA_dpas:.+]] = vector.shape_cast %[[vA_dpas_flat]] : vector<128xf16> to vector<8x16xf16> -// CHECK-COUNT-3: vector.extract_strided_slice - -// Perform DPAS computation. -// CHECK: %[[dpas:.+]] = xegpu.dpas %[[vA_dpas]], %[[vB]], %[[acc]] -// CHECK-COUNT-7: xegpu.dpas - -// Yield the results to the next iteration. -// CHECK: scf.yield %[[dpas]],{{.*}}%[[new_tA]],{{.*}}%[[new_tB]] -// CHECK: } - -// Truncate results to the original output precision. -// CHECK: %[[res_f16:.+]] = arith.truncf %[[res]]#0 -// CHECK-COUNT-7: arith.truncf - -// Store back the final results. -// CHECH: xegpu.store_nd %[[res_f16]], %[[tC]] -// CHECK-COUNT-7: xegpu.store_nd - -// CHECK: gpu.terminator diff --git a/test/mlir/test/gc/Transforms/GPU/IMEX/linalg-to-xegpu-slm.mlir b/test/mlir/test/gc/Transforms/GPU/IMEX/linalg-to-xegpu-slm.mlir deleted file mode 100644 index 8b7e4646a..000000000 --- a/test/mlir/test/gc/Transforms/GPU/IMEX/linalg-to-xegpu-slm.mlir +++ /dev/null @@ -1,91 +0,0 @@ -// RUN: gc-opt %s -linalg-to-xegpu="dpas-tile=8,16,16 k-tile=16" -canonicalize -split-input-file -cse | FileCheck %s - -#map = affine_map<(d0) -> (d0 * 64)> -#map1 = affine_map<(d0) -> (d0 * 16)> - -func.func @entry(%arg0: memref<128x1024xf16>, %arg1: memref<1024x1024xf16>, %arg2: memref<128x1024xf16>) { - // CHECK: %[[loadAccumMatmul:.+]] = arith.constant dense<0.000000e+00> : vector<4x32xf16> - // CHECK: %[[ZERO:.+]] = arith.constant dense<0.000000e+00> : vector<32xf16> - // CHECK: %[[colTileShift:.+]] = arith.constant dense<[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271]> : vector<32xindex> - // CHECK: %[[loadOffset:.+]] = arith.constant dense<512> : vector<32xindex> - %cst = arith.constant 0.000000e+00 : f16 - %c1 = arith.constant 1 : index - %c2 = arith.constant 2 : index - %c4 = arith.constant 4 : index - %c16 = arith.constant 16 : index - gpu.launch blocks(%arg3, %arg4, %arg5) in (%arg11 = %c2, %arg12 = %c4, %arg13 = %c1) threads(%arg6, %arg7, %arg8) in (%arg14 = %c4, %arg15 = %c16, %arg16 = %c1) { - %x_group_idx = affine.apply #map(%arg3) - %y_group_idx = affine.apply #map(%arg4) - - // CHECK: %[[X_THREAD_IDX:.+]] = affine.apply #map1(%arg6) - // CHECK: %[[Y_THREAD_IDX:.+]] = affine.apply #map1(%arg7) - %x_thread_idx = affine.apply #map1(%arg6) - %y_thread_idx = affine.apply #map1(%arg7) - - %x_global_idx = arith.addi %x_group_idx, %x_thread_idx : index - %y_global_idx = arith.addi %y_group_idx, %y_thread_idx : index - - %a_subview = memref.subview %arg0[%x_global_idx, 0] [16, 1024] [1, 1] : memref<128x1024xf16> to memref<16x1024xf16, strided<[1024, 1], offset: ?>> - %b_subview = memref.subview %arg1[0, %y_global_idx] [1024, 16] [1, 1] : memref<1024x1024xf16> to memref<1024x16xf16, strided<[1024, 1], offset: ?>> - - // CHECK: %[[SLM_BUFF:.+]] = memref.alloc() : memref<64x256xf16, 3> - %slm_buff = memref.alloc() : memref<64x256xf16, 3> - // CHECK-NOT: .* = memref.subview %[[SLM_BUFF]] .* - // CHECK: %[[SLM_X_OFF:.+]] = arith.muli %[[X_THREAD_IDX]], %c256 : index - // CHECK: %[[SLM_THREAD_OFF:.+]] = arith.addi %[[SLM_X_OFF]], %[[Y_THREAD_IDX]] : index - // CHECK: %[[FLAT_SLM:.+]] = memref.reinterpret_cast %[[SLM_BUFF]] to offset: [%c0], sizes: [%c16384], strides: [%c1] : memref<64x256xf16, 3> to memref<16384xf16, 3> - %slm_subview = memref.subview %slm_buff[%x_thread_idx, %y_thread_idx] [16, 16] [1, 1] : memref<64x256xf16, 3> to memref<16x16xf16, strided<[256, 1], offset: ?>, 3> - - // CHECK: %[[SLM_THREAD_OFF_V:.+]] = vector.splat %[[SLM_THREAD_OFF]] : vector<32xindex> - // CHECK: %[[DESC_OFFSET0:.+]] = arith.addi %[[SLM_THREAD_OFF_V]], %[[colTileShift]] : vector<32xindex> - // CHECK: %[[ROOT_DESC:.+]] = xegpu.create_tdesc %[[FLAT_SLM]], %[[DESC_OFFSET0]] : memref<16384xf16, 3>, vector<32xindex> -> !xegpu.tensor_desc<32xf16, #xegpu.scatter_tdesc_attr> - // CHECK: %[[FILL_DESC1:.+]] = xegpu.update_offset %[[ROOT_DESC]], %[[loadOffset]] - // CHECK: %[[FILL_DESC2:.+]] = xegpu.update_offset %[[FILL_DESC1]], %[[loadOffset]] - // CHECK-COUNT-5: xegpu.update_offset - - // CHECK: xegpu.store %[[ZERO]], %[[ROOT_DESC]] - // CHECK: xegpu.store %[[ZERO]], %[[FILL_DESC1]] - // CHECK-COUNT-6: xegpu.store - linalg.fill ins(%cst : f16) outs(%slm_subview : memref<16x16xf16, strided<[256, 1], offset: ?>, 3>) - - // CHECK: %[[MATMUL_DESC1:.+]] = xegpu.update_offset %[[ROOT_DESC]], %[[loadOffset]] - // CHECK: %[[MATMUL_DESC2:.+]] = xegpu.update_offset %[[MATMUL_DESC1]], %[[loadOffset]] - // CHECK-COUNT-5: xegpu.update_offset - - // CHECK: %[[MATMUL_LOAD0:.+]] = xegpu.load %[[ROOT_DESC]] - // CHECK-NEXT: %[[loadAccumMatmul1:.+]] = vector.insert %[[MATMUL_LOAD0]], %[[loadAccumMatmul]] [0] - // CHECK-NEXT: %[[MATMUL_LOAD1:.+]] = xegpu.load %[[MATMUL_DESC1]] - // CHECK-NEXT: %[[loadAccumMatmul2:.+]] = vector.insert %[[MATMUL_LOAD1]], %[[loadAccumMatmul1]] [1] - // CHECK-COUNT-2: xegpu.load - - // CHECK: vector.shape_cast - // CHECK-SAME: vector<4x32xf16> to vector<128xf16> - // CHECK: vector.shape_cast - // CHECK-SAME: vector<128xf16> to vector<8x16xf16> - - // CHECK-COUNT-4: xegpu.load - // CHECK: vector.shape_cast - // CHECK-SAME: vector<4x32xf16> to vector<128xf16> - // CHECK: vector.shape_cast - // CHECK-SAME: vector<128xf16> to vector<8x16xf16> - - // STORE: - // %[[FLAT_MATMUL_RES0:.+]] = vector.shape_cast %[[MATMUL_RES0:.+]] : vector<8x16xf16> to vector<128xf16> - // %[[STORE_TILE0:.+]] = vector.extract_strided_slice %[[FLAT_MATMUL_RES0]] {offsets = [0], sizes = [32], strides = [1]} : vector<128xf16> to vector<32xf16> - // xegpu.store %[[STORE_TILE0]], %[[ROOT_DESC]] - // %[[STORE_TILE1:.+]] = vector.extract_strided_slice %[[FLAT_MATMUL_RES0]] {offsets = [32], sizes = [32], strides = [1]} : vector<128xf16> to vector<32xf16> - // xegpu.store %[[STORE_TILE0]], %[[MATMUL_DESC1]] - // CHECK-COUNT-2: xegpu.store - - // %[[FLAT_MATMUL_RES1:.+]] = vector.shape_cast %[[MATMUL_RES1:.+]] : vector<8x16xf16> to vector<128xf16> - // %[[STORE_TILE1_0:.+]] = vector.extract_strided_slice %[[FLAT_MATMUL_RES1]] {offsets = [0], sizes = [32], strides = [1]} : vector<128xf16> to vector<32xf16> - // xegpu.store %[[STORE_TILE1_0]] - // %[[STORE_TILE1_1:.+]] = vector.extract_strided_slice %[[FLAT_MATMUL_RES1]] {offsets = [32], sizes = [32], strides = [1]} : vector<128xf16> to vector<32xf16> - // xegpu.store %[[STORE_TILE1_1]] - // CHECK-COUNT-2: xegpu.store - - linalg.matmul ins(%a_subview, %b_subview : memref<16x1024xf16, strided<[1024, 1], offset: ?>>, memref<1024x16xf16, strided<[1024, 1], offset: ?>>) outs(%slm_subview : memref<16x16xf16, strided<[256, 1], offset: ?>, 3>) - gpu.terminator - } - return -} diff --git a/test/mlir/test/gc/Transforms/GPU/IMEX/linalg-to-xegpu-squeeze.mlir b/test/mlir/test/gc/Transforms/GPU/IMEX/linalg-to-xegpu-squeeze.mlir deleted file mode 100644 index 362867fb2..000000000 --- a/test/mlir/test/gc/Transforms/GPU/IMEX/linalg-to-xegpu-squeeze.mlir +++ /dev/null @@ -1,61 +0,0 @@ -// RUN: gc-opt %s -linalg-to-xegpu="dpas-tile=8,16,16 k-tile=16" -canonicalize -split-input-file -cse | FileCheck %s - -!input_type = memref<2x4x8x16xf16> -!chunk_type = memref<1x1x8x16xf16, strided<[512, 128, 16, 1], offset: ?>> -!slm_chunk = memref<1x1x8x16xf16, strided<[128, 128, 16, 1], offset: ?>, 3> - -// The map that computes an offset for SLM -// CHECK: #map = affine_map<(d0, d1) -> (d0 * 4 + d1)> -#map = affine_map<(xi, yi) -> (xi * 4 + yi)> - -func.func @entry(%arg0: !input_type, %arg1: !input_type, %arg2: !input_type) { - %c1 = arith.constant 1 : index - %c2 = arith.constant 2 : index - %c4 = arith.constant 4 : index - - gpu.launch blocks(%arg3, %arg4, %arg5) in (%arg9 = %c1, %arg10 = %c1, %arg11 = %c1) threads(%arg6, %arg7, %arg8) in (%arg12 = %c2, %arg13 = %c4, %arg14 = %c1) { - // CHECK: %[[ARG0_SB:.+]] = memref.subview %arg0[%arg6, %arg7, 0, 0] [1, 1, 8, 16] [1, 1, 1, 1] - %arg0_sb = memref.subview %arg0[%arg6, %arg7, 0, 0] [1, 1, 8, 16] [1, 1, 1, 1] : !input_type to !chunk_type - // CHECK: %[[ARG1_SB:.+]] = memref.subview %arg1[%arg6, %arg7, 0, 0] [1, 1, 8, 16] [1, 1, 1, 1] - %arg1_sb = memref.subview %arg1[%arg6, %arg7, 0, 0] [1, 1, 8, 16] [1, 1, 1, 1] : !input_type to !chunk_type - // CHECK: %[[ARG2_SB:.+]] = memref.subview %arg2[%arg6, %arg7, 0, 0] [1, 1, 8, 16] [1, 1, 1, 1] - %arg2_sb = memref.subview %arg2[%arg6, %arg7, 0, 0] [1, 1, 8, 16] [1, 1, 1, 1] : !input_type to !chunk_type - - // CHECK: %[[SLM_BUFF:.+]] = memref.alloc() : memref<8x1x8x16xf16, 3> - %slm_root = memref.alloc() : memref<8x1x8x16xf16, 3> - - %slm_idx = affine.apply #map(%arg6, %arg7) - %slm = memref.subview %slm_root[%slm_idx, 0, 0, 0] [1, 1, 8, 16] [1, 1, 1, 1] : memref<8x1x8x16xf16, 3> to !slm_chunk - - // Squeezing the arguments of 'linalg.mul' - // CHECK: %[[ARG0_SQUEEZ:.+]] = memref.subview %[[ARG0_SB]][0, 0, 0, 0] [1, 1, 8, 16] [1, 1, 1, 1] : - // CHECK-SAME: memref<1x1x8x16xf16, strided<[512, 128, 16, 1], offset: ?>> to memref<8x16xf16, strided<[16, 1], offset: ?>> - - // CHECK: %[[ARG1_SQUEEZ:.+]] = memref.subview %[[ARG1_SB]][0, 0, 0, 0] [1, 1, 8, 16] [1, 1, 1, 1] : - // CHECK-SAME: memref<1x1x8x16xf16, strided<[512, 128, 16, 1], offset: ?>> to memref<8x16xf16, strided<[16, 1], offset: ?>> - - // Verify that tensor descriptors are created from the squeezed memrefs - // CHECK: xegpu.create_nd_tdesc %[[ARG0_SQUEEZ]] - // CHECK: xegpu.create_nd_tdesc %[[ARG1_SQUEEZ]] - - // Verify that the SLM output of linalg.mul is squeezed correctly - // CHECK-NOT: .* = memref.subview %[[SLM_BUFF]] .* - // CHECK: %[[SLM_THREAD_OFF:.+]] = affine.apply #map(%arg6, %arg7) - // CHECK: %[[SLM_OFF:.+]] = arith.muli %[[SLM_THREAD_OFF]], %c128 : index - // CHECK: %[[FLAT_SLM:.+]] = memref.reinterpret_cast %[[SLM_BUFF]] to offset: [%c0], sizes: [%c1024], strides: [%c1] : memref<8x1x8x16xf16, 3> to memref<1024xf16, 3> - // CHECK: xegpu.create_tdesc %[[FLAT_SLM]] - linalg.mul ins(%arg0_sb, %arg1_sb : !chunk_type, !chunk_type) outs(%slm : !slm_chunk) - - // Squeezing the result buffer of 'linalg.add' - // CHECK: %[[ARG2_SQUEEZ:.+]] = memref.subview %[[ARG2_SB]][0, 0, 0, 0] [1, 1, 8, 16] [1, 1, 1, 1] : - // CHECK-SAME: memref<1x1x8x16xf16, strided<[512, 128, 16, 1], offset: ?>> to memref<8x16xf16, strided<[16, 1], offset: ?>> - - // Verify that tensor descriptors are created from the squeezed memrefs - // CHECK: xegpu.create_nd_tdesc %[[ARG2_SQUEEZ]] - linalg.add ins(%arg0_sb, %slm : !chunk_type, !slm_chunk) outs(%arg2_sb : !chunk_type) - - gpu.terminator - } {SCFToGPU_visited} - - return -} diff --git a/test/mlir/test/gc/Transforms/GPU/IMEX/linalg-to-xegpu-stages.mlir b/test/mlir/test/gc/Transforms/GPU/IMEX/linalg-to-xegpu-stages.mlir deleted file mode 100644 index 0f59221bb..000000000 --- a/test/mlir/test/gc/Transforms/GPU/IMEX/linalg-to-xegpu-stages.mlir +++ /dev/null @@ -1,53 +0,0 @@ -// RUN: gc-opt %s -linalg-to-xegpu="dpas-tile=8,16,16 stages=1" -canonicalize -split-input-file | FileCheck %s --check-prefix=STAGES-1 - -// RUN: gc-opt %s -linalg-to-xegpu="dpas-tile=8,16,16 stages=2" -canonicalize -split-input-file | FileCheck %s --check-prefix=STAGES-2 - -#map = affine_map<()[s0, s1] -> (s0 + s1)> -module { - func.func @matmul_multistage_coop_prefetch(%arg0: memref<1024x1024xf16>, %arg1: memref<1024x1024xf16>, %arg2: memref<1024x1024xf16>) { - %c32 = arith.constant 32 : index - %c0 = arith.constant 0 : index - %c1024 = arith.constant 1024 : index - %c128 = arith.constant 128 : index - scf.parallel (%arg3, %arg4) = (%c0, %c0) to (%c1024, %c1024) step (%c128, %c128) { - %subview = memref.subview %arg2[%arg3, %arg4] [128, 128] [1, 1] : memref<1024x1024xf16> to memref<128x128xf16, strided<[1024, 1], offset: ?>> - scf.parallel (%arg5, %arg6) = (%c0, %c0) to (%c128, %c128) step (%c32, %c32) { - %subview_0 = memref.subview %subview[%arg5, %arg6] [32, 32] [1, 1] : memref<128x128xf16, strided<[1024, 1], offset: ?>> to memref<32x32xf16, strided<[1024, 1], offset: ?>> - %0 = affine.apply #map()[%arg5, %arg3] - %subview_1 = memref.subview %arg0[%0, 0] [32, 1024] [1, 1] : memref<1024x1024xf16> to memref<32x1024xf16, strided<[1024, 1], offset: ?>> - %1 = affine.apply #map()[%arg6, %arg4] - %subview_2 = memref.subview %arg1[0, %1] [1024, 32] [1, 1] : memref<1024x1024xf16> to memref<1024x32xf16, strided<[1024, 1], offset: ?>> - linalg.matmul ins(%subview_1, %subview_2 : memref<32x1024xf16, strided<[1024, 1], offset: ?>>, memref<1024x32xf16, strided<[1024, 1], offset: ?>>) outs(%subview_0 : memref<32x32xf16, strided<[1024, 1], offset: ?>>) - scf.reduce - } - scf.reduce - } - return - } -} - -// STAGES-1-LABEL: func.func @matmul_multistage_coop_prefetch -// STAGES-1-SAME: %[[A:.+]]: memref<1024x1024xf16>, %[[B:.+]]: memref<1024x1024xf16>, %[[C:.+]]: memref<1024x1024xf16> -// STAGES-1: %[[s1_A:.+]] = xegpu.create_nd_tdesc %[[A]] -// STAGES-1: %[[s1_B:.+]] = xegpu.create_nd_tdesc %[[B]] -// STAGES-1: xegpu.prefetch_nd %[[s1_A]] -// STAGES-1: xegpu.prefetch_nd %[[s1_B]] -// STAGES-1: %[[loop_pref_A:.+]] = xegpu.update_nd_offset %[[s1_A]] -// STAGES-1: %[[loop_pref_B:.+]] = xegpu.update_nd_offset %[[s1_B]] -// STAGES-1-NOT: xegpu.prefetch_nd -// STAGES-1: scf.for - -// STAGES-2-LABEL: func.func @matmul_multistage_coop_prefetch -// STAGES-2-SAME: %[[A:.+]]: memref<1024x1024xf16>, %[[B:.+]]: memref<1024x1024xf16>, %[[C:.+]]: memref<1024x1024xf16> -// STAGES-2: %[[s1_A:.+]] = xegpu.create_nd_tdesc %[[A]] -// STAGES-2: %[[s1_B:.+]] = xegpu.create_nd_tdesc %[[B]] -// STAGES-2: xegpu.prefetch_nd %[[s1_A]] -// STAGES-2: xegpu.prefetch_nd %[[s1_B]] -// STAGES-2: %[[s2_A:.+]] = xegpu.update_nd_offset %[[s1_A]] -// STAGES-2: %[[s2_B:.+]] = xegpu.update_nd_offset %[[s1_B]] -// STAGES-2: xegpu.prefetch_nd %[[s2_A]] -// STAGES-2: xegpu.prefetch_nd %[[s2_B]] -// STAGES-2: %[[loop_pref_A:.+]] = xegpu.update_nd_offset %[[s2_A]] -// STAGES-2: %[[loop_pref_B:.+]] = xegpu.update_nd_offset %[[s2_B]] -// STAGES-2-NOT: xegpu.prefetch_nd -// STAGES-2: scf.for diff --git a/test/mlir/test/gc/Transforms/GPU/IMEX/linalg-to-xegpu.mlir b/test/mlir/test/gc/Transforms/GPU/IMEX/linalg-to-xegpu.mlir deleted file mode 100644 index 0f0ab3c2b..000000000 --- a/test/mlir/test/gc/Transforms/GPU/IMEX/linalg-to-xegpu.mlir +++ /dev/null @@ -1,217 +0,0 @@ -// RUN: gc-opt %s -linalg-to-xegpu="dpas-tile=8,16,16 k-tile=16" -canonicalize -split-input-file | FileCheck %s - -func.func @matmul(%arg0: memref<8x16xf16>, %arg1: memref<16x16xf16>, %arg2: memref<8x16xf32>) { - linalg.matmul ins(%arg0, %arg1 : memref<8x16xf16>, memref<16x16xf16>) - outs(%arg2 : memref<8x16xf32>) - return -} - -// CHECK-LABEL: func.func @matmul -// CHECK-COUNT-3: xegpu.load_nd -// CHECK: xegpu.dpas -// CHECH: xegpu.store_nd - -// ----- - -#map = affine_map<(d0, d1, d2) -> (d0, d2)> -#map1 = affine_map<(d0, d1, d2) -> (d2, d1)> -#map2 = affine_map<(d0, d1, d2) -> (d0, d1)> -module { - func.func @generic_matmul(%arg0: memref<8x16xf16>, %arg1: memref<16x16xf16>, %arg2: memref<8x16xf16>) { - linalg.generic {indexing_maps = [#map, #map1, #map2], iterator_types = ["parallel", "parallel", "reduction"]} ins(%arg0, %arg1 : memref<8x16xf16>, memref<16x16xf16>) outs(%arg2 : memref<8x16xf16>) { - ^bb0(%in: f16, %in_0: f16, %out: f16): - %0 = arith.mulf %in, %in_0 : f16 - %1 = arith.addf %out, %0 : f16 - linalg.yield %1 : f16 - } - return - } -} - -// CHECK-LABEL: func.func @generic_matmul -// CHECK-COUNT-3: xegpu.load_nd -// CHECK: xegpu.dpas -// CHECH: xegpu.store_nd - -// ----- - -func.func @matmul_trunc_result(%arg0: memref<8x16xf16>, %arg1: memref<16x16xf16>, %arg2: memref<8x16xf16>) { - linalg.matmul ins(%arg0, %arg1 : memref<8x16xf16>, memref<16x16xf16>) - outs(%arg2 : memref<8x16xf16>) - return -} - -// CHECK-LABEL: func.func @matmul_trunc_result -// CHECK: arith.extf -// CHECK: xegpu.dpas -// CHECK: arith.truncf -// CHECH: xegpu.store_nd - -// ----- - -func.func @abs(%arg0: memref<8x16xf16>, %arg1: memref<8x16xf16>) { - linalg.abs ins(%arg0 : memref<8x16xf16>) - outs(%arg1 : memref<8x16xf16>) - return -} - -// CHECK-LABEL: func.func @abs -// CHECK-COUNT-1: xegpu.load_nd -// CHECK: math.absf -// CHECK: xegpu.store_nd - -// ----- - -func.func @add(%arg0: memref<8x16xf16>, %arg1: memref<8x16xf16>, %arg2: memref<8x16xf16>) { - linalg.add ins(%arg0, %arg1 : memref<8x16xf16>, memref<8x16xf16>) - outs(%arg2 : memref<8x16xf16>) - return -} - -// CHECK-LABEL: func.func @add -// CHECK-COUNT-2: xegpu.load_nd -// CHECK: arith.addf -// CHECK: xegpu.store_nd - -// ----- - -func.func @ceil(%arg0: memref<8x16xf16>, %arg1: memref<8x16xf16>) { - linalg.ceil ins(%arg0 : memref<8x16xf16>) - outs(%arg1 : memref<8x16xf16>) - return -} - -// CHECK-LABEL: func.func @ceil -// CHECK-COUNT-1: xegpu.load_nd -// CHECK: math.ceil -// CHECK: xegpu.store_nd - -// ----- - -func.func @div(%arg0: memref<8x16xf16>, %arg1: memref<8x16xf16>, %arg2: memref<8x16xf16>) { - linalg.div ins(%arg0, %arg1 : memref<8x16xf16>, memref<8x16xf16>) - outs(%arg2 : memref<8x16xf16>) - return -} - -// CHECK-LABEL: func.func @div -// CHECK-COUNT-2: xegpu.load_nd -// CHECK: arith.divf -// CHECK: xegpu.store_nd - -// ----- - -func.func @div_unsigned(%arg0: memref<8x16xi16>, %arg1: memref<8x16xi16>, %arg2: memref<8x16xi16>) { - linalg.div_unsigned ins(%arg0, %arg1 : memref<8x16xi16>, memref<8x16xi16>) - outs(%arg2 : memref<8x16xi16>) - return -} - -// CHECK-LABEL: func.func @div_unsigned -// CHECK-COUNT-2: xegpu.load_nd -// CHECK: arith.divui -// CHECK: xegpu.store_nd - -// ----- - -func.func @exp(%arg0: memref<8x16xf16>, %arg1: memref<8x16xf16>) { - linalg.exp ins(%arg0 : memref<8x16xf16>) - outs(%arg1 : memref<8x16xf16>) - return -} - -// CHECK-LABEL: func.func @exp -// CHECK-COUNT-1: xegpu.load_nd -// CHECK: math.exp -// CHECK: xegpu.store_nd - -// ----- - -func.func @floor(%arg0: memref<8x16xf16>, %arg1: memref<8x16xf16>) { - linalg.floor ins(%arg0 : memref<8x16xf16>) - outs(%arg1 : memref<8x16xf16>) - return -} - -// CHECK-LABEL: func.func @floor -// CHECK-COUNT-1: xegpu.load_nd -// CHECK: math.floor -// CHECK: xegpu.store_nd - -// ----- - -func.func @max(%arg0: memref<8x16xf16>, %arg1: memref<8x16xf16>, %arg2: memref<8x16xf16>) { - linalg.max ins(%arg0, %arg1 : memref<8x16xf16>, memref<8x16xf16>) - outs(%arg2 : memref<8x16xf16>) - return -} - -// CHECK-LABEL: func.func @max -// CHECK-COUNT-2: xegpu.load_nd -// CHECK: arith.maximumf -// CHECK: xegpu.store_nd - -// ----- - -func.func @mul(%arg0: memref<8x16xf16>, %arg1: memref<8x16xf16>, %arg2: memref<8x16xf16>) { - linalg.mul ins(%arg0, %arg1 : memref<8x16xf16>, memref<8x16xf16>) - outs(%arg2 : memref<8x16xf16>) - return -} - -// CHECK-LABEL: func.func @mul -// CHECK-COUNT-2: xegpu.load_nd -// CHECK: arith.mulf -// CHECK: xegpu.store_nd - -// ----- - -func.func @negf(%arg0: memref<8x16xf16>, %arg1: memref<8x16xf16>) { - linalg.negf ins(%arg0 : memref<8x16xf16>) - outs(%arg1 : memref<8x16xf16>) - return -} - -// CHECK-LABEL: func.func @negf -// CHECK-COUNT-1: xegpu.load_nd -// CHECK: arith.negf -// CHECK: xegpu.store_nd - -// ----- - -func.func @sub(%arg0: memref<8x16xf16>, %arg1: memref<8x16xf16>, %arg2: memref<8x16xf16>) { - linalg.sub ins(%arg0, %arg1 : memref<8x16xf16>, memref<8x16xf16>) - outs(%arg2 : memref<8x16xf16>) - return -} - -// CHECK-LABEL: func.func @sub -// CHECK-COUNT-2: xegpu.load_nd -// CHECK: arith.subf -// CHECK: xegpu.store_nd - -// ----- - -func.func @add_large_f16(%arg0: memref<64x64xf16>, %arg1: memref<64x64xf16>, %arg2: memref<64x64xf16>) { - linalg.add ins(%arg0, %arg1 : memref<64x64xf16>, memref<64x64xf16>) - outs(%arg2 : memref<64x64xf16>) - return -} - -// CHECK-LABEL: func.func @add_large_f16 -// CHECK: xegpu.load_nd{{.*}}: !xegpu.tensor_desc<32x32xf16{{.*}}> -> vector<32x32xf16> -// CHECK: arith.addf{{.*}}: vector<8x32xf16> -// CHECK: xegpu.store_nd{{.*}}: vector<8x32xf16> - -// ----- - -func.func @add_large_f32(%arg0: memref<64x64xf32>, %arg1: memref<64x64xf32>, %arg2: memref<64x64xf32>) { - linalg.add ins(%arg0, %arg1 : memref<64x64xf32>, memref<64x64xf32>) - outs(%arg2 : memref<64x64xf32>) - return -} - -// CHECK-LABEL: func.func @add_large_f32 -// CHECK: xegpu.load_nd{{.*}}: !xegpu.tensor_desc<32x16xf32{{.*}}> -> vector<32x16xf32> -// CHECK: arith.addf{{.*}}: vector<16x16xf32> -// CHECK: xegpu.store_nd{{.*}}: vector<16x16xf32> diff --git a/test/mlir/test/gc/Transforms/GPU/IMEX/lit.local.cfg b/test/mlir/test/gc/Transforms/GPU/IMEX/lit.local.cfg deleted file mode 100644 index f180dd41b..000000000 --- a/test/mlir/test/gc/Transforms/GPU/IMEX/lit.local.cfg +++ /dev/null @@ -1,2 +0,0 @@ -if not config.gc_use_imex: - config.unsupported = True \ No newline at end of file diff --git a/test/mlir/test/gc/Transforms/GPU/OCL/gpu-to-gpuocl-inlined.mlir b/test/mlir/test/gc/Transforms/GPU/OCL/gpu-to-gpuocl-inlined.mlir deleted file mode 100644 index 60158e0ee..000000000 --- a/test/mlir/test/gc/Transforms/GPU/OCL/gpu-to-gpuocl-inlined.mlir +++ /dev/null @@ -1,102 +0,0 @@ -// RUN: gc-opt %s --gc-gpu-pipeline | FileCheck %s - -module @test attributes {gpu.container_module} { - llvm.func @entry(%arg0: !llvm.ptr, %arg1: !llvm.ptr, %arg2: i64, %arg3: i64, %arg4: i64, %arg5: i64, %arg6: i64, %arg7: !llvm.ptr, %arg8: !llvm.ptr, %arg9: i64) attributes {llvm.emit_c_interface} { - %0 = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - %1 = llvm.insertvalue %arg0, %0[0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - %2 = llvm.insertvalue %arg1, %1[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - %3 = llvm.insertvalue %arg2, %2[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - %4 = llvm.insertvalue %arg3, %3[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - %5 = llvm.insertvalue %arg4, %4[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - %6 = llvm.insertvalue %arg5, %5[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - %7 = llvm.insertvalue %arg6, %6[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - %8 = builtin.unrealized_conversion_cast %7 : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> to memref<64x64xf32> - %gpu_mem = gpu.alloc host_shared () : memref<64x64xf32> - gpu.memcpy %gpu_mem, %8 : memref<64x64xf32>, memref<64x64xf32> - %9 = llvm.mlir.constant(32 : index) : i64 - %10 = builtin.unrealized_conversion_cast %9 : i64 to index - %11 = llvm.mlir.constant(2 : index) : i64 - %12 = builtin.unrealized_conversion_cast %11 : i64 to index - %13 = llvm.mlir.constant(1 : index) : i64 - %14 = builtin.unrealized_conversion_cast %13 : i64 to index - - %floaat = llvm.mlir.constant(1.1 : f32) : f32 - %a_ptr_as_idx = memref.extract_aligned_pointer_as_index %gpu_mem : memref<64x64xf32> -> index - %a_ptr_as_i64 = arith.index_cast %a_ptr_as_idx : index to i64 - %a_ptr = llvm.inttoptr %a_ptr_as_i64 : i64 to !llvm.ptr - %a_ptr_casted = llvm.addrspacecast %a_ptr : !llvm.ptr to !llvm.ptr<1> - - gpu.launch blocks(%arg10, %arg11, %arg12) in (%arg16 = %12, %arg17 = %12, %arg18 = %12) threads(%arg13, %arg14, %arg15) in (%arg19 = %14, %arg20 = %14, %arg21 = %14) { - llvm.store %floaat, %a_ptr_casted : f32, !llvm.ptr<1> - gpu.terminator - } - gpu.memcpy %8, %gpu_mem : memref<64x64xf32>, memref<64x64xf32> - gpu.dealloc %gpu_mem : memref<64x64xf32> - llvm.return - } -} - -// CHECK: llvm.mlir.global internal constant @gcGpuOclKernel_entry_kernel_SPIRV -// CHECK: llvm.mlir.global internal constant @gcGpuOclKernel_entry_kernel_Name -// CHECK: llvm.mlir.global internal @gcGpuOclKernel_entry_kernel_Ptr - -// CHECK: llvm.func @createGcGpuOclKernel_entry_kernel([[CTX:%.+]]: !llvm.ptr) -> !llvm.ptr -// CHECK: [[NEW_PTR:%.+]] = llvm.call @gcGpuOclKernelCreate([[CTX]] -// CHECK: [[ZERO:%.+]] = llvm.mlir.zero -// CHECK: [[PTR_ADDR:%.+]] = llvm.mlir.addressof @gcGpuOclKernel_entry_kernel_Ptr -// CHECK: [[CMPXCHG:%.+]] = llvm.cmpxchg [[PTR_ADDR]], [[ZERO]], [[NEW_PTR]] -// CHECK: [[FLAG:%.+]] = llvm.extractvalue [[CMPXCHG]][1] -// CHECK: llvm.cond_br [[FLAG]], [[BB1:\^.+]], [[BB2:\^.+]] -// CHECK: [[BB1]]: -// CHECK: llvm.return [[NEW_PTR]] -// CHECK: [[BB2]]: -// CHECK: [[ONE:%.+]] = llvm.mlir.constant(1 : i64) : i64 -// CHECK: [[ARRAY:%.+]] = llvm.alloca [[ONE]] -// CHECK: [[ADDR:%.+]] = llvm.getelementptr [[ARRAY]] -// CHECK: llvm.store [[NEW_PTR]], [[ADDR]] -// CHECK: llvm.call @gcGpuOclKernelDestroy([[ONE]], [[ARRAY]]) -// CHECK: [[OLD_PTR:%.+]] = llvm.extractvalue [[CMPXCHG]][0] -// CHECK: llvm.return [[OLD_PTR]] - -// CHECK: llvm.func internal @getGcGpuOclKernel_entry_kernel([[CTX:%.+]]: !llvm.ptr) -> !llvm.ptr attributes {always_inline} -// CHECK: [[ZERO:%.+]] = llvm.mlir.zero -// CHECK: [[PTR_ADDR:%.+]] = llvm.mlir.addressof @gcGpuOclKernel_entry_kernel_Ptr -// CHECK: [[PTR:%.+]] = llvm.load [[PTR_ADDR]] -// CHECK: [[ICMP:%.+]] = llvm.icmp "eq" [[PTR]], [[ZERO]] -// CHECK: llvm.cond_br [[ICMP]], [[BB1:\^.+]], [[BB2:\^.+]] -// CHECK: [[BB1]]: -// CHECK: [[NEW_PTR:%.+]] = llvm.call @createGcGpuOclKernel_entry_kernel([[CTX]]) -// CHECK: llvm.return [[NEW_PTR]] -// CHECK: [[BB2]]: -// CHECK: llvm.return [[PTR]] - -// CHECK: llvm.func @entry(%arg0: !llvm.ptr, %arg1: !llvm.ptr, %arg2: i64, %arg3: i64, %arg4: i64, %arg5: i64, %arg6: i64, [[CTX:%.+]]: !llvm.ptr, %arg8: !llvm.ptr, %arg9: i64) -// CHECK: [[SIZE:%.+]] = llvm.mlir.constant(16384 : i64) : i64 -// CHECK: llvm.call @gcGpuOclMallocShared([[CTX]], [[SIZE]]) -// CHECK: [[SIZE:%.+]] = llvm.mlir.constant(16384 : i64) : i64 -// CHECK: [[SRC:%.+]] = llvm.extractvalue -// CHECK: [[DST:%.+]] = llvm.extractvalue [[GPU_MEMREF:%.+]][1] -// CHECK: llvm.call @gcGpuOclMemcpy([[CTX]], [[SRC]], [[DST]], [[SIZE]]) -// CHECK: [[KERNEL:%.+]] = llvm.call @getGcGpuOclKernel_entry_kernel([[CTX:%.+]]) : (!llvm.ptr) -> !llvm.ptr -// CHECK: llvm.call @gcGpuOclKernelLaunch([[CTX]], [[KERNEL]], -// CHECK: [[SIZE:%.+]] = llvm.mlir.constant(16384 : i64) : i64 -// CHECK: [[SRC:%.+]] = llvm.extractvalue [[GPU_MEMREF:%.+]][1] -// CHECK: [[DST:%.+]] = llvm.extractvalue -// CHECK: llvm.call @gcGpuOclMemcpy([[CTX]], [[SRC]], [[DST]], [[SIZE]]) -// CHECK: [[GPU_PTR:%.+]] = llvm.extractvalue [[GPU_MEMREF:%.+]][0] -// CHECK: llvm.call @gcGpuOclDealloc([[CTX]], [[GPU_PTR]]) - -// CHECK: llvm.func @gcGpuOclKernelCreate -// CHECK: llvm.func @gcGpuOclKernelDestroy -// CHECK: llvm.func @gcGpuOclKernelLaunch - - -// CHECK: llvm.func @gcGpuOclModuleDestructor() -// CHECK: llvm.fence acquire -// CHECK: [[PTR_ADDR:%.+]] = llvm.mlir.addressof @gcGpuOclKernel_entry_kernel_Ptr -// CHECK: [[PTR:%.+]] = llvm.load [[PTR_ADDR]] -// CHECK: [[ONE:%.+]] = llvm.mlir.constant(1 : i64) : i64 -// CHECK: [[ARRAY:%.+]] = llvm.alloca [[ONE]] -// CHECK: [[ADDR:%.+]] = llvm.getelementptr [[ARRAY]] -// CHECK: llvm.store [[PTR]], [[ADDR]] -// CHECK: llvm.call @gcGpuOclKernelDestroy([[ONE]], [[ARRAY]]) diff --git a/test/mlir/test/gc/Transforms/GPU/OCL/gpu-to-gpuocl.mlir b/test/mlir/test/gc/Transforms/GPU/OCL/gpu-to-gpuocl.mlir deleted file mode 100644 index 74b28f6a1..000000000 --- a/test/mlir/test/gc/Transforms/GPU/OCL/gpu-to-gpuocl.mlir +++ /dev/null @@ -1,94 +0,0 @@ -// RUN: gc-opt %s --gpu-to-gpuocl | FileCheck %s - -module @test attributes {gpu.container_module} { - llvm.func @entry(%arg0: !llvm.ptr, %arg1: !llvm.ptr, %arg2: i64, %arg3: i64, %arg4: i64, %arg5: i64, %arg6: i64, %arg7: !llvm.ptr, %arg8: !llvm.ptr, %arg9: i64) attributes {llvm.emit_c_interface} { - %0 = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - %1 = llvm.insertvalue %arg0, %0[0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - %2 = llvm.insertvalue %arg1, %1[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - %3 = llvm.insertvalue %arg2, %2[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - %4 = llvm.insertvalue %arg3, %3[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - %5 = llvm.insertvalue %arg4, %4[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - %6 = llvm.insertvalue %arg5, %5[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - %7 = llvm.insertvalue %arg6, %6[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - %8 = builtin.unrealized_conversion_cast %7 : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> to memref<64x64xf32> - %gpu_mem = gpu.alloc host_shared () : memref<64x64xf32> - gpu.memcpy %gpu_mem, %8 : memref<64x64xf32>, memref<64x64xf32> - %9 = llvm.mlir.constant(32 : index) : i64 - %10 = builtin.unrealized_conversion_cast %9 : i64 to index - %11 = llvm.mlir.constant(2 : index) : i64 - %12 = builtin.unrealized_conversion_cast %11 : i64 to index - %13 = llvm.mlir.constant(1 : index) : i64 - %14 = builtin.unrealized_conversion_cast %13 : i64 to index - gpu.launch_func @entry_kernel::@entry_kernel blocks in (%12, %12, %14) threads in (%14, %14, %14) args(%10 : index, %gpu_mem : memref<64x64xf32>) - gpu.memcpy %8, %gpu_mem : memref<64x64xf32>, memref<64x64xf32> - gpu.dealloc %gpu_mem : memref<64x64xf32> - llvm.return - } - - gpu.binary @entry_kernel [#gpu.object<#xevm.target, "\03\02#\07\00\04\01\00\14\00+\00\0F\00\00\00\00\00\00\00\11\00\02\00\06\00\00\00\11\00\02\00\04\00\00\00\11\00\02\00\0B\00\00\00\0B\00\05\00\01\00\00\00OpenCL.std\00\00\0E\00\03\00\02\00\00\00\02\00\00\00\0F\00\07\00\06\00\00\00\05\00\00\00entry_kernel\00\00\00\00\10\00\03\00\05\00\00\00\1F\00\00\00\03\00\03\00\04\00\00\00\A0\86\01\00\05\00\06\00\05\00\00\00entry_kernel\00\00\00\00\15\00\04\00\02\00\00\00@\00\00\00\00\00\00\00\13\00\02\00\03\00\00\00!\00\04\00\04\00\00\00\03\00\00\00\02\00\00\006\00\05\00\03\00\00\00\05\00\00\00\04\00\00\00\04\00\00\007\00\03\00\02\00\00\00\06\00\00\00\F8\00\02\00\07\00\00\00\FD\00\01\008\00\01\00">] -} - -// CHECK: llvm.mlir.global internal constant @gcGpuOclKernel_entry_kernel_SPIRV -// CHECK: llvm.mlir.global internal constant @gcGpuOclKernel_entry_kernel_Name -// CHECK: llvm.mlir.global internal @gcGpuOclKernel_entry_kernel_Ptr - -// CHECK: llvm.func @createGcGpuOclKernel_entry_kernel([[CTX:%.+]]: !llvm.ptr) -> !llvm.ptr -// CHECK: [[NEW_PTR:%.+]] = llvm.call @gcGpuOclKernelCreate([[CTX]] -// CHECK: [[ZERO:%.+]] = llvm.mlir.zero -// CHECK: [[PTR_ADDR:%.+]] = llvm.mlir.addressof @gcGpuOclKernel_entry_kernel_Ptr -// CHECK: [[CMPXCHG:%.+]] = llvm.cmpxchg [[PTR_ADDR]], [[ZERO]], [[NEW_PTR]] -// CHECK: [[FLAG:%.+]] = llvm.extractvalue [[CMPXCHG]][1] -// CHECK: llvm.cond_br [[FLAG]], [[BB1:\^.+]], [[BB2:\^.+]] -// CHECK: [[BB1]]: -// CHECK: llvm.return [[NEW_PTR]] -// CHECK: [[BB2]]: -// CHECK: [[ONE:%.+]] = llvm.mlir.constant(1 : i64) : i64 -// CHECK: [[ARRAY:%.+]] = llvm.alloca [[ONE]] -// CHECK: [[ADDR:%.+]] = llvm.getelementptr [[ARRAY]] -// CHECK: llvm.store [[NEW_PTR]], [[ADDR]] -// CHECK: llvm.call @gcGpuOclKernelDestroy([[ONE]], [[ARRAY]]) -// CHECK: [[OLD_PTR:%.+]] = llvm.extractvalue [[CMPXCHG]][0] -// CHECK: llvm.return [[OLD_PTR]] - -// CHECK: llvm.func internal @getGcGpuOclKernel_entry_kernel([[CTX:%.+]]: !llvm.ptr) -> !llvm.ptr attributes {always_inline} -// CHECK: [[ZERO:%.+]] = llvm.mlir.zero -// CHECK: [[PTR_ADDR:%.+]] = llvm.mlir.addressof @gcGpuOclKernel_entry_kernel_Ptr -// CHECK: [[PTR:%.+]] = llvm.load [[PTR_ADDR]] -// CHECK: [[ICMP:%.+]] = llvm.icmp "eq" [[PTR]], [[ZERO]] -// CHECK: llvm.cond_br [[ICMP]], [[BB1:\^.+]], [[BB2:\^.+]] -// CHECK: [[BB1]]: -// CHECK: [[NEW_PTR:%.+]] = llvm.call @createGcGpuOclKernel_entry_kernel([[CTX]]) -// CHECK: llvm.return [[NEW_PTR]] -// CHECK: [[BB2]]: -// CHECK: llvm.return [[PTR]] - -// CHECK: llvm.func @entry(%arg0: !llvm.ptr, %arg1: !llvm.ptr, %arg2: i64, %arg3: i64, %arg4: i64, %arg5: i64, %arg6: i64, [[CTX:%.+]]: !llvm.ptr, %arg8: !llvm.ptr, %arg9: i64) -// CHECK: [[SIZE:%.+]] = llvm.mlir.constant(16384 : i64) : i64 -// CHECK: llvm.call @gcGpuOclMallocShared([[CTX]], [[SIZE]]) -// CHECK: [[SIZE:%.+]] = llvm.mlir.constant(16384 : i64) : i64 -// CHECK: [[SRC:%.+]] = llvm.extractvalue -// CHECK: [[DST:%.+]] = llvm.extractvalue [[GPU_MEMREF:%.+]][1] -// CHECK: llvm.call @gcGpuOclMemcpy([[CTX]], [[SRC]], [[DST]], [[SIZE]]) -// CHECK: [[KERNEL:%.+]] = llvm.call @getGcGpuOclKernel_entry_kernel([[CTX:%.+]]) : (!llvm.ptr) -> !llvm.ptr -// CHECK: llvm.call @gcGpuOclKernelLaunch([[CTX]], [[KERNEL]], -// CHECK: [[SIZE:%.+]] = llvm.mlir.constant(16384 : i64) : i64 -// CHECK: [[SRC:%.+]] = llvm.extractvalue [[GPU_MEMREF:%.+]][1] -// CHECK: [[DST:%.+]] = llvm.extractvalue -// CHECK: llvm.call @gcGpuOclMemcpy([[CTX]], [[SRC]], [[DST]], [[SIZE]]) -// CHECK: [[GPU_PTR:%.+]] = llvm.extractvalue [[GPU_MEMREF:%.+]][0] -// CHECK: llvm.call @gcGpuOclDealloc([[CTX]], [[GPU_PTR]]) - -// CHECK: llvm.func @gcGpuOclKernelCreate -// CHECK: llvm.func @gcGpuOclKernelDestroy -// CHECK: llvm.func @gcGpuOclKernelLaunch - - -// CHECK: llvm.func @gcGpuOclModuleDestructor() -// CHECK: llvm.fence acquire -// CHECK: [[PTR_ADDR:%.+]] = llvm.mlir.addressof @gcGpuOclKernel_entry_kernel_Ptr -// CHECK: [[PTR:%.+]] = llvm.load [[PTR_ADDR]] -// CHECK: [[ONE:%.+]] = llvm.mlir.constant(1 : i64) : i64 -// CHECK: [[ARRAY:%.+]] = llvm.alloca [[ONE]] -// CHECK: [[ADDR:%.+]] = llvm.getelementptr [[ARRAY]] -// CHECK: llvm.store [[PTR]], [[ADDR]] -// CHECK: llvm.call @gcGpuOclKernelDestroy([[ONE]], [[ARRAY]]) diff --git a/test/mlir/test/gc/Transforms/GPU/OCL/lit.local.cfg b/test/mlir/test/gc/Transforms/GPU/OCL/lit.local.cfg deleted file mode 100644 index 736deccc1..000000000 --- a/test/mlir/test/gc/Transforms/GPU/OCL/lit.local.cfg +++ /dev/null @@ -1,2 +0,0 @@ -if not config.gc_use_gpu or config.gc_use_imex: - config.unsupported = True \ No newline at end of file diff --git a/test/mlir/test/gc/Transforms/GPU/allocs-to-slm-1d.mlir b/test/mlir/test/gc/Transforms/GPU/allocs-to-slm-1d.mlir deleted file mode 100644 index 302267da6..000000000 --- a/test/mlir/test/gc/Transforms/GPU/allocs-to-slm-1d.mlir +++ /dev/null @@ -1,43 +0,0 @@ -// RUN: gc-opt %s --allocs-to-slm | FileCheck %s - -// Computex thread offset for SLM: (Xthread_idx * Yblock_sz * Zblock_sz + Ythread_idx * Zblock_sz + Zthread_idx) * Xchunk_size -// CHECK: #map = affine_map<(d0, d1, d2) -> ((d0 * 12 + d1 * 4 + d2) * 256)> - -func.func @entry() { - %c1 = arith.constant 1 : index - %c2 = arith.constant 2 : index - %c3 = arith.constant 3 : index - %c4 = arith.constant 4 : index - - // Memory space wasn't assigned as it's allocated outside of gpu.launch block - // CHECK: %[[NEW_MEMREF_0:.*]] = memref.alloc() : memref<256xf16> - %0 = memref.alloc() : memref<256xf16> - // Capture thread-id variables - // CHECK: gpu.launch blocks(%[[ARG0:.+]], %[[ARG1:.+]], %[[ARG2:.+]]) in (%[[ARG6:.+]] = %c2, %[[ARG7:.+]] = %c2, %[[ARG8:.+]] = %c1) threads - // CHECK-SAME: (%[[THREAD_X:.+]], %[[THREAD_Y:.+]], %[[THREAD_Z:.+]]) in - // CHECK-SAME: (%[[ARG9:.+]] = %c2, %[[ARG10:.+]] = %c3, %[[ARG11:.+]] = %c4) { - gpu.launch blocks(%bx, %by, %bz) in (%sz_bx = %c2, %sz_by = %c2, %sz_bz = %c1) - threads(%tx, %ty, %tz) in (%sz_tx = %c2, %sz_ty = %c3, %sz_tz = %c4) { - // Memory space was changed as it's explicitly specifided - // CHECK: %[[NEW_MEMREF_1:.*]] = memref.alloc() : memref<256xf16, 1> - %1 = memref.alloc() : memref<256xf16, 1> - // Added 'shared' memory space and allocated SLM for each thread (2 * 3 * 4 = 24; 24 * 256 = 6144) - // CHECK: %[[NEW_MEMREF_2:.*]] = memref.alloc() : memref<6144xf16, 3> - // CHECK: %[[OFF_X:.*]] = affine.apply #map(%[[THREAD_X]], %[[THREAD_Y]], %[[THREAD_Z]]) - // CHECK: %[[NEW_MEMREF_3:.*]] = memref.subview %[[NEW_MEMREF_2]][%[[OFF_X]]] [256] [1] - // CHECK-SAME: memref<6144xf16, 3> to memref<256xf16, strided<[1], offset: ?>, 3> - %2 = memref.alloc() : memref<256xf16> - - // CHECK: linalg.add ins(%[[NEW_MEMREF_1]], %[[NEW_MEMREF_3]] : - // CHECK-SAME: memref<256xf16, 1>, memref<256xf16, strided<[1], offset: ?>, 3>) outs(%[[NEW_MEMREF_0]] : memref<256xf16>) - linalg.add ins(%1, %2 :memref<256xf16, 1>, memref<256xf16>) outs(%0 : memref<256xf16>) - // CHECK: memref.dealloc %[[NEW_MEMREF_1]] : memref<256xf16, 1> - // Verify that there are no deallocs for SLM - // CHECK-NOT: memref.dealloc %[[NEW_MEMREF_2]] .* - // CHECK-NOT: memref.dealloc %[[NEW_MEMREF_3]] .* - memref.dealloc %1 : memref<256xf16, 1> - memref.dealloc %2 : memref<256xf16> - gpu.terminator - } - return -} diff --git a/test/mlir/test/gc/Transforms/GPU/allocs-to-slm-2d.mlir b/test/mlir/test/gc/Transforms/GPU/allocs-to-slm-2d.mlir deleted file mode 100644 index b41fab1f1..000000000 --- a/test/mlir/test/gc/Transforms/GPU/allocs-to-slm-2d.mlir +++ /dev/null @@ -1,43 +0,0 @@ -// RUN: gc-opt %s --allocs-to-slm | FileCheck %s - -// Computex thread offset for SLM: (Xthread_idx * Yblock_sz * Zblock_sz + Ythread_idx * Zblock_sz + Zthread_idx) * Xchunk_size -// CHECK: #map = affine_map<(d0, d1, d2) -> ((d0 * 12 + d1 * 4 + d2) * 16)> - -func.func @entry() { - %c1 = arith.constant 1 : index - %c2 = arith.constant 2 : index - %c3 = arith.constant 3 : index - %c4 = arith.constant 4 : index - - // Memory space wasn't assigned as it's allocated outside of gpu.launch block - // CHECK: %[[NEW_MEMREF_0:.*]] = memref.alloc() : memref<16x32xf16> - %0 = memref.alloc() : memref<16x32xf16> - // Capture thread-id variables - // CHECK: gpu.launch blocks(%[[ARG0:.+]], %[[ARG1:.+]], %[[ARG2:.+]]) in (%[[ARG6:.+]] = %c2, %[[ARG7:.+]] = %c2, %[[ARG8:.+]] = %c1) threads - // CHECK-SAME: (%[[THREAD_X:.+]], %[[THREAD_Y:.+]], %[[THREAD_Z:.+]]) in - // CHECK-SAME: (%[[ARG9:.+]] = %c2, %[[ARG10:.+]] = %c3, %[[ARG11:.+]] = %c4) { - gpu.launch blocks(%bx, %by, %bz) in (%sz_bx = %c2, %sz_by = %c2, %sz_bz = %c1) - threads(%tx, %ty, %tz) in (%sz_tx = %c2, %sz_ty = %c3, %sz_tz = %c4) { - // Memory space was changed as it's explicitly specifided - // CHECK: %[[NEW_MEMREF_1:.*]] = memref.alloc() : memref<16x32xf16, 1> - %1 = memref.alloc() : memref<16x32xf16, 1> - // Added 'shared' memory space and allocated SLM for each thread (2 * 3 * 4 = 24; 24 * 16 = 384) - // CHECK: %[[NEW_MEMREF_2:.*]] = memref.alloc() : memref<384x32xf16, 3> - // CHECK: %[[OFF_X:.*]] = affine.apply #map(%[[THREAD_X]], %[[THREAD_Y]], %[[THREAD_Z]]) - // CHECK: %[[NEW_MEMREF_3:.*]] = memref.subview %[[NEW_MEMREF_2]][%[[OFF_X]], 0] [16, 32] [1, 1] - // CHECK-SAME: memref<384x32xf16, 3> to memref<16x32xf16, strided<[32, 1], offset: ?>, 3> - %2 = memref.alloc() : memref<16x32xf16> - - // CHECK: linalg.add ins(%[[NEW_MEMREF_1]], %[[NEW_MEMREF_3]] : - // CHECK-SAME: memref<16x32xf16, 1>, memref<16x32xf16, strided<[32, 1], offset: ?>, 3>) outs(%[[NEW_MEMREF_0]] : memref<16x32xf16>) - linalg.add ins(%1, %2 :memref<16x32xf16, 1>, memref<16x32xf16>) outs(%0 : memref<16x32xf16>) - // CHECK: memref.dealloc %[[NEW_MEMREF_1]] : memref<16x32xf16, 1> - // Verify that there are no deallocs for SLM - // CHECK-NOT: memref.dealloc %[[NEW_MEMREF_2]] .* - // CHECK-NOT: memref.dealloc %[[NEW_MEMREF_3]] .* - memref.dealloc %1 : memref<16x32xf16, 1> - memref.dealloc %2 : memref<16x32xf16> - gpu.terminator - } - return -} diff --git a/test/mlir/test/gc/Transforms/GPU/allocs-to-slm-4d.mlir b/test/mlir/test/gc/Transforms/GPU/allocs-to-slm-4d.mlir deleted file mode 100644 index 01659795c..000000000 --- a/test/mlir/test/gc/Transforms/GPU/allocs-to-slm-4d.mlir +++ /dev/null @@ -1,43 +0,0 @@ -// RUN: gc-opt %s --allocs-to-slm | FileCheck %s - -// Computex thread offset for SLM: (Xthread_idx * Yblock_sz * Zblock_sz + Ythread_idx * Zblock_sz + Zthread_idx) * Xchunk_size -// CHECK: #map = affine_map<(d0, d1, d2) -> ((d0 * 12 + d1 * 4 + d2) * 2)> - -func.func @entry() { - %c1 = arith.constant 1 : index - %c2 = arith.constant 2 : index - %c3 = arith.constant 3 : index - %c4 = arith.constant 4 : index - - // Memory space wasn't assigned as it's allocated outside of gpu.launch block - // CHECK: %[[NEW_MEMREF_0:.*]] = memref.alloc() : memref<2x3x16x32xf16> - %0 = memref.alloc() : memref<2x3x16x32xf16> - // Capture thread-id variables - // CHECK: gpu.launch blocks(%[[ARG0:.+]], %[[ARG1:.+]], %[[ARG2:.+]]) in (%[[ARG6:.+]] = %c2, %[[ARG7:.+]] = %c2, %[[ARG8:.+]] = %c1) threads - // CHECK-SAME: (%[[THREAD_X:.+]], %[[THREAD_Y:.+]], %[[THREAD_Z:.+]]) in - // CHECK-SAME: (%[[ARG9:.+]] = %c2, %[[ARG10:.+]] = %c3, %[[ARG11:.+]] = %c4) { - gpu.launch blocks(%bx, %by, %bz) in (%sz_bx = %c2, %sz_by = %c2, %sz_bz = %c1) - threads(%tx, %ty, %tz) in (%sz_tx = %c2, %sz_ty = %c3, %sz_tz = %c4) { - // Memory space was changed as it's explicitly specifided - // CHECK: %[[NEW_MEMREF_1:.*]] = memref.alloc() : memref<2x3x16x32xf16, 1> - %1 = memref.alloc() : memref<2x3x16x32xf16, 1> - // Added 'shared' memory space and allocated SLM for each thread (2 * 3 * 4 = 24; 24 * 2 = 48) - // CHECK: %[[NEW_MEMREF_2:.*]] = memref.alloc() : memref<48x3x16x32xf16, 3> - // CHECK: %[[OFF_X:.*]] = affine.apply #map(%[[THREAD_X]], %[[THREAD_Y]], %[[THREAD_Z]]) - // CHECK: %[[NEW_MEMREF_3:.*]] = memref.subview %[[NEW_MEMREF_2]][%[[OFF_X]], 0, 0, 0] [2, 3, 16, 32] [1, 1, 1, 1] - // CHECK-SAME: memref<48x3x16x32xf16, 3> to memref<2x3x16x32xf16, strided<[1536, 512, 32, 1], offset: ?>, 3> - %2 = memref.alloc() : memref<2x3x16x32xf16> - - // CHECK: linalg.add ins(%[[NEW_MEMREF_1]], %[[NEW_MEMREF_3]] : - // CHECK-SAME: memref<2x3x16x32xf16, 1>, memref<2x3x16x32xf16, strided<[1536, 512, 32, 1], offset: ?>, 3>) outs(%[[NEW_MEMREF_0]] : memref<2x3x16x32xf16>) - linalg.add ins(%1, %2 :memref<2x3x16x32xf16, 1>, memref<2x3x16x32xf16>) outs(%0 : memref<2x3x16x32xf16>) - // CHECK: memref.dealloc %[[NEW_MEMREF_1]] : memref<2x3x16x32xf16, 1> - // Verify that there are no deallocs for SLM - // CHECK-NOT: memref.dealloc %[[NEW_MEMREF_2]] .* - // CHECK-NOT: memref.dealloc %[[NEW_MEMREF_3]] .* - memref.dealloc %1 : memref<2x3x16x32xf16, 1> - memref.dealloc %2 : memref<2x3x16x32xf16> - gpu.terminator - } - return -} diff --git a/test/mlir/test/gc/Transforms/GPU/module-to-binary-xevm.mlir b/test/mlir/test/gc/Transforms/GPU/module-to-binary-xevm.mlir deleted file mode 100644 index 3b3f4a26e..000000000 --- a/test/mlir/test/gc/Transforms/GPU/module-to-binary-xevm.mlir +++ /dev/null @@ -1,12 +0,0 @@ -// RUN: gc-opt %s --gpu-to-llvm --convert-gpu-to-llvm-spv --gpu-module-to-binary | FileCheck %s - -module attributes {gpu.container_module} { - // CHECK-LABEL:gpu.binary @entry_kernel - // CHECK:[#gpu.object<#xevm.target, - gpu.module @entry_kernel [#xevm.target] { - gpu.func @entry_kernel(%arg0: index) kernel attributes {} { - gpu.return - } - } -} - diff --git a/test/mlir/test/gc/Transforms/GPU/xevm-attach-target.mlir b/test/mlir/test/gc/Transforms/GPU/xevm-attach-target.mlir deleted file mode 100644 index d9eecabbd..000000000 --- a/test/mlir/test/gc/Transforms/GPU/xevm-attach-target.mlir +++ /dev/null @@ -1,10 +0,0 @@ -// RUN: gc-opt %s --xevm-attach-target | FileCheck %s -module attributes {gpu.container_module} { - //CHECK:gpu.module @entry_kernel [#xevm.target] - gpu.module @entry_kernel { - gpu.func @entry_kernel(%arg0: index) kernel attributes {} { - gpu.return - } - } -} - diff --git a/test/mlir/test/gc/Transforms/Pipeline/run.mlir b/test/mlir/test/gc/Transforms/Pipeline/run.mlir deleted file mode 100644 index 71feb0843..000000000 --- a/test/mlir/test/gc/Transforms/Pipeline/run.mlir +++ /dev/null @@ -1,23 +0,0 @@ -// RUN: gc-opt %s --gc-cpu-pipeline | gc-cpu-runner -e main -entry-point-result=void | FileCheck %s - -module { -func.func @aaa() -> tensor<128xf32> { - %c2 = arith.constant 2.0 : f32 - %a = tensor.empty() : tensor<128xf32> - %2 = linalg.fill ins(%c2 : f32) outs(%a : tensor<128xf32>) -> tensor<128xf32> - return %2 : tensor<128xf32> -} - -func.func @main() { - %result = call @aaa() : ()-> tensor<128xf32> - %c0 = arith.constant 0 : index - %c128 = arith.constant 128 : index - %c1 = arith.constant 1 : index - scf.for %iv = %c0 to %c128 step %c1 { - %4 = tensor.extract %result[%iv] : tensor<128xf32> - cpuruntime.printf "%f\n" %4 : f32 - } - return -} -// CHECK-COUNT-128: 2.000000 -} \ No newline at end of file diff --git a/test/mlir/test/gc/Transforms/Pipeline/tensor_args.mlir b/test/mlir/test/gc/Transforms/Pipeline/tensor_args.mlir deleted file mode 100644 index adcfb3bd8..000000000 --- a/test/mlir/test/gc/Transforms/Pipeline/tensor_args.mlir +++ /dev/null @@ -1,13 +0,0 @@ -// RUN: gc-opt %s --gc-cpu-pipeline | FileCheck %s - -module { -// CHECK: aaa -// check that the func returns void -// CHECK-NOT: ) -> !llvm.struct< -func.func @aaa(%a: tensor<128xf32>, %b: tensor<128xf32>) -> tensor<128xf32> { - %out = tensor.empty() : tensor<128xf32> - %2 = linalg.add ins(%a, %b : tensor<128xf32>,tensor<128xf32>) outs(%out : tensor<128xf32>) -> tensor<128xf32> - // CHECK-NOT: memcpy - return %2 : tensor<128xf32> -} -} \ No newline at end of file diff --git a/test/mlir/test/gc/Transforms/bf16Legalization.mlir b/test/mlir/test/gc/Transforms/bf16Legalization.mlir deleted file mode 100644 index abd20c35a..000000000 --- a/test/mlir/test/gc/Transforms/bf16Legalization.mlir +++ /dev/null @@ -1,53 +0,0 @@ -// RUN: gc-opt %s --math-extend-to-supported-types --arith-emulate-unsupported-floats="source-types=bf16 target-type=f32" --canonicalize | FileCheck %s - -// CHECK-LABEL: @sin -// CHECK-SAME: ([[ARG0:%.+]]: bf16) -func.func @sin(%arg0: bf16) -> bf16 { - // CHECK: [[EXTF:%.+]] = arith.extf [[ARG0]] - // CHECK: [[SIN:%.+]] = math.sin [[EXTF]] - // CHECK: [[TRUNCF:%.+]] = arith.truncf [[SIN]] - // CHECK: return [[TRUNCF]] : bf16 - %0 = math.sin %arg0 : bf16 - return %0 : bf16 -} - -// CHECK-LABEL: @abs_vector -// CHECK-SAME: ([[ARG0:%.+]]: vector<2xf16>) -// CHECK: [[EXTF:%.+]] = arith.extf [[ARG0]] -// CHECK: [[ABS:%.+]] = math.absf [[EXTF]] -// CHECK: [[TRUNCF:%.+]] = arith.truncf [[ABS]] -// CHECK: return [[TRUNCF]] : vector<2xf16> -func.func @abs_vector(%arg0: vector<2xf16>) -> vector<2xf16> { - %0 = math.absf %arg0 : vector<2xf16> - return %0 : vector<2xf16> -} - -// CHECK-LABEL: @sequences_bf16 -// CHECK-SAME: ([[ARG0:%.+]]: bf16) -// CHECK: [[EXTF0:%.+]] = arith.extf [[ARG0]] -// CHECK: [[ABSF:%.+]] = math.absf [[EXTF0]] -// CHECK: [[SIN:%.+]] = math.sin [[ABSF]] -// CHECK: [[TRUNCF:%.+]] = arith.truncf [[SIN]] -// CHECK: return [[TRUNCF]] : bf16 -func.func @sequences_bf16(%arg0: bf16) -> bf16 { - %0 = math.absf %arg0 : bf16 - %1 = math.sin %0 : bf16 - return %1 : bf16 -} - -// CHECK-LABEL: @bf16_branch_vector -// CHECK-SAME: ([[ARG0:%.+]]: vector<2xbf16>) -// CHECK: [[EXTF:%.+]] = arith.extf [[ARG0]] -// CHECK: [[ABSF:%.+]] = math.absf [[EXTF]] -// CHECK: [[SIN:%.+]] = math.sin [[ABSF]] -// CHECK: [[COS:%.+]] = math.cos [[ABSF]] -// CHECK: [[ADDF:%.+]] = arith.addf -// CHECK: [[TRUNCF:%.+]] = arith.truncf [[ADDF]] -// CHECK: return [[TRUNCF]] : vector<2xbf16> -func.func @bf16_branch_vector(%arg0: vector<2xbf16>) -> vector<2xbf16> { - %0 = math.absf %arg0 : vector<2xbf16> - %1 = math.sin %0 : vector<2xbf16> - %2 = math.cos %0 : vector<2xbf16> - %3 = arith.addf %1, %2 : vector<2xbf16> - return %3 : vector<2xbf16> -} diff --git a/test/mlir/test/gc/Transforms/buffer-merge-invalid.mlir b/test/mlir/test/gc/Transforms/buffer-merge-invalid.mlir deleted file mode 100644 index f90ca901a..000000000 --- a/test/mlir/test/gc/Transforms/buffer-merge-invalid.mlir +++ /dev/null @@ -1,11 +0,0 @@ -// RUN: gc-opt -allow-unregistered-dialect -split-input-file -verify-diagnostics --gc-merge-alloc %s - -func.func @block() { - %mref = memref.alloc() : memref<8 x f32> - %mref2 = memref.alloc() : memref<8 x f32> - // expected-error@+1 {{expecting RegionBranchOpInterface for merge-alloc}} - "some.block"() ({ - ^bb0: - "some.use"(%mref) : (memref<8 x f32>) -> () - }) : () -> () -} \ No newline at end of file diff --git a/test/mlir/test/gc/Transforms/buffer-merge-lifetime.mlir b/test/mlir/test/gc/Transforms/buffer-merge-lifetime.mlir deleted file mode 100644 index 0f5f321fd..000000000 --- a/test/mlir/test/gc/Transforms/buffer-merge-lifetime.mlir +++ /dev/null @@ -1,168 +0,0 @@ -// RUN: gc-opt -allow-unregistered-dialect -p 'builtin.module(func.func(gc-merge-alloc{analysis-only}))' %s | FileCheck %s - -// CHECK-DAG: func.func @basic() -> memref<8x64xf32> attributes {__mergealloc_scope = [[TOPSCOPE:[0-9]+]] -func.func @basic() -> memref<8x64xf32> { - %c0 = arith.constant 0 : index - %c1 = arith.constant 1 : index - %c2 = arith.constant 2 : index - %c3 = arith.constant 3 : index - %c5 = arith.constant 5 : index - %ctrue = arith.constant 1 : i1 - // b is used in return, complex lifetime - // CHECK-DAG: %[[B:.*]] = memref.alloc() {__mergealloc_lifetime = array} - %b = memref.alloc() : memref<8x64xf32> - "test.source"(%b) : (memref<8x64xf32>) -> () - // c and d has overlapping lifetime - // CHECK-DAG: %[[C:.*]] = memref.alloc() {__mergealloc_lifetime = array} - %c = memref.alloc() : memref<8x64xf32> - "test.source"(%c) : (memref<8x64xf32>) -> () - // CHECK-DAG: %[[D:.*]] = memref.alloc() {__mergealloc_lifetime = array} - %d = memref.alloc() : memref<8x64xf32> - "test.source"(%d) : (memref<8x64xf32>) -> () - "test.source"(%c) : (memref<8x64xf32>) -> () - // e and f have overlapping lifetime due to the loop - // CHECK-DAG: %[[E:.*]] = memref.alloc() {__mergealloc_lifetime = array} - // CHECK-DAG: %[[F:.*]] = memref.alloc() {__mergealloc_lifetime = array} - %e = memref.alloc() : memref<8x64xf32> - %f = memref.alloc() : memref<8x64xf32> - // CHECK: scf.for - scf.for %i = %c0 to %c5 step %c1 { - "test.source"(%e) : (memref<8x64xf32>) -> () - "test.source"(%f) : (memref<8x64xf32>) -> () - // CHECK-DAG: %[[G:.*]] = memref.alloc() {__mergealloc_lifetime = array} - %g = memref.alloc() : memref<8x64xf32> - "test.source"(%g) : (memref<8x64xf32>) -> () - } - // CHECK-DAG: %[[H:.*]] = memref.alloc() {__mergealloc_lifetime = array} - %h = memref.alloc() : memref<8x64xf32> - // CHECK: scf.forall - scf.forall (%iv) in (%c5) { - // check that the alloc in the forall should switch to another scope id - // CHECK-NOT: array} - %l = memref.alloc() : memref<8x64xf32> - "test.source"(%h) : (memref<8x64xf32>) -> () - "test.source"(%l) : (memref<8x64xf32>) -> () - scf.for %i = %c0 to %c5 step %c1 { - // CHECK-DAG: %[[G:.*]] = memref.alloc() {__mergealloc_lifetime = array} - %g = memref.alloc() : memref<8x64xf32> - "test.source"(%g) : (memref<8x64xf32>) -> () - } - // CHECK-DAG: %[[K:.*]] = memref.alloc() {__mergealloc_lifetime = array} - %k = memref.alloc() : memref<8x64xf32> - scf.if %ctrue { - // CHECK-DAG: %[[J:.*]] = memref.alloc() {__mergealloc_lifetime = array} - %j = memref.alloc() : memref<8x64xf32> - "test.source"(%j) : (memref<8x64xf32>) -> () - } else { - "test.source"(%k) : (memref<8x64xf32>) -> () - } - // CHECK-DAG: {__mergealloc_scope = [[FORSCOPE]] : i64} - } - return %b : memref<8x64xf32> -} - -// CHECK-DAG: func.func @basic2() attributes {__mergealloc_scope = [[TOPSCOPE2:[0-9]+]] -func.func @basic2() { - // CHECK-DAG: %[[B:.*]] = memref.alloc() {__mergealloc_lifetime = array} - %b = memref.alloc() : memref<8x64xi8> - %cur = memref.subview %b[1,0][1,64][1,1] : memref<8x64xi8> to memref<1x64xi8, strided<[64, 1], offset: 64>> - "test.source"(%cur) : (memref<1x64xi8, strided<[64, 1], offset: 64>>) -> () - %cur2 = memref.subview %cur[0,0][1,16][1,1] : memref<1x64xi8, strided<[64, 1], offset: 64>> to memref<1x16xi8, strided<[64, 1], offset: 64>> - "test.source"(%cur2) : (memref<1x16xi8, strided<[64, 1], offset: 64>>) -> () - // CHECK-DAG: %[[C:.*]] = memref.alloc() {__mergealloc_lifetime = array} - %c = memref.alloc() : memref<8x64xi8> - "test.source"(%c) : (memref<8x64xi8>) -> () - return -} - -// check that the operations without memory effects do not contribute to the lifetime of the buffer -// CHECK-DAG: func.func @no_mem_effect() attributes {__mergealloc_scope = [[TOPSCOPE3:[0-9]+]] -func.func @no_mem_effect() { - %c0 = arith.constant 0 : index - // CHECK: %[[B:.*]] = memref.alloc() {__mergealloc_lifetime = array} - %b = memref.alloc() : memref<8x64xi8> - %x = memref.dim %b, %c0 : memref<8x64xi8> - "test.source"(%b) : (memref<8x64xi8>) -> () - return -} - -// check that if pointer is extracted, the alloc is untraceable -// CHECK-DAG: func.func @extract_pointer() attributes {__mergealloc_scope = [[TOPSCOPEExt:[0-9]+]] -func.func @extract_pointer() { - // CHECK: %[[BExt:.*]] = memref.alloc() {__mergealloc_lifetime = array} - %b = memref.alloc() : memref<8x64xi8> - %0 = memref.extract_aligned_pointer_as_index %b : memref<8x64xi8> -> index - "test.source"(%b) : (memref<8x64xi8>) -> () - return -} - -// check that Alias Buffers' lifetimes work well -// CHECK-DAG: func.func @alias_ref(%[[ARG0:.*]]: i1) attributes {__mergealloc_scope = [[TOPSCOPE4:[0-9]+]] -func.func @alias_ref(%pred : i1) { - // CHECK: %[[A:.*]] = memref.alloc() {__mergealloc_lifetime = array} - %a = memref.alloc() : memref<8x64xi8> - // CHECK: %[[B:.*]] = memref.alloc() {__mergealloc_lifetime = array} - %b = memref.alloc() : memref<8x64xi8> - %c = arith.select %pred, %a, %b : i1, memref<8x64xi8> - "test.source"(%c) : (memref<8x64xi8>) -> () - "test.source"(%b) : (memref<8x64xi8>) -> () - return -} - -// CHECK-DAG: func.func @escape_from_if() attributes {__mergealloc_scope = [[TOPSCOPE5:[0-9]+]] -func.func @escape_from_if() { - %ctrue = arith.constant 1 : i1 - // check that f lives at the whole range of the following scf.if - // CHECK-DAG: %[[F:.*]] = memref.alloc() {__mergealloc_lifetime = array} - %f = memref.alloc() : memref<8x64xf32> - // tick of the scf.if starts from 4 and ends at 14 - // CHECK: scf.if - %c = scf.if %ctrue -> memref<8x64xf32> { - "test.source"(%f) : (memref<8x64xf32>) -> () - // CHECK-DAG: %[[G:.*]] = memref.alloc() {__mergealloc_lifetime = array} - %g = memref.alloc() : memref<8x64xf32> - "test.source"(%g) : (memref<8x64xf32>) -> () - scf.yield %g : memref<8x64xf32> - } else { - // h fully overlaps with g - // CHECK-DAG: %[[H:.*]] = memref.alloc() {__mergealloc_lifetime = array} - %h = memref.alloc() : memref<8x64xf32> - "test.source"(%h) : (memref<8x64xf32>) -> () - // J only used in the scf.if, don't need conservative lifetime - // CHECK-DAG: %[[J:.*]] = memref.alloc() {__mergealloc_lifetime = array} - %j = memref.alloc() : memref<8x64xf32> - "test.source"(%j) : (memref<8x64xf32>) -> () - scf.yield %h : memref<8x64xf32> - } - "test.source"(%c) : (memref<8x64xf32>) -> () - return -} - -// CHECK-DAG: func.func @escape_from_for() attributes {__mergealloc_scope = [[TOPSCOPE6:[0-9]+]] -func.func @escape_from_for() { - %c0 = arith.constant 0 : index - %c1 = arith.constant 1 : index - %c2 = arith.constant 2 : index - %c3 = arith.constant 3 : index - %c5 = arith.constant 5 : index - // check that f has untraceable lifetime, due to being yielded by for loop - // CHECK-DAG: %[[F:.*]] = memref.alloc() {__mergealloc_lifetime = array} - %f = memref.alloc() : memref<8x64xf32> - %out = scf.for %i = %c0 to %c5 step %c1 iter_args(%buf = %f) -> (memref<8x64xf32>) { - "test.source"(%buf) : (memref<8x64xf32>) -> () - // check that f has untraceable lifetime, due to being yielded by for loop - // CHECK-DAG: %[[G:.*]] = memref.alloc() {__mergealloc_lifetime = array} - %g = memref.alloc() : memref<8x64xf32> - "test.source"(%g) : (memref<8x64xf32>) -> () - %ctrue = "test.source"() : () -> i1 - %c = scf.if %ctrue -> memref<8x64xf32> { - scf.yield %g : memref<8x64xf32> - } else { - scf.yield %buf : memref<8x64xf32> - } - scf.yield %c : memref<8x64xf32> - } - "test.source"(%out) : (memref<8x64xf32>) -> () - return -} \ No newline at end of file diff --git a/test/mlir/test/gc/Transforms/buffer-merge.mlir b/test/mlir/test/gc/Transforms/buffer-merge.mlir deleted file mode 100644 index 3b3bdbd4f..000000000 --- a/test/mlir/test/gc/Transforms/buffer-merge.mlir +++ /dev/null @@ -1,396 +0,0 @@ -// RUN: gc-opt -allow-unregistered-dialect -one-shot-bufferize="unknown-type-conversion=identity-layout-map function-boundary-type-conversion=identity-layout-map bufferize-function-boundaries" --gc-merge-alloc %s | FileCheck %s - -func.func @mlp(%x: tensor<128x128xf32>, %y: tensor<128x128xf32>) -> tensor<128x128xf32> { - // CHECK-DAG: %[[ALLOC:.*]] = memref.alloc() {alignment = 64 : i64} : memref<131072xi8> - // CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index - // CHECK-DAG: %[[VIEW_A:.*]] = memref.view %[[ALLOC]][%[[C0]]][] : memref<131072xi8> to memref<128x128xf32> - %a0 = tensor.empty() : tensor<128x128xf32> - // CHECK: linalg.matmul ins - // CHECK-SAME: outs(%[[VIEW_A]] : memref<128x128xf32>) - %a = linalg.matmul ins(%x, %y: tensor<128x128xf32>, tensor<128x128xf32>) outs(%a0: tensor<128x128xf32>) -> tensor<128x128xf32> - // CHECK-DAG: %[[C65536:.*]] = arith.constant 65536 : index - // CHECK-DAG: %[[VIEW_B:.*]] = memref.view %[[ALLOC]][%[[C65536]]][] : memref<131072xi8> to memref<128x128xf32> - %b0 = tensor.empty() : tensor<128x128xf32> - // CHECK: linalg.matmul ins(%[[VIEW_A]], - // CHECK-SAME: outs(%[[VIEW_B]] : memref<128x128xf32>) - %b = linalg.matmul ins(%a, %y: tensor<128x128xf32>, tensor<128x128xf32>) outs(%b0: tensor<128x128xf32>) -> tensor<128x128xf32> - // CHECK-DAG: %[[C0_2:.*]] = arith.constant 0 : index - // CHECK-DAG: %[[VIEW_C:.*]] = memref.view %[[ALLOC]][%[[C0_2]]][] : memref<131072xi8> to memref<128x128xf32> - %c0 = tensor.empty() : tensor<128x128xf32> - // CHECK: linalg.matmul ins(%[[VIEW_B]], - // CHECK-SAME: outs(%[[VIEW_C]] : memref<128x128xf32>) - %c = linalg.matmul ins(%b, %y: tensor<128x128xf32>, tensor<128x128xf32>) outs(%c0: tensor<128x128xf32>) -> tensor<128x128xf32> - // CHECK-DAG: %[[D:.*]] = memref.alloc() {alignment = 64 : i64} : memref<128x128xf32> - // CHECK: linalg.matmul ins(%[[VIEW_C]], - // CHECK-SAME: outs(%[[D]] : memref<128x128xf32>) - %d0 = tensor.empty() : tensor<128x128xf32> - %d = linalg.matmul ins(%c, %y: tensor<128x128xf32>, tensor<128x128xf32>) outs(%d0: tensor<128x128xf32>) -> tensor<128x128xf32> - // CHECK: return %[[D]] - return %d : tensor<128x128xf32> -} - -// CHECK-LABEL: @basic -func.func @basic() -> memref<8x64xf32> { - // CHECK-DAG: %[[BASE:.*]] = memref.alloc() {alignment = 64 : i64} : memref<4096xi8> - // b is used in return, complex lifetime - // CHECK-DAG: %[[B:.*]] = memref.alloc() - %b = memref.alloc() : memref<8x64xf32> - // CHECK: "test.source"(%[[B]]) - "test.source"(%b) : (memref<8x64xf32>) -> () - // c and d has overlapping lifetime - // CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index - // CHECK-DAG: %[[C:.*]] = memref.view %[[BASE]][%[[C0]]][] : memref<4096xi8> to memref<8x64xf32> - %c = memref.alloc() : memref<8x64xf32> - // CHECK: "test.source"(%[[C]]) - "test.source"(%c) : (memref<8x64xf32>) -> () - // CHECK-DAG: %[[C2048:.*]] = arith.constant 2048 : index - // CHECK-DAG: %[[D:.*]] = memref.view %[[BASE]][%[[C2048]]][] : memref<4096xi8> to memref<8x64xf32> - %d = memref.alloc() : memref<8x64xf32> - // CHECK: "test.source"(%[[D]]) - "test.source"(%d) : (memref<8x64xf32>) -> () - // CHECK: "test.source"(%[[C]]) - "test.source"(%c) : (memref<8x64xf32>) -> () - // e can reuse the above memory - // CHECK-DAG: %[[C0_2:.*]] = arith.constant 0 : index - // CHECK-DAG: %[[E:.*]] = memref.view %[[BASE]][%[[C0_2]]][] : memref<4096xi8> to memref<8x64xf32> - %e = memref.alloc() : memref<8x64xf32> - // CHECK: "test.source"(%[[E]]) - "test.source"(%e) : (memref<8x64xf32>) -> () - // CHECK: return %[[B]] - return %b : memref<8x64xf32> -} - -// CHECK-LABEL: @different_size_and_dtype -func.func @different_size_and_dtype() { - // CHECK-DAG: %[[BASE_3:.*]] = memref.alloc() {alignment = 64 : i64} : memref<3072xi8> - // c and d has overlapping lifetime - // CHECK-DAG: %[[C0_3:.*]] = arith.constant 0 : index - // CHECK-DAG: %[[C_3:.*]] = memref.view %[[BASE_3]][%[[C0_3]]][] : memref<3072xi8> to memref<8x64xf32> - %c = memref.alloc() : memref<8x64xf32> - // CHECK: "test.source"(%[[C_3]]) - "test.source"(%c) : (memref<8x64xf32>) -> () - // CHECK-DAG: %[[C2048_3:.*]] = arith.constant 2048 : index - // CHECK-DAG: %[[D_3:.*]] = memref.view %[[BASE_3]][%[[C2048_3]]][] : memref<3072xi8> to memref<64xf32> - %d = memref.alloc() : memref<64xf32> - // CHECK: "test.source"(%[[D_3]]) - "test.source"(%d) : (memref<64xf32>) -> () - // last use of d - // e can reuse the d's memory - // CHECK-DAG: %[[C2048_3_2:.*]] = arith.constant 2048 : index - // CHECK-DAG: %[[E_3:.*]] = memref.view %[[BASE_3]][%[[C2048_3_2]]][] : memref<3072xi8> to memref<2x2x32xf64> - %e = memref.alloc() : memref<2x2x32xf64> - // CHECK: "test.source"(%[[E_3]]) - "test.source"(%e) : (memref<2x2x32xf64>) -> () - // CHECK: "test.source"(%[[C_3]]) - "test.source"(%c) : (memref<8x64xf32>) -> () - - // e and c are free'd. f can reuse the memory across e and c - // CHECK-DAG: %[[C0_3_2:.*]] = arith.constant 0 : index - // CHECK-DAG: %[[F_3:.*]] = memref.view %[[BASE_3]][%[[C0_3_2]]][] : memref<3072xi8> to memref<2x4x32xf64> - %f = memref.alloc() : memref<2x4x32xf64> - // CHECK: "test.source"(%[[F_3]]) - "test.source"(%f) : (memref<2x4x32xf64>) -> () - // CHECK: return - return -} - -// CHECK-LABEL: @withloop -func.func @withloop() { - // CHECK-DAG: %[[BASE2:.*]] = memref.alloc() {alignment = 64 : i64} : memref<6144xi8> - // CHECK-DAG: %[[C033:.*]] = arith.constant 0 : index - // CHECK-DAG: %[[F:.*]] = memref.view %[[BASE2]][%[[C033]]][] : memref<6144xi8> to memref<8x64xf32> - %f = memref.alloc() : memref<8x64xf32> - // CHECK-DAG: %[[C2048:.*]] = arith.constant 2048 : index - // CHECK-DAG: %[[G:.*]] = memref.view %[[BASE2]][%[[C2048]]][] : memref<6144xi8> to memref<8x64xf32> - %g = memref.alloc() : memref<8x64xf32> - - %c0 = arith.constant 0 : index - %c1 = arith.constant 1 : index - %c2 = arith.constant 2 : index - %c3 = arith.constant 3 : index - %c5 = arith.constant 5 : index - // CHECK: scf.for - scf.for %i = %c0 to %c5 step %c1 { - // CHECK: "test.source"(%[[F]]) - "test.source"(%f) : (memref<8x64xf32>) -> () - // CHECK: "test.source"(%[[G]]) - "test.source"(%g) : (memref<8x64xf32>) -> () - // CHECK-DAG: %[[C4096:.*]] = arith.constant 4096 : index - // CHECK-DAG: %[[H:.*]] = memref.view %[[BASE2]][%[[C4096]]][] : memref<6144xi8> to memref<8x64xf32> - %h = memref.alloc() : memref<8x64xf32> - // CHECK: "test.source"(%[[H]]) - "test.source"(%h) : (memref<8x64xf32>) -> () - // CHECK-DAG: %[[C4096_3:.*]] = arith.constant 4096 : index - // CHECK-DAG: %[[J:.*]] = memref.view %[[BASE2]][%[[C4096_3]]][] : memref<6144xi8> to memref<8x64xf32> - %j = memref.alloc() : memref<8x64xf32> - // CHECK: "test.source"(%[[J]]) - "test.source"(%j) : (memref<8x64xf32>) -> () - } - return -} - - -// CHECK-LABEL: @different_mem_space -func.func @different_mem_space() { - // CHECK-DAG: %[[BASE_DEFAULT:.*]] = memref.alloc() {alignment = 64 : i64} : memref<4096xi8> - // CHECK-DAG: %[[BASE_WG:.*]] = memref.alloc() {alignment = 64 : i64} : memref<3584xi8, #gpu.address_space> - // CHECK-DAG: %[[BASE_PRIVATE:.*]] = memref.alloc() {alignment = 64 : i64} : memref<1280xi8, #gpu.address_space> - // CHECK-DAG: %[[C0_4:.*]] = arith.constant 0 : index - // CHECK-DAG: %[[defaultv:.*]] = memref.view %[[BASE_DEFAULT]][%[[C0_4]]][] : memref<4096xi8> to memref<8x64xf32> - %defaultv = memref.alloc() : memref<8x64xf32> - // CHECK: "test.source"(%[[defaultv]]) - "test.source"(%defaultv) : (memref<8x64xf32>) -> () - // CHECK-DAG: %[[C2048_4:.*]] = arith.constant 2048 : index - // CHECK-DAG: %[[defaultv2:.*]] = memref.view %[[BASE_DEFAULT]][%[[C2048_4]]][] : memref<4096xi8> to memref<8x64xf32> - %defaultv2 = memref.alloc() : memref<8x64xf32> - // CHECK-DAG: %[[C2048_4_2:.*]] = arith.constant 0 : index - // CHECK-DAG: %[[priv:.*]] = memref.view %[[BASE_PRIVATE]][%[[C2048_4_2]]][] : memref<1280xi8, #gpu.address_space> to memref<4x64xf32, #gpu.address_space> - %priv = memref.alloc() : memref<4x64xf32, #gpu.address_space> - // CHECK: "test.source"(%[[priv]]) - "test.source"(%priv) : (memref<4x64xf32, #gpu.address_space>) -> () - // CHECK-DAG: %[[C0_4_2:.*]] = arith.constant 1024 : index - // CHECK-DAG: %[[priv2:.*]] = memref.view %[[BASE_PRIVATE]][%[[C0_4_2]]][] : memref<1280xi8, #gpu.address_space> to memref<64xf32, #gpu.address_space> - %priv2 = memref.alloc() : memref<64xf32, #gpu.address_space> - // CHECK: "test.source"(%[[priv2]]) - "test.source"(%priv2) : (memref<64xf32, #gpu.address_space>) -> () - // CHECK: "test.source"(%[[priv]]) - "test.source"(%priv) : (memref<4x64xf32, #gpu.address_space>) -> () - - - // CHECK-DAG: %[[C2048_4_3:.*]] = arith.constant 1536 : index - // CHECK-DAG: %[[sharedv:.*]] = memref.view %[[BASE_WG]][%[[C2048_4_3]]][] : memref<3584xi8, #gpu.address_space> to memref<8x64xf32, #gpu.address_space> - %sharedv = memref.alloc() : memref<8x64xf32, #gpu.address_space> - // CHECK-DAG: %[[C0_4_3:.*]] = arith.constant 0 : index - // CHECK-DAG: %[[sharedv2:.*]] = memref.view %[[BASE_WG]][%[[C0_4_3]]][] : memref<3584xi8, #gpu.address_space> to memref<6x64xf32, #gpu.address_space> - %sharedv2 = memref.alloc() : memref<6x64xf32, #gpu.address_space> - // CHECK: "test.source"(%[[sharedv2]]) - // CHECK: "test.source"(%[[sharedv]]) - // CHECK: "test.source"(%[[sharedv2]]) - "test.source"(%sharedv2) : (memref<6x64xf32, #gpu.address_space>) -> () - "test.source"(%sharedv) : (memref<8x64xf32, #gpu.address_space>) -> () - "test.source"(%sharedv2) : (memref<6x64xf32, #gpu.address_space>) -> () - - - // CHECK: "test.source"(%[[defaultv2]]) - // CHECK: "test.source"(%[[defaultv]]) - "test.source"(%defaultv2) : (memref<8x64xf32>) -> () - "test.source"(%defaultv) : (memref<8x64xf32>) -> () - return -} - -func.func @nested_forall(%arg0: memref<2xf32>) { - %c0 = arith.constant 0 : index - %c2 = arith.constant 2 : index - %c4 = arith.constant 4 : index - %c16 = arith.constant 16 : index - %alloc = memref.alloc() : memref<2xf32> - memref.copy %arg0, %alloc : memref<2xf32> to memref<2xf32> - scf.forall (%i) = (%c0) to (%c16) step (%c2) { - %alloc_0 = memref.alloc() : memref<4xf32> - %alloc_1 = memref.alloc() : memref<8xf32> // hoist to outermost - scf.forall (%j) in (%c4) { - "test.source"(%alloc) : (memref<2xf32>) -> () - "test.source"(%alloc_1) : (memref<8xf32>) -> () - %alloc_2 = memref.alloc() : memref<16xf32> - "test.source"(%alloc_2) : (memref<16xf32>) -> () - } - "test.source"(%alloc) : (memref<2xf32>) -> () - "test.source"(%alloc_0) : (memref<4xf32>) -> () - } - return -} -// CHECK-LABEL: func @nested_forall -// CHECK: %[[ALLOC0:.*]] = memref.alloc() {alignment = 64 : i64} : memref<320xi8> -// CHECK: %[[VIEW0:.*]] = memref.view %[[ALLOC0]][%{{.*}}][] : memref<320xi8> to memref<2xf32> -// CHECK-NEXT: memref.copy %arg0, %[[VIEW0]] -// CHECK-NEXT: scf.forall -// CHECK: %[[ALLOC1:.*]] = memref.alloc() {alignment = 64 : i64} : memref<64xi8> -// CHECK: %[[VIEW1:.*]] = memref.view %[[ALLOC1]][%{{.*}}][] : memref<64xi8> to memref<4xf32> -// CHECK: %[[VIEW2:.*]] = memref.view %[[ALLOC0]][%{{.*}}][] : memref<320xi8> to memref<8xf32> -// CHECK-NEXT: scf.forall -// CHECK-NEXT: %[[ALLOC2:.*]] = memref.alloc() {alignment = 64 : i64} : memref<64xi8> -// CHECK: %[[VIEW3:.*]] = memref.view %[[ALLOC2]][%{{.*}}][] : memref<64xi8> to memref<16xf32> - -func.func @nested_forall_multi_var(%arg0: memref<2xf32>) { - %c0 = arith.constant 0 : index - %c1 = arith.constant 1 : index - %c2 = arith.constant 2 : index - %c4 = arith.constant 4 : index - %c16 = arith.constant 16 : index - %alloc = memref.alloc() : memref<2xf32> - memref.copy %arg0, %alloc : memref<2xf32> to memref<2xf32> - scf.forall (%i, %j) = (%c0, %c0) to (%c16, %c4) step (%c4, %c1) { - %alloc_0 = memref.alloc() : memref<4xf32> - %alloc_1 = memref.alloc() : memref<8xf32> // hoist to outermost - scf.forall (%k) in (%c4) { - "test.source"(%alloc) : (memref<2xf32>) -> () - "test.source"(%alloc_1) : (memref<8xf32>) -> () - %alloc_2 = memref.alloc() : memref<16xf32> - "test.source"(%alloc_2) : (memref<16xf32>) -> () - } - "test.source"(%alloc) : (memref<2xf32>) -> () - "test.source"(%alloc_0) : (memref<4xf32>) -> () - } - return -} -// CHECK-LABEL: func @nested_forall_multi_var -// CHECK: %[[ALLOC0:.*]] = memref.alloc() {alignment = 64 : i64} : memref<576xi8> -// CHECK: %[[VIEW0:.*]] = memref.view %[[ALLOC0]][%{{.*}}][] : memref<576xi8> to memref<2xf32> -// CHECK-NEXT: memref.copy %arg0, %[[VIEW0]] -// CHECK-NEXT: scf.forall -// CHECK: %[[ALLOC1:.*]] = memref.alloc() {alignment = 64 : i64} : memref<64xi8> -// CHECK: %[[VIEW1:.*]] = memref.view %[[ALLOC1]][%{{.*}}][] : memref<64xi8> to memref<4xf32> -// CHECK: %[[VIEW2:.*]] = memref.view %[[ALLOC0]][%{{.*}}][] : memref<576xi8> to memref<8xf32> -// CHECK-NEXT: scf.forall -// CHECK-NEXT: %[[ALLOC2:.*]] = memref.alloc() {alignment = 64 : i64} : memref<64xi8> -// CHECK: %[[VIEW3:.*]] = memref.view %[[ALLOC2]][%{{.*}}][] : memref<64xi8> to memref<16xf32> - -func.func @mixed_forall_and_for( - %lb: index, - %ub: index, - %step: index, - %arg0: memref<2xf32>) { - %c4 = arith.constant 4 : index - %c16 = arith.constant 16 : index - %0 = memref.alloc() : memref<2xf32> - scf.forall (%i) in (%c16) { - %alloc = memref.alloc() : memref<4xf32> //hoist to outermost - scf.forall (%j) in (%c4) { - %alloc_0 = memref.alloc() : memref<8xf32> - %1 = scf.for %k = %lb to %ub step %step - iter_args(%iterBuf = %arg0) -> (memref<2xf32>) { - "test.source"(%alloc) : (memref<4xf32>) -> () - "test.source"(%alloc_0) : (memref<8xf32>) -> () - scf.yield %0 : memref<2xf32> - } - } - } - return -} -// CHECK-LABEL: func @mixed_forall_and_for -// CHECK: %[[ALLOC0:.*]] = memref.alloc() {alignment = 64 : i64} : memref<256xi8> -// CHECK: %[[ALLOC1:.*]] = memref.alloc() : memref<2xf32> -// CHECK-NEXT: scf.forall -// CHECK: %[[VIEW0:.*]] = memref.view %[[ALLOC0]][%{{.*}}][] : memref<256xi8> to memref<4xf32> -// CHECK-NEXT: scf.forall -// CHECK-NEXT: %[[ALLOC2:.*]] = memref.alloc() {alignment = 64 : i64} : memref<64xi8> -// CHECK: %[[VIEW1:.*]] = memref.view %[[ALLOC2]][%{{.*}}][] : memref<64xi8> to memref<8xf32> -// CHECK-NEXT: scf.for -// CHECK: scf.yield %[[ALLOC1]] - -func.func @nested_forall_with_dynamic_shape(%arg0: index) { - %c4 = arith.constant 4 : index - %c16 = arith.constant 16 : index - scf.forall (%i) in (%c16) { - %alloc = memref.alloc(%arg0) : memref - scf.forall (%j) in (%c4) { - "test.source"(%alloc) : (memref) -> () - } - } - return -} -// CHECK-LABEL: func @nested_forall_with_dynamic_shape -// CHECK-NOT: memref.alloc -// CHECK: scf.forall -// CHECK-NEXT: %[[ALLOC0:.*]] = memref.alloc(%arg0) : memref -// CHECK-NEXT: scf.forall - -func.func @nested_forall_with_dynamic_range(%num_threads: index) { - %c4 = arith.constant 4 : index - %c16 = arith.constant 16 : index - scf.forall (%i) in (%num_threads) { - %alloc = memref.alloc() : memref<2xf32> - scf.forall (%j) in (%c4) { - %alloc_0 = memref.alloc() : memref<4xf32> - "test.source"(%alloc) : (memref<2xf32>) -> () - "test.source"(%alloc_0) : (memref<4xf32>) -> () - } - } - return -} -// CHECK-LABEL: func @nested_forall_with_dynamic_range -// CHECK-NOT: memref.alloc -// CHECK: scf.forall -// CHECK-NEXT: %[[ALLOC0:.*]] = memref.alloc() {alignment = 64 : i64} : memref<64xi8> -// CHECK: %[[VIEW0:.*]] = memref.view %[[ALLOC0]][%{{.*}}][] : memref<64xi8> to memref<2xf32> -// CHECK-NEXT: scf.forall -// CHECK: %[[ALLOC1:.*]] = memref.alloc() {alignment = 64 : i64} : memref<64xi8> -// CHECK: %[[VIEW1:.*]] = memref.view %[[ALLOC1]][%{{.*}}][] : memref<64xi8> to memref<4xf32> - -func.func @nested_forall_with_multi_blocks( - %lb: index, - %ub: index, - %step: index, - %arg0: memref<2xf32>) { - %c4 = arith.constant 4 : index - %c8 = arith.constant 8 : index - %c16 = arith.constant 16 : index - %0 = memref.alloc() : memref<2xf32> - scf.forall (%i) in (%c16) { - scf.forall(%j1) in (%c4) { - %alloc = memref.alloc() : memref<4xf32> //hoist to outermost - scf.forall (%k1) in (%c4) { - %1 = scf.for %kk = %lb to %ub step %step - iter_args(%iterBuf = %arg0) -> (memref<2xf32>) { - %alloc_0 = memref.alloc() : memref<2xf32> //hoist to upper - "test.source"(%alloc) : (memref<4xf32>) -> () - "test.source"(%alloc_0) : (memref<2xf32>) -> () - scf.yield %0 : memref<2xf32> - } - } - } - - scf.forall (%j2) in (%c8) { - %alloc_2 = memref.alloc() : memref<8xf32> //hoist - scf.forall (%k2) in (%c4) { - "test.source"(%alloc_2) : (memref<8xf32>) -> () - } - } - } - return -} -// CHECK-LABEL: func @nested_forall_with_multi_blocks -// CHECK: %[[ALLOC0:.*]] = memref.alloc() {alignment = 64 : i64} : memref<4096xi8> -// CHECK: %[[ALLOC1:.*]] = memref.alloc() : memref<2xf32> -// CHECK-NEXT: scf.forall -// CHECK-NEXT: scf.forall -// CHECK-NOT: memref.alloc -// CHECK: %[[VIEW1:.*]] = memref.view %[[ALLOC0]][%{{.*}}][] : memref<4096xi8> to memref<4xf32> -// CHECK-NEXT: scf.forall -// CHECK: %[[ALLOC2:.*]] = memref.alloc() {alignment = 64 : i64} : memref<64xi8> -// CHECK: scf.for -// CHECK: %[[VIEW2:.*]] = memref.view %[[ALLOC2]][%{{.*}}][] : memref<64xi8> to memref<2xf32> -// CHECK: scf.yield %[[ALLOC1]] : memref<2xf32> -// CHECK: scf.forall -// CHECK-NOT: memref.alloc -// CHECK: %[[VIEW3:.*]] = memref.view %[[ALLOC0]][%{{.*}}][] : memref<4096xi8> to memref<8xf32> -// CHECK-NEXT: scf.forall - -func.func @nested_forall_nd() { - %c4 = arith.constant 4 : index - %c16 = arith.constant 16 : index - scf.forall (%i) in (%c16) { - %alloc_0 = memref.alloc() : memref<2x4xf32> // hoist to outermost - %alloc_1 = memref.alloc() : memref<4x8xf32> - scf.forall (%j) in (%c4) { - "test.source"(%alloc_0) : (memref<2x4xf32>) -> () - %alloc_2 = memref.alloc() : memref<2x16xf32> - "test.source"(%alloc_2) : (memref<2x16xf32>) -> () - } - "test.source"(%alloc_1) : (memref<4x8xf32>) -> () - } - return -} -// CHECK-LABEL: func @nested_forall_nd -// CHECK: %[[ALLOC0:.*]] = memref.alloc() {alignment = 64 : i64} : memref<512xi8> -// CHECK: scf.forall -// CHECK: %[[ALLOC1:.*]] = memref.alloc() {alignment = 64 : i64} : memref<128xi8> -// CHECK: %[[C0:.*]] = arith.constant 0 : index -// CHECK: %[[C32:.*]] = arith.constant 32 : index -// CHECK: %[[V0:.*]] = arith.muli %arg0, %[[C32]] : index -// CHECK: %[[V1:.*]] = arith.addi %[[C0]], %[[V0]] : index -// CHECK: %[[VIEW0:.*]] = memref.view %[[ALLOC0]][%[[V1]]][] : memref<512xi8> to memref<2x4xf32> -// CHECK: %[[VIEW1:.*]] = memref.view %[[ALLOC1]][%{{.*}}][] : memref<128xi8> to memref<4x8xf32> -// CHECK-NEXT: scf.forall -// CHECK-NEXT: %[[ALLOC2:.*]] = memref.alloc() {alignment = 64 : i64} : memref<128xi8> -// CHECK: %[[VIEW2:.*]] = memref.view %[[ALLOC2]][%{{.*}}][] : memref<128xi8> to memref<2x16xf32> diff --git a/test/mlir/test/gc/Transforms/deepTileContractionNamedOp.mlir b/test/mlir/test/gc/Transforms/deepTileContractionNamedOp.mlir deleted file mode 100644 index 61848dcb7..000000000 --- a/test/mlir/test/gc/Transforms/deepTileContractionNamedOp.mlir +++ /dev/null @@ -1,311 +0,0 @@ -// RUN: gc-opt --split-input-file --deep-tile-contraction-op %s | FileCheck %s - -// ----- - -/// CHECK-LABEL: @matmul_2Dx2D_f32 -func.func @matmul_2Dx2D_f32(%arg0: tensor<4096x4096xf32>, %arg1: tensor<4096x4096xf32>) -> tensor<4096x4096xf32> { - %cst_0 = arith.constant 0.000000e+00 : f32 - %0 = tensor.empty() : tensor<4096x4096xf32> - %1 = linalg.fill ins(%cst_0 : f32) outs(%0 : tensor<4096x4096xf32>) -> tensor<4096x4096xf32> - // CHECK: scf.forall {{.*}} (0) to (4096) step (1024) {{.*}} (tensor<4096x4096xf32>) { - // CHECK: tensor.extract_slice {{.*}} [1024, 4096] [1, 1] - // CHECK: scf.forall {{.*}} (0) to (4096) step (2048) {{.*}} (tensor<1024x4096xf32>) - // CHECK: tensor.extract_slice {{.*}} [1024, 2048] [1, 1] - // CHECK: scf.for - // CHECK: tensor.extract_slice {{.*}} [256, 2048] [1, 1] - // CHECK: scf.for - // CHECK: tensor.extract_slice {{.*}} [256, 256] [1, 1] - // CHECK: scf.for - // CHECK: scf.for - // CHECK: tensor.extract_slice {{.*}} [32, 256] [1, 1] - // CHECK: tensor.extract_slice {{.*}} [32, 256] [1, 1] - // CHECK: scf.for - // CHECK: tensor.extract_slice {{.*}} [256, 32] [1, 1] - // CHECK: tensor.extract_slice {{.*}} [32, 32] [1, 1] - // CHECK: linalg.transpose {{.*}} permutation = [1, 0, 2] - // CHECK: tensor.expand_shape {{.*}} output_shape [8, 32, 32] : tensor<256x32xf32> into tensor<8x32x32xf32> - // CHECK: scf.if - // CHECK: linalg.fill - // CHECK: linalg.batch_reduce_matmul - // CHECK: else - // CHECK: linalg.batch_reduce_matmul - // CHECK: tensor.insert_slice {{.*}} [32, 256] [1, 1] - %2 = linalg.matmul {MThreads = 4 : i32, NThreads = 2 : i32, KThreads = 1 : i32, MBlock = 256 : i32, NBlock = 256 : i32, KBlock = 256 : i32,innermostMBlock = 32 : i32, innermostNBlock = 32 : i32, innermostKBlock = 32 : i32 } ins(%arg0, %arg1 : tensor<4096x4096xf32>, tensor<4096x4096xf32>) outs(%1 : tensor<4096x4096xf32>) -> tensor<4096x4096xf32> - return %2 : tensor<4096x4096xf32> -} - -// ----- - -// CHECK: #[[mapA:.+]] = affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d3 * 2 + d4)> -// CHECK: #[[mapB:.+]] = affine_map<(d0, d1, d2, d3, d4) -> (d0, d3, d2, d4)> -// CHECK: #[[mapC:.+]] = affine_map<(d0, d1, d2, d3, d4) -> (d1, d2)> -/// CHECK-LABEL: @matmul_4Dx4D_bf16 -func.func @matmul_4Dx4D_bf16(%arg0: tensor<128x128x32x32xbf16>, %arg1: tensor<128x128x16x32x2xbf16>) -> tensor<128x128x32x32xbf16> { - %cst_0 = arith.constant 0.000000e+00 : bf16 - // CHECK: tensor.empty() : tensor<128x128x32x32xbf16> - %0 = tensor.empty() : tensor<128x128x32x32xbf16> - // CHECK-NOT: linalg.fill - %1 = linalg.fill ins(%cst_0 : bf16) outs(%0 : tensor<128x128x32x32xbf16>) -> tensor<128x128x32x32xbf16> - // CHECK: scf.forall {{.*}} (0) to (128) step (8) {{.*}} (tensor<128x128x32x32xbf16>) - // CHECK: tensor.extract_slice {{.*}} [8, 128, 32, 32] [1, 1, 1, 1] - // CHECK: scf.forall {{.*}} (0) to (128) step (64) {{.*}} (tensor<8x128x32x32xbf16>) - // CHECK: tensor.extract_slice {{.*}} [8, 64, 32, 32] [1, 1, 1, 1] - // CHECK: scf.for - // CHECK: tensor.extract_slice {{.*}} [8, 8, 32, 32] [1, 1, 1, 1] - // CHECK: tensor.empty() : tensor<8x8x32x32xf32> - // CHECK: scf.for - // CHECK: scf.for - // CHECK: tensor.extract_slice {{.*}} [1, 8, 32, 32] [1, 1, 1, 1] - // CHECK: tensor.extract_slice {{.*}} [1, 8, 32, 32] [1, 1, 1, 1] - // CHECK: scf.for - // CHECK: tensor.collapse_shape {{.*}} tensor<1x8x32x32xbf16> into tensor<8x32x32xbf16> - // CHECK: tensor.extract_slice {{.*}} [1, 8, 16, 32, 2] [1, 1, 1, 1, 1] - // CHECK: tensor.collapse_shape {{.*}} tensor<1x8x16x32x2xbf16> into tensor<8x16x32x2xbf16> - // CHECK: tensor.extract_slice {{.*}} [1, 1, 32, 32] [1, 1, 1, 1] - // CHECK: tensor.collapse_shape {{.*}} tensor<1x1x32x32xf32> into tensor<32x32xf32> - // CHECK: tensor.extract_slice {{.*}} [1, 1, 32, 32] [1, 1, 1, 1] - // CHECK: tensor.collapse_shape {{.*}} tensor<1x1x32x32xbf16> into tensor<32x32xbf16> - // CHECK: scf.if - // CHECK: linalg.fill - // CHECK: linalg.generic {indexing_maps = [#[[mapA]], #[[mapB]], #[[mapC]]], iterator_types = ["reduction", "parallel", "parallel", "reduction", "reduction"]} - // CHECK: else - // CHECK: linalg.generic {indexing_maps = [#[[mapA]], #[[mapB]], #[[mapC]]], iterator_types = ["reduction", "parallel", "parallel", "reduction", "reduction"]} - // CHECK: scf.if - // CHECK: linalg.copy - // CHECK: else - %2 = linalg.generic { - MThreads = 16 : i32, NThreads = 2 : i32, KThreads = 1 : i32, MBlock = 256 : i32, NBlock = 256 : i32, KBlock = 256 : i32,innermostMBlock = 32 : i32, innermostNBlock = 32 : i32, innermostKBlock = 32 : i32, - indexing_maps = [affine_map<(d0, d1, d2, d3, d4, d5, d6) -> (d0, d4, d1, d5 * 2 + d6)>, - affine_map<(d0, d1, d2, d3, d4, d5, d6) -> (d2, d4, d5, d3, d6)>, - affine_map<(d0, d1, d2, d3, d4, d5, d6) -> (d0, d2, d1, d3)>], - iterator_types = ["parallel", "parallel", "parallel", "parallel", "reduction", "reduction", "reduction"] - } - ins(%arg0, %arg1 : tensor<128x128x32x32xbf16>, tensor<128x128x16x32x2xbf16>) - outs(%1 : tensor<128x128x32x32xbf16>) { - ^bb0(%in: bf16, %in_0: bf16, %out: bf16): - %b0 = arith.mulf %in, %in_0 : bf16 - %b1 = arith.addf %out, %b0 : bf16 - linalg.yield %b1 : bf16 - } -> tensor<128x128x32x32xbf16> - - return %2 : tensor<128x128x32x32xbf16> -} - -// ----- - -// CHECK: #[[mapA:.+]] = affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d3 * 2 + d4)> -// CHECK: #[[mapB:.+]] = affine_map<(d0, d1, d2, d3, d4) -> (d0, d3, d2, d4)> -// CHECK: #[[mapC:.+]] = affine_map<(d0, d1, d2, d3, d4) -> (d1, d2)> -/// CHECK-LABEL: @matmul_2Dx4D_bf16 -func.func @matmul_2Dx4D_bf16(%arg0: tensor<4096x4096xbf16>, %arg1: tensor<128x128x16x32x2xbf16>) -> tensor<4096x4096xbf16> { - %cst_0 = arith.constant 0.000000e+00 : bf16 - %0 = tensor.empty() : tensor<4096x4096xbf16> - %1 = linalg.fill ins(%cst_0 : bf16) outs(%0 : tensor<4096x4096xbf16>) -> tensor<4096x4096xbf16> - // CHECK: scf.forall {{.*}} (2) {{.*}} (tensor<2x1x1x4096x4096xf32>) - // CHECK: tensor.extract_slice {{.*}} [1, 1, 1, 4096, 4096] [1, 1, 1, 1, 1] - // CHECK: scf.forall {{.*}} (0) to (4096) step (256) {{.*}} (tensor<4096x4096xf32>) - // CHECK: tensor.extract_slice {{.*}} [256, 4096] [1, 1] - // CHECK: scf.forall {{.*}} (0) to (128) step (64) {{.*}} (tensor<256x4096xf32>) - // CHECK: tensor.extract_slice {{.*}} [256, 2048] [1, 1] - // CHECK: scf.for - // CHECK: tensor.extract_slice {{.*}} [256, 256] [1, 1] - // CHECK: scf.for - // CHECK: scf.for - // CHECK: tensor.extract_slice {{.*}} [32, 256] [1, 1] - // CHECK: tensor.extract_slice {{.*}} [32, 256] [1, 1] - // CHECK: scf.for - // CHECK: tensor.extract_slice {{.*}} [1, 8, 16, 32, 2] [1, 1, 1, 1, 1] - // CHECK: tensor.collapse_shape {{.*}} tensor<1x8x16x32x2xbf16> into tensor<8x16x32x2xbf16> - // CHECK: tensor.extract_slice {{.*}} [32, 32] [1, 1] - // CHECK: linalg.transpose {{.*}} permutation = [1, 0, 2] - // CHECK: scf.if - // CHECK: linalg.fill - // CHECK: linalg.generic {indexing_maps = [#[[mapA]], #[[mapB]], #[[mapC]]], iterator_types = ["reduction", "parallel", "parallel", "reduction", "reduction"]} - // CHECK: else - // CHECK: linalg.generic {indexing_maps = [#[[mapA]], #[[mapB]], #[[mapC]]], iterator_types = ["reduction", "parallel", "parallel", "reduction", "reduction"]} - // CHECK: scf.forall.in_parallel - // CHECK: scf.forall.in_parallel - // CHECK: scf.forall.in_parallel - // CHECK: linalg.reduce {{.*}} dimensions = [0, 1, 2] - // CHECK: linalg.copy - %2 = linalg.generic { - MThreads = 32 : i32, NThreads = 2 : i32, KThreads = 2 : i32, MBlock = 256 : i32, NBlock = 256 : i32, KBlock = 256 : i32,innermostMBlock = 32 : i32, innermostNBlock = 32 : i32, innermostKBlock = 32 : i32, - indexing_maps = [affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d3 * 32 + d4 * 2 + d5)>, - affine_map<(d0, d1, d2, d3, d4, d5) -> (d1, d3, d4, d2, d5)>, - affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d1 * 32 + d2)>], - iterator_types = ["parallel", "parallel", "parallel", "reduction", "reduction", "reduction"] - } - ins(%arg0, %arg1 : tensor<4096x4096xbf16>, tensor<128x128x16x32x2xbf16>) - outs(%1 : tensor<4096x4096xbf16>) { - ^bb0(%in: bf16, %in_0: bf16, %out: bf16): - %b0 = arith.mulf %in, %in_0 : bf16 - %b1 = arith.addf %out, %b0 : bf16 - linalg.yield %b1 : bf16 - } -> tensor<4096x4096xbf16> - - return %2 : tensor<4096x4096xbf16> -} - -// ----- - -module attributes { - dlti.target_system_spec = #dlti.target_system_spec< - "CPU": #dlti.target_device_spec< - #dlti.dl_entry<"L1_cache_size_in_bytes", 49152 : i32>, - #dlti.dl_entry<"L2_cache_size_in_bytes", 2097152 : i32>, - #dlti.dl_entry<"L3_cache_size_in_bytes", 110100480 : i32>, - #dlti.dl_entry<"num_threads", 56 : i32>, - #dlti.dl_entry<"max_vector_width", 512 : i32>> - >} { - // CHECK: #[[mapA:.+]] = affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d3 * 2 + d4)> - // CHECK: #[[mapB:.+]] = affine_map<(d0, d1, d2, d3, d4) -> (d0, d3, d2, d4)> - // CHECK: #[[mapC:.+]] = affine_map<(d0, d1, d2, d3, d4) -> (d1, d2)> - /// CHECK-LABEL: @matmul_2Dx4D_bf16_with_dlti -func.func @matmul_2Dx4D_bf16_with_dlti(%arg0: tensor<4096x4096xbf16>, %arg1: tensor<128x128x16x32x2xbf16>) -> tensor<4096x4096xbf16> { - %cst_0 = arith.constant 0.000000e+00 : bf16 - %0 = tensor.empty() : tensor<4096x4096xbf16> - %1 = linalg.fill ins(%cst_0 : bf16) outs(%0 : tensor<4096x4096xbf16>) -> tensor<4096x4096xbf16> - // CHECK: scf.forall - // CHECK: tensor.extract_slice - // CHECK: scf.for - // CHECK: tensor.extract_slice - // CHECK: scf.for - // CHECK: tensor.extract_slice - // CHECK: tensor.extract_slice - // CHECK: scf.for - // CHECK: tensor.extract_slice - // CHECK: tensor.extract_slice - // CHECK: linalg.transpose - // CHECK: scf.if - // CHECK: linalg.fill - // CHECK: linalg.generic {indexing_maps = [#[[mapA]], #[[mapB]], #[[mapC]]], iterator_types = ["reduction", "parallel", "parallel", "reduction", "reduction"]} - // CHECK: else - // CHECK: linalg.generic {indexing_maps = [#[[mapA]], #[[mapB]], #[[mapC]]], iterator_types = ["reduction", "parallel", "parallel", "reduction", "reduction"]} - // CHECK: scf.forall.in_parallel - %2 = linalg.generic { - indexing_maps = [affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d3 * 32 + d4 * 2 + d5)>, - affine_map<(d0, d1, d2, d3, d4, d5) -> (d1, d3, d4, d2, d5)>, - affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d1 * 32 + d2)>], - iterator_types = ["parallel", "parallel", "parallel", "reduction", "reduction", "reduction"] - } - ins(%arg0, %arg1 : tensor<4096x4096xbf16>, tensor<128x128x16x32x2xbf16>) - outs(%1 : tensor<4096x4096xbf16>) { - ^bb0(%in: bf16, %in_0: bf16, %out: bf16): - %b0 = arith.mulf %in, %in_0 : bf16 - %b1 = arith.addf %out, %b0 : bf16 - linalg.yield %b1 : bf16 - } -> tensor<4096x4096xbf16> - return %2 : tensor<4096x4096xbf16> -} - -} - -// ----- - -// CHECK: #[[mapA:.+]] = affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d3 * 2 + d4)> -// CHECK: #[[mapB:.+]] = affine_map<(d0, d1, d2, d3, d4) -> (d0, d3, d2, d4)> -// CHECK: #[[mapC:.+]] = affine_map<(d0, d1, d2, d3, d4) -> (d1, d2)> -/// CHECK-LABEL: @matmul_4Dx4D_bf16_generic -func.func @matmul_4Dx4D_bf16_generic(%arg0: tensor<128x128x32x32xbf16>, %arg1: tensor<128x128x16x32x2xbf16>) -> tensor<128x128x32x32xbf16> { - %cst_0 = arith.constant 0.000000e+00 : bf16 - // CHECK: tensor.empty() : tensor<128x128x32x32xbf16> - %0 = tensor.empty() : tensor<128x128x32x32xbf16> - // CHECK-NOT: linalg.fill - %1 = linalg.fill ins(%cst_0 : bf16) outs(%0 : tensor<128x128x32x32xbf16>) -> tensor<128x128x32x32xbf16> - // CHECK: scf.forall {{.*}} (0) to (128) step (8) {{.*}} (tensor<128x128x32x32xbf16>) - // CHECK: tensor.extract_slice {{.*}} [8, 128, 32, 32] [1, 1, 1, 1] - // CHECK: scf.forall {{.*}} (0) to (128) step (64) {{.*}} (tensor<8x128x32x32xbf16>) - // CHECK: tensor.extract_slice {{.*}} [8, 64, 32, 32] [1, 1, 1, 1] - // CHECK: scf.for - // CHECK: tensor.extract_slice {{.*}} [8, 8, 32, 32] [1, 1, 1, 1] - // CHECK: tensor.empty() : tensor<8x8x32x32xf32> - // CHECK: scf.for - // CHECK: scf.for - // CHECK: tensor.extract_slice {{.*}} [1, 8, 32, 32] [1, 1, 1, 1] - // CHECK: tensor.extract_slice {{.*}} [1, 8, 32, 32] [1, 1, 1, 1] - // CHECK: scf.for - // CHECK: tensor.collapse_shape {{.*}} tensor<1x8x32x32xbf16> into tensor<8x32x32xbf16> - // CHECK: tensor.extract_slice {{.*}} [1, 8, 16, 32, 2] [1, 1, 1, 1, 1] - // CHECK: tensor.collapse_shape {{.*}} tensor<1x8x16x32x2xbf16> into tensor<8x16x32x2xbf16> - // CHECK: tensor.extract_slice {{.*}} [1, 1, 32, 32] [1, 1, 1, 1] - // CHECK: tensor.collapse_shape {{.*}} tensor<1x1x32x32xf32> into tensor<32x32xf32> - // CHECK: tensor.extract_slice {{.*}} [1, 1, 32, 32] [1, 1, 1, 1] - // CHECK: tensor.collapse_shape {{.*}} tensor<1x1x32x32xbf16> into tensor<32x32xbf16> - // CHECK: scf.if - // CHECK: linalg.fill - // CHECK: linalg.generic {indexing_maps = [#[[mapA]], #[[mapB]], #[[mapC]]], iterator_types = ["reduction", "parallel", "parallel", "reduction", "reduction"]} - // CHECK: else - // CHECK: linalg.generic {indexing_maps = [#[[mapA]], #[[mapB]], #[[mapC]]], iterator_types = ["reduction", "parallel", "parallel", "reduction", "reduction"]} - // CHECK: scf.if - // CHECK: linalg.copy - // CHECK: else - %2 = linalg.generic { - MThreads = 16 : i32, NThreads = 2 : i32, KThreads = 1 : i32, MBlock = 256 : i32, NBlock = 256 : i32, KBlock = 256 : i32,innermostMBlock = 32 : i32, innermostNBlock = 32 : i32, innermostKBlock = 32 : i32, - indexing_maps = [affine_map<(d0, d1, d2, d3, d4, d5, d6) -> (d0, d4, d1, d5 * 2 + d6)>, - affine_map<(d0, d1, d2, d3, d4, d5, d6) -> (d2, d4, d5, d3, d6)>, - affine_map<(d0, d1, d2, d3, d4, d5, d6) -> (d0, d2, d1, d3)>], - iterator_types = ["parallel", "parallel", "parallel", "parallel", "reduction", "reduction", "reduction"] - } - ins(%arg0, %arg1 : tensor<128x128x32x32xbf16>, tensor<128x128x16x32x2xbf16>) - outs(%1 : tensor<128x128x32x32xbf16>) { - ^bb0(%in: bf16, %in_0: bf16, %out: bf16): - %3 = arith.mulf %in, %in_0 : bf16 - %4 = arith.addf %out, %3 : bf16 - linalg.yield %4 : bf16 - } -> tensor<128x128x32x32xbf16> return %2 : tensor<128x128x32x32xbf16> -} - -// ----- - -// CHECK: #[[mapA:.+]] = affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d3 * 2 + d4)> -// CHECK: #[[mapB:.+]] = affine_map<(d0, d1, d2, d3, d4) -> (d0, d3, d2, d4)> -// CHECK: #[[mapC:.+]] = affine_map<(d0, d1, d2, d3, d4) -> (d1, d2)> -/// CHECK-LABEL: @matmul_2Dx4D_bf16_generic -func.func @matmul_2Dx4D_bf16_generic(%arg0: tensor<4096x4096xbf16>, %arg1: tensor<128x128x16x32x2xbf16>) -> tensor<4096x4096xbf16> { - %cst_0 = arith.constant 0.000000e+00 : bf16 - %0 = tensor.empty() : tensor<4096x4096xbf16> - %1 = linalg.fill ins(%cst_0 : bf16) outs(%0 : tensor<4096x4096xbf16>) -> tensor<4096x4096xbf16> - // CHECK: scf.forall {{.*}} (2) {{.*}} (tensor<2x1x1x4096x4096xf32>) - // CHECK: tensor.extract_slice {{.*}} [1, 1, 1, 4096, 4096] [1, 1, 1, 1, 1] - // CHECK: scf.forall {{.*}} (0) to (4096) step (256) {{.*}} (tensor<4096x4096xf32>) - // CHECK: tensor.extract_slice {{.*}} [256, 4096] [1, 1] - // CHECK: scf.forall {{.*}} (0) to (128) step (64) {{.*}} (tensor<256x4096xf32>) - // CHECK: tensor.extract_slice {{.*}} [256, 2048] [1, 1] - // CHECK: scf.for - // CHECK: tensor.extract_slice {{.*}} [256, 256] [1, 1] - // CHECK: scf.for - // CHECK: scf.for - // CHECK: tensor.extract_slice {{.*}} [32, 256] [1, 1] - // CHECK: tensor.extract_slice {{.*}} [32, 256] [1, 1] - // CHECK: scf.for - // CHECK: tensor.extract_slice {{.*}} [1, 8, 16, 32, 2] [1, 1, 1, 1, 1] - // CHECK: tensor.collapse_shape {{.*}} tensor<1x8x16x32x2xbf16> into tensor<8x16x32x2xbf16> - // CHECK: tensor.extract_slice {{.*}} [32, 32] [1, 1] - // CHECK: linalg.transpose {{.*}} permutation = [1, 0, 2] - // CHECK: scf.if - // CHECK: linalg.fill - // CHECK: linalg.generic {indexing_maps = [#[[mapA]], #[[mapB]], #[[mapC]]], iterator_types = ["reduction", "parallel", "parallel", "reduction", "reduction"]} - // CHECK: else - // CHECK: linalg.generic {indexing_maps = [#[[mapA]], #[[mapB]], #[[mapC]]], iterator_types = ["reduction", "parallel", "parallel", "reduction", "reduction"]} - // CHECK: scf.forall.in_parallel - // CHECK: scf.forall.in_parallel - // CHECK: scf.forall.in_parallel - // CHECK: linalg.reduce {{.*}} dimensions = [0, 1, 2] - // CHECK: linalg.copy - %2 = linalg.generic { - MThreads = 32 : i32, NThreads = 2 : i32, KThreads = 2 : i32, MBlock = 256 : i32, NBlock = 256 : i32, KBlock = 256 : i32,innermostMBlock = 32 : i32, innermostNBlock = 32 : i32, innermostKBlock = 32 : i32, - indexing_maps = [affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d3 * 32 + d4 * 2 + d5)>, - affine_map<(d0, d1, d2, d3, d4, d5) -> (d1, d3, d4, d2, d5)>, - affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d1 * 32 + d2)>], - iterator_types = ["parallel", "parallel", "parallel", "reduction", "reduction", "reduction"] - } - ins(%arg0, %arg1 : tensor<4096x4096xbf16>, tensor<128x128x16x32x2xbf16>) - outs(%1 : tensor<4096x4096xbf16>) { - ^bb0(%in: bf16, %in_0: bf16, %out: bf16): - %3 = arith.mulf %in, %in_0 : bf16 - %4 = arith.addf %out, %3 : bf16 - linalg.yield %4 : bf16 - } -> tensor<4096x4096xbf16> - return %2 : tensor<4096x4096xbf16> -} - diff --git a/test/mlir/test/gc/Transforms/iterative-tiling-and-fusion.mlir b/test/mlir/test/gc/Transforms/iterative-tiling-and-fusion.mlir deleted file mode 100644 index fdd3ce1bc..000000000 --- a/test/mlir/test/gc/Transforms/iterative-tiling-and-fusion.mlir +++ /dev/null @@ -1,629 +0,0 @@ -// RUN: gc-opt --split-input-file -iterative-tiling-and-fusion %s --cse - -module { - /// CHECK-LABEL: @fuse_mlp - func.func @fuse_mlp(%arg0: tensor<128x512xbf16>, %arg1: tensor<32x8x16x32xbf16>, %arg2: tensor<256xbf16>) -> tensor<128x256xbf16> { - %c32 = arith.constant 32 : index - %c512 = arith.constant 512 : index - %c128 = arith.constant 128 : index - %c64 = arith.constant 64 : index - %c0 = arith.constant 0 : index - %cst = arith.constant 0.000000e+00 : bf16 - /// CHECK: tensor.empty - %0 = tensor.empty() : tensor<128x256xbf16> - %1 = linalg.fill ins(%cst : bf16) outs(%0 : tensor<128x256xbf16>) -> tensor<128x256xbf16> - /// CHECK: tensor.empty - %dest = tensor.empty() : tensor<512x256xbf16> - %unpack = tensor.unpack %arg1 inner_dims_pos = [0, 1] inner_tiles = [16, 32] into %dest : tensor<32x8x16x32xbf16> -> tensor<512x256xbf16> - /// CHECK: %[[FINAL_RESULT:.*]]:3 = scf.forall (%{{.*}}) in (2, 2) - %2 = scf.forall (%arg3, %arg4) in (2, 2) shared_outs(%arg5 = %1) -> (tensor<128x256xbf16>) { - %5 = affine.apply affine_map<(d0) -> (d0 * 64)>(%arg3) - %6 = affine.apply affine_map<(d0) -> (d0 * 128)>(%arg4) - %7 = affine.apply affine_map<(d0) -> (d0 * 64)>(%arg3) - %8 = affine.apply affine_map<(d0) -> (d0 * 128)>(%arg4) - %extracted_slice = tensor.extract_slice %arg0[%5, 0] [64, 512] [1, 1] : tensor<128x512xbf16> to tensor<64x512xbf16> - %extracted_slice_0 = tensor.extract_slice %unpack[0, %6] [512, 128] [1, 1] : tensor<512x256xbf16> to tensor<512x128xbf16> - %extracted_slice_1 = tensor.extract_slice %arg5[%7, %8] [64, 128] [1, 1] : tensor<128x256xbf16> to tensor<64x128xbf16> - /// CHECK: scf.for - /// CHECK: scf.for - /// CHECK: scf.for - %9 = scf.for %arg6 = %c0 to %c64 step %c64 iter_args(%arg7 = %extracted_slice_1) -> (tensor<64x128xbf16>) { - %12 = scf.for %arg8 = %c0 to %c128 step %c128 iter_args(%arg9 = %arg7) -> (tensor<64x128xbf16>) { - %13 = scf.for %arg10 = %c0 to %c512 step %c512 iter_args(%arg11 = %arg9) -> (tensor<64x128xbf16>) { - %extracted_slice_2 = tensor.extract_slice %extracted_slice[%arg6, %arg10] [64, 512] [1, 1] : tensor<64x512xbf16> to tensor<64x512xbf16> - %extracted_slice_3 = tensor.extract_slice %extracted_slice_0[%arg10, %arg8] [512, 128] [1, 1] : tensor<512x128xbf16> to tensor<512x128xbf16> - %extracted_slice_4 = tensor.extract_slice %arg11[%arg6, %arg8] [64, 128] [1, 1] : tensor<64x128xbf16> to tensor<64x128xbf16> - /// CHECK: scf.for - /// CHECK: scf.for - /// CHECK: scf.for - %14 = scf.for %arg12 = %c0 to %c64 step %c32 iter_args(%arg13 = %extracted_slice_4) -> (tensor<64x128xbf16>) { - %15 = scf.for %arg14 = %c0 to %c128 step %c32 iter_args(%arg15 = %arg13) -> (tensor<64x128xbf16>) { - %16 = scf.for %arg16 = %c0 to %c512 step %c512 iter_args(%arg17 = %arg15) -> (tensor<64x128xbf16>) { - %extracted_slice_5 = tensor.extract_slice %extracted_slice_2[%arg12, %arg16] [32, 512] [1, 1] : tensor<64x512xbf16> to tensor<32x512xbf16> - %extracted_slice_6 = tensor.extract_slice %extracted_slice_3[%arg16, %arg14] [512, 32] [1, 1] : tensor<512x128xbf16> to tensor<512x32xbf16> - %extracted_slice_7 = tensor.extract_slice %arg17[%arg12, %arg14] [32, 32] [1, 1] : tensor<64x128xbf16> to tensor<32x32xbf16> - /// CHECK: %[[UNPACK_OUT:.*]] = tensor.unpack - /// CHECK: %[[FILL_OUT:.*]] = linalg.fill - /// CHECK: %[[EXPAND_OUT_1:.*]] = tensor.expand_shape - /// CHECK: %[[EXPAND_OUT_2:.*]] = tensor.expand_shape - %expanded = tensor.expand_shape %extracted_slice_5 [[0, 1], [2]] output_shape [1, 32, 512] : tensor<32x512xbf16> into tensor<1x32x512xbf16> - %expanded_8 = tensor.expand_shape %extracted_slice_6 [[0, 1], [2]] output_shape [1, 32, 512] : tensor<512x32xbf16> into tensor<1x512x32xbf16> - /// CHECK: %[[MATMUL_OUT:.*]] = linalg.batch_reduce_matmul ins(%[[EXPAND_OUT_1]], %[[EXPAND_OUT_2]] : - %17 = linalg.batch_reduce_matmul ins(%expanded, %expanded_8 : tensor<1x32x512xbf16>, tensor<1x512x32xbf16>) outs(%extracted_slice_7 : tensor<32x32xbf16>) -> tensor<32x32xbf16> - /// CHECK: %[[BROADCAST_OUT:.*]] = linalg.broadcast - /// CHECK: %[[ADD_OUT:.*]] = linalg.add ins(%[[MATMUL_OUT]], %[[BROADCAST_OUT]] : - /// CHECK: %[[EXP_OUT:.*]] = linalg.exp ins(%[[ADD_OUT]] : - %inserted_slice_9 = tensor.insert_slice %17 into %arg17[%arg12, %arg14] [32, 32] [1, 1] : tensor<32x32xbf16> into tensor<64x128xbf16> - /// CHECK: scf.yield {{.*}}, {{.*}}, {{.*}} : tensor<64x128xbf16>, tensor<64x128xbf16>, tensor<64x128xbf16> - scf.yield %inserted_slice_9 : tensor<64x128xbf16> - } - /// CHECK: scf.yield {{.*}}, {{.*}}, {{.*}} : tensor<64x128xbf16>, tensor<64x128xbf16>, tensor<64x128xbf16> - scf.yield %16 : tensor<64x128xbf16> - } - /// CHECK: scf.yield {{.*}}, {{.*}}, {{.*}} : tensor<64x128xbf16>, tensor<64x128xbf16>, tensor<64x128xbf16> - scf.yield %15 : tensor<64x128xbf16> - } - /// CHECK: tensor.insert_slice - /// CHECK: tensor.insert_slice - /// CHECK: tensor.insert_slice - %inserted_slice = tensor.insert_slice %14 into %arg11[%arg6, %arg8] [64, 128] [1, 1] : tensor<64x128xbf16> into tensor<64x128xbf16> - /// CHECK: scf.yield {{.*}}, {{.*}}, {{.*}} : tensor<64x128xbf16>, tensor<64x128xbf16>, tensor<64x128xbf16> - scf.yield %inserted_slice : tensor<64x128xbf16> - } - /// CHECK: scf.yield {{.*}}, {{.*}}, {{.*}} : tensor<64x128xbf16>, tensor<64x128xbf16>, tensor<64x128xbf16> - scf.yield %13 : tensor<64x128xbf16> - } - /// CHECK: scf.yield {{.*}}, {{.*}}, {{.*}} : tensor<64x128xbf16>, tensor<64x128xbf16>, tensor<64x128xbf16> - scf.yield %12 : tensor<64x128xbf16> - } - %10 = affine.apply affine_map<(d0) -> (d0 * 64)>(%arg3) - %11 = affine.apply affine_map<(d0) -> (d0 * 128)>(%arg4) - scf.forall.in_parallel { - /// CHECK: tensor.parallel_insert_slice - /// CHECK: tensor.parallel_insert_slice - /// CHECK: tensor.parallel_insert_slice - tensor.parallel_insert_slice %9 into %arg5[%10, %11] [64, 128] [1, 1] : tensor<64x128xbf16> into tensor<128x256xbf16> - } - } - %broadcasted = linalg.broadcast ins(%arg2 : tensor<256xbf16>) outs(%0 : tensor<128x256xbf16>) dimensions = [0] - %3 = linalg.add ins(%2, %broadcasted : tensor<128x256xbf16>, tensor<128x256xbf16>) outs(%0 : tensor<128x256xbf16>) -> tensor<128x256xbf16> - %4 = linalg.exp ins(%3 : tensor<128x256xbf16>) outs(%0 : tensor<128x256xbf16>) -> tensor<128x256xbf16> - /// CHECK: return %[[FINAL_RESULT]]#2 - return %4 : tensor<128x256xbf16> - } -} - -// ----- - -#map = affine_map<(d0) -> (d0 * 32)> -#map1 = affine_map<(d0) -> (d0 * 16)> -module { - /// CHECK-LABEL: @fuse_mlp_vnni - func.func @fuse_mlp_vnni(%arg0: tensor<128x1024xbf16>, %arg1: tensor<1024x512xbf16>, %arg2: tensor<512xbf16>) -> tensor<128x512xbf16> attributes {llvm.emit_c_interface} { - %c2 = arith.constant 2 : index - %c64 = arith.constant 64 : index - %c0 = arith.constant 0 : index - %cst = arith.constant dense<0.000000e+00> : tensor<128x512xbf16> - /// CHECK: tensor.empty - %0 = tensor.empty() : tensor<128x512xbf16> - /// CHECK: tensor.empty - %1 = tensor.empty() : tensor<16x64x16x32xbf16> - %pack = tensor.pack %arg1 outer_dims_perm = [1, 0] inner_dims_pos = [0, 1] inner_tiles = [16, 32] into %1 : tensor<1024x512xbf16> -> tensor<16x64x16x32xbf16> - /// CHECK: tensor.empty - %2 = tensor.empty() : tensor<16x64x8x32x2xbf16> - %pack_0 = tensor.pack %pack inner_dims_pos = [2] inner_tiles = [2] into %2 : tensor<16x64x16x32xbf16> -> tensor<16x64x8x32x2xbf16> - /// CHECK: %[[FINAL_RESULT:.*]]:3 = scf.forall (%{{.*}}) in (16) - %3 = scf.forall (%arg3) in (16) shared_outs(%arg4 = %0) -> (tensor<128x512xbf16>) { - %9 = affine.apply #map(%arg3) - %extracted_slice = tensor.extract_slice %arg4[0, %9] [128, 32] [1, 1] : tensor<128x512xbf16> to tensor<128x32xbf16> - /// CHECK: tensor.empty - %10 = tensor.empty() : tensor<128x32xf32> - /// CHECK: linalg.copy - %11 = linalg.copy ins(%extracted_slice : tensor<128x32xbf16>) outs(%10 : tensor<128x32xf32>) -> tensor<128x32xf32> - /// CHECK: %[[TMP_RESULT:.*]]:2 = scf.for - %12:2 = scf.for %arg5 = %c0 to %c64 step %c2 iter_args(%arg6 = %11, %arg7 = %extracted_slice) -> (tensor<128x32xf32>, tensor<128x32xbf16>) { - %14 = affine.apply #map1(%arg5) - %extracted_slice_1 = tensor.extract_slice %arg0[0, %14] [128, 32] [1, 1] : tensor<128x1024xbf16> to tensor<128x32xbf16> - /// CHECK: %[[PACK_OUT:.*]] = tensor.pack - /// CHECK: %[[PACK_OUT_VNNI:.*]] = tensor.pack %[[PACK_OUT]] - %extracted_slice_2 = tensor.extract_slice %pack_0[%arg3, %arg5, 0, 0, 0] [1, 2, 8, 32, 2] [1, 1, 1, 1, 1] : tensor<16x64x8x32x2xbf16> to tensor<1x2x8x32x2xbf16> - /// CHECK: %[[COLLAPSE_OUT:.*]] = tensor.collapse_shape %[[PACK_OUT_VNNI]] - %collapsed = tensor.collapse_shape %extracted_slice_2 [[0, 1], [2], [3], [4]] : tensor<1x2x8x32x2xbf16> into tensor<2x8x32x2xbf16> - /// CHECK: %[[EXPAND_OUT:.*]] = tensor.expand_shape - %expanded = tensor.expand_shape %extracted_slice_1 [[0], [1, 2]] output_shape [128, 2, 16] : tensor<128x32xbf16> into tensor<128x2x16xbf16> - %15 = tensor.empty() : tensor<2x128x16xbf16> - /// CHECK: %[[TRANSPOSE_OUT:.*]] = linalg.transpose ins(%[[EXPAND_OUT]] : - %transposed = linalg.transpose ins(%expanded : tensor<128x2x16xbf16>) outs(%15 : tensor<2x128x16xbf16>) permutation = [1, 0, 2] - /// CHECK: %[[MATMUL_OUT:.*]] = linalg.generic - /// CHECK-SAME: ins(%[[TRANSPOSE_OUT]], %[[COLLAPSE_OUT]] : - %16 = linalg.generic { - indexing_maps = [affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d3 * 2 + d4)>, - affine_map<(d0, d1, d2, d3, d4) -> (d0, d3, d2, d4)>, - affine_map<(d0, d1, d2, d3, d4) -> (d1, d2)>], - iterator_types = ["reduction", "parallel", "parallel", "reduction", "reduction"]} - ins(%transposed, %collapsed : tensor<2x128x16xbf16>, tensor<2x8x32x2xbf16>) - outs(%arg6 : tensor<128x32xf32>) { - ^bb0(%in: bf16, %in_0: bf16, %out: f32): - %b1 = arith.extf %in : bf16 to f32 - %b2 = arith.extf %in_0 : bf16 to f32 - %b3 = arith.mulf %b1, %b2 : f32 - %b4 = arith.addf %out, %b3 : f32 - linalg.yield %b4 : f32 - } -> tensor<128x32xf32> - %17 = arith.addi %arg5, %c2 : index - %18 = arith.cmpi sge, %17, %c64 : index - /// CHECK: %[[IF_RESULT:.*]] = scf.if - %19 = scf.if %18 -> (tensor<128x32xbf16>) { - %20 = linalg.copy ins(%16 : tensor<128x32xf32>) outs(%arg7 : tensor<128x32xbf16>) -> tensor<128x32xbf16> - scf.yield %20 : tensor<128x32xbf16> - } else { - scf.yield %arg7 : tensor<128x32xbf16> - } - /// CHECK: scf.yield %[[MATMUL_OUT]], %[[IF_RESULT]] : - scf.yield %16, %19 : tensor<128x32xf32>, tensor<128x32xbf16> - } - /// CHECK: %[[BROADCAST_OUT:.*]] = linalg.broadcast - /// CHECK: %[[ADD_OUT:.*]] = linalg.add ins(%[[TMP_RESULT]]#1, %[[BROADCAST_OUT]] : - /// CHECK: %[[MAX_OUT:.*]] = linalg.max ins(%[[ADD_OUT]], - %13 = affine.apply #map(%arg3) - scf.forall.in_parallel { - /// CHECK: tensor.parallel_insert_slice %[[MAX_OUT]] - /// CHECK: tensor.parallel_insert_slice - /// CHECK: tensor.parallel_insert_slice - tensor.parallel_insert_slice %12#1 into %arg4[0, %13] [128, 32] [1, 1] : tensor<128x32xbf16> into tensor<128x512xbf16> - } - } - %4 = tensor.empty() : tensor<128x512xbf16> - %broadcasted = linalg.broadcast ins(%arg2 : tensor<512xbf16>) outs(%4 : tensor<128x512xbf16>) dimensions = [0] - %5 = tensor.empty() : tensor<128x512xbf16> - %6 = linalg.add ins(%3, %broadcasted : tensor<128x512xbf16>, tensor<128x512xbf16>) outs(%5 : tensor<128x512xbf16>) -> tensor<128x512xbf16> - %7 = tensor.empty() : tensor<128x512xbf16> - %8 = linalg.max ins(%6, %cst : tensor<128x512xbf16>, tensor<128x512xbf16>) outs(%7 : tensor<128x512xbf16>) -> tensor<128x512xbf16> - /// CHECK: return %[[FINAL_RESULT]]#2 - return %8 : tensor<128x512xbf16> - } -} - -// ----- - -#map = affine_map<(d0) -> (d0 * 128)> -module { - /// CHECK-LABEL: @fuse_multiple_consumer - func.func @fuse_multiple_consumer(%arg0: tensor<256x512xf32>, %arg1: tensor<512x256xf32>, %arg2: tensor<256x256xf32>, %arg3: tensor<256x256xf32>) -> (tensor<256x256xf32>, tensor<256x256xf32>) { - %c0 = arith.constant 0 : index - %c64 = arith.constant 64 : index - %c128 = arith.constant 128 : index - %cst = arith.constant 0.000000e+00 : f32 - %dest0 = tensor.empty() : tensor<256x256xf32> - %dest1 = linalg.fill ins(%cst : f32) outs(%dest0 : tensor<256x256xf32>) -> tensor<256x256xf32> - /// CHECK: %[[FINAL_RESULT:.*]]:3 = scf.forall (%{{.*}}) in (2, 2) - %1 = scf.forall (%arg4, %arg5) in (2, 2) shared_outs(%arg6 = %dest1) -> tensor<256x256xf32> { - %iv0 = affine.apply #map(%arg4) - %iv1 = affine.apply #map(%arg5) - %extracted_slice_1 = tensor.extract_slice %arg6[%iv0, %iv1] [128, 128] [1, 1] : tensor<256x256xf32> to tensor<128x128xf32> - %extracted_slice_2 = tensor.extract_slice %arg0[%iv0, 0] [128, 512] [1, 1] : tensor<256x512xf32> to tensor<128x512xf32> - %extracted_slice_3 = tensor.extract_slice %arg1[0, %iv1] [512, 128] [1, 1] : tensor<512x256xf32> to tensor<512x128xf32> - /// CHECK: scf.for - /// CHECK: scf.for - %2 = scf.for %arg7 = %c0 to %c128 step %c64 iter_args(%arg8 = %extracted_slice_1) -> (tensor<128x128xf32>) { - %3 = scf.for %arg9 = %c0 to %c128 step %c64 iter_args(%arg10 = %arg8) -> (tensor<128x128xf32>) { - %extracted_slice_4 = tensor.extract_slice %arg10[%arg7, %arg9] [64, 64] [1, 1] : tensor<128x128xf32> to tensor<64x64xf32> - %extracted_slice_5 = tensor.extract_slice %extracted_slice_2[%arg7, 0] [64, 512] [1, 1] : tensor<128x512xf32> to tensor<64x512xf32> - %extracted_slice_6 = tensor.extract_slice %extracted_slice_3[0, %arg9] [512, 64] [1, 1] : tensor<512x128xf32> to tensor<512x64xf32> - /// CHECK: %[[MATMUL_OUT:.*]] = linalg.matmul - %4 = linalg.matmul ins(%extracted_slice_5, %extracted_slice_6 : tensor<64x512xf32>, tensor<512x64xf32>) outs(%extracted_slice_4 : tensor<64x64xf32>) -> tensor<64x64xf32> - /// CHECK: %[[MUL_OUT:.*]] = linalg.mul ins(%[[MATMUL_OUT]], - /// CHECK: %[[ADD_OUT:.*]] = linalg.add ins(%[[MATMUL_OUT]], - %insert_slice = tensor.insert_slice %4 into %arg10[%arg7, %arg9] [64, 64] [1, 1] : tensor<64x64xf32> into tensor<128x128xf32> - /// CHECK: scf.yield {{.*}}, {{.*}}, {{.*}} : tensor<128x128xf32>, tensor<128x128xf32>, tensor<128x128xf32> - scf.yield %insert_slice : tensor<128x128xf32> - } - /// CHECK: scf.yield {{.*}}, {{.*}}, {{.*}} : tensor<128x128xf32>, tensor<128x128xf32>, tensor<128x128xf32> - scf.yield %3 : tensor<128x128xf32> - } - scf.forall.in_parallel { - /// CHECK: tensor.parallel_insert_slice - /// CHECK: tensor.parallel_insert_slice - /// CHECK: tensor.parallel_insert_slice - tensor.parallel_insert_slice %2 into %arg6[%iv0, %iv1] [128, 128] [1, 1] : tensor<128x128xf32> into tensor<256x256xf32> - } - } - %5 = linalg.add ins(%1, %arg2 : tensor<256x256xf32>, tensor<256x256xf32>) outs(%dest0 : tensor<256x256xf32>) -> tensor<256x256xf32> - %6 = linalg.mul ins(%1, %arg3 : tensor<256x256xf32>, tensor<256x256xf32>) outs(%dest0 : tensor<256x256xf32>) -> tensor<256x256xf32> - /// CHECK: return %[[FINAL_RESULT]]#2, %[[FINAL_RESULT]]#1 - return %5, %6 : tensor<256x256xf32>, tensor<256x256xf32> - } -} - -// ----- - -#map = affine_map<(d0) -> (d0 * 128)> -module { - /// CHECK-LABEL: @fuse_reduce - func.func @fuse_reduce(%arg0: tensor<256x512xf32>, %arg1: tensor<512x256xf32>, %arg2: tensor<256x256xf32>) -> tensor<256xf32> { - %c0 = arith.constant 0 : index - %c64 = arith.constant 64 : index - %c128 = arith.constant 128 : index - %c256 = arith.constant 256 : index - %cst = arith.constant 0.000000e+00 : f32 - %dest0 = tensor.empty() : tensor<256x256xf32> - %dest1 = linalg.fill ins(%cst : f32) outs(%dest0 : tensor<256x256xf32>) -> tensor<256x256xf32> - /// CHECK: %[[FINAL_RESULT:.*]]:3 = scf.forall (%{{.*}}) in (2, 1) - %1 = scf.forall (%arg3, %arg4) in (2, 1) shared_outs(%arg5 = %dest1) -> tensor<256x256xf32> { - %iv0 = affine.apply #map(%arg3) - %iv1 = affine.apply #map(%arg4) - %extracted_slice_1 = tensor.extract_slice %arg5[%iv0, %iv1] [128, 256] [1, 1] : tensor<256x256xf32> to tensor<128x256xf32> - %extracted_slice_2 = tensor.extract_slice %arg0[%iv0, 0] [128, 512] [1, 1] : tensor<256x512xf32> to tensor<128x512xf32> - %extracted_slice_3 = tensor.extract_slice %arg1[0, %iv1] [512, 256] [1, 1] : tensor<512x256xf32> to tensor<512x256xf32> - /// CHECK: %[[FOR_RESULT:.*]]:2 = scf.for - /// CHECK: scf.for - %2 = scf.for %arg6 = %c0 to %c128 step %c64 iter_args(%arg7 = %extracted_slice_1) -> (tensor<128x256xf32>) { - %3 = scf.for %arg8 = %c0 to %c256 step %c64 iter_args(%arg9 = %arg7) -> (tensor<128x256xf32>) { - %extracted_slice_4 = tensor.extract_slice %arg9[%arg6, %arg8] [64, 64] [1, 1] : tensor<128x256xf32> to tensor<64x64xf32> - %extracted_slice_5 = tensor.extract_slice %extracted_slice_2[%arg6, 0] [64, 512] [1, 1] : tensor<128x512xf32> to tensor<64x512xf32> - %extracted_slice_6 = tensor.extract_slice %extracted_slice_3[0, %arg8] [512, 64] [1, 1] : tensor<512x256xf32> to tensor<512x64xf32> - /// CHECK: %[[MATMUL_OUT:.*]] = linalg.matmul - %4 = linalg.matmul ins(%extracted_slice_5, %extracted_slice_6 : tensor<64x512xf32>, tensor<512x64xf32>) outs(%extracted_slice_4 : tensor<64x64xf32>) -> tensor<64x64xf32> - /// CHECK: %[[ADD_OUT:.*]] = linalg.add ins(%[[MATMUL_OUT]], - %insert_slice = tensor.insert_slice %4 into %arg9[%arg6, %arg8] [64, 64] [1, 1] : tensor<64x64xf32> into tensor<128x256xf32> - /// CHECK: scf.yield {{.*}}, {{.*}} : tensor<128x256xf32>, tensor<128x256xf32> - scf.yield %insert_slice : tensor<128x256xf32> - } - /// CHECK: scf.yield {{.*}}, {{.*}} : tensor<128x256xf32>, tensor<128x256xf32> - scf.yield %3 : tensor<128x256xf32> - } - /// CHECK: %[[REDUCE_OUT:.*]] = linalg.reduce { arith.addf } ins(%[[FOR_RESULT]]#1 : - scf.forall.in_parallel { - /// CHECK: tensor.parallel_insert_slice - /// CHECK: tensor.parallel_insert_slice - /// CHECK: tensor.parallel_insert_slice - tensor.parallel_insert_slice %2 into %arg5[%iv0, %iv1] [128, 256] [1, 1] : tensor<128x256xf32> into tensor<256x256xf32> - } - } - %5 = linalg.add ins(%1, %arg2 : tensor<256x256xf32>, tensor<256x256xf32>) outs(%dest0 : tensor<256x256xf32>) -> tensor<256x256xf32> - %dest2 = tensor.empty() : tensor<256xf32> - %6 = linalg.reduce { arith.addf } ins(%5 : tensor<256x256xf32>) outs(%dest2 : tensor<256xf32>) dimensions = [1] - /// CHECK: return %[[FINAL_RESULT]]#2 - return %6 : tensor<256xf32> - } -} - -// ----- - -module { - /// CHECK-LABEL: @fuse_with_default_tiling - func.func @fuse_with_default_tiling(%arg0: tensor<128x256x256xf32>, %arg1: tensor<128x256x256xf32>) -> tensor<128x256xf32> { - %dest0 = tensor.empty() : tensor<128x256x256xf32> - %0 = linalg.add ins(%arg0, %arg1 : tensor<128x256x256xf32>, tensor<128x256x256xf32>) outs(%dest0 : tensor<128x256x256xf32>) -> tensor<128x256x256xf32> - %dest1 = tensor.empty() : tensor<128x256xf32> - /// CHECK: %[[FINAL_RESULT:.*]] = scf.forall (%{{.*}}) = (0, 0) to (128, 256) step (1, 32) - /// CHECK: tensor.extract_slice {{.*}} [1, 256, 32] [1, 1, 1] - /// CHECK: tensor.extract_slice {{.*}} [1, 256, 32] [1, 1, 1] - /// CHECK: tensor.extract_slice {{.*}} [1, 256, 32] [1, 1, 1] - /// CHECK: %[[ADD_OUT:.*]] = linalg.add - /// CHECK: tensor.extract_slice {{.*}} [1, 32] [1, 1] - /// CHECK: %[[REDUCE_OUT:.*]] = linalg.reduce { arith.addf } ins(%[[ADD_OUT]] : - %1 = linalg.reduce { arith.addf } ins(%0 : tensor<128x256x256xf32>) outs(%dest1 : tensor<128x256xf32>) dimensions = [1] - /// CHECK: scf.forall.in_parallel - /// CHECK: tensor.parallel_insert_slice - /// CHECK: return %[[FINAL_RESULT]] - return %1 : tensor<128x256xf32> - } -} - -// ----- - -module { - /// CHECK-LABEL: @fuse_llama2_matmul_add - func.func @fuse_llama2_matmul_add(%arg0: tensor<32x4096xbf16>, %arg1: tensor<4096x4096xbf16>, %arg2: tensor<32x4096xbf16>) -> tensor<32x4096xbf16> { - %c1 = arith.constant 1 : index - %c32 = arith.constant 32 : index - %c16 = arith.constant 16 : index - %c128 = arith.constant 128 : index - %c0 = arith.constant 0 : index - %cst = arith.constant 0.000000e+00 : bf16 - %0 = tensor.empty() : tensor<1x128x32x32xbf16> - %pack = tensor.pack %arg0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [32, 32] into %0 : tensor<32x4096xbf16> -> tensor<1x128x32x32xbf16> - %1 = tensor.empty() : tensor<128x128x32x32xbf16> - %pack_0 = tensor.pack %arg1 outer_dims_perm = [1, 0] inner_dims_pos = [0, 1] inner_tiles = [32, 32] into %1 : tensor<4096x4096xbf16> -> tensor<128x128x32x32xbf16> - %2 = tensor.empty() : tensor<1x128x32x32xbf16> - %3 = tensor.empty() : tensor<128x128x16x32x2xbf16> - %pack_1 = tensor.pack %pack_0 inner_dims_pos = [2] inner_tiles = [2] into %3 : tensor<128x128x32x32xbf16> -> tensor<128x128x16x32x2xbf16> - /// CHECK: %[[FINAL_RESULT:.*]]:3 = scf.for - %4 = scf.for %arg3 = %c0 to %c128 step %c16 iter_args(%arg4 = %2) -> (tensor<1x128x32x32xbf16>) { - %extracted_slice = tensor.extract_slice %arg4[0, %arg3, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : tensor<1x128x32x32xbf16> to tensor<1x16x32x32xbf16> - %9 = tensor.empty() : tensor<1x16x32x32xf32> - /// CHECK: scf.for - /// CHECK: scf.for - %10:2 = scf.for %arg5 = %c0 to %c128 step %c32 iter_args(%arg6 = %9, %arg7 = %extracted_slice) -> (tensor<1x16x32x32xf32>, tensor<1x16x32x32xbf16>) { - %11:2 = scf.for %arg8 = %c0 to %c16 step %c1 iter_args(%arg9 = %arg6, %arg10 = %arg7) -> (tensor<1x16x32x32xf32>, tensor<1x16x32x32xbf16>) { - %extracted_slice_3 = tensor.extract_slice %pack[0, %arg5, 0, 0] [1, 32, 32, 32] [1, 1, 1, 1] : tensor<1x128x32x32xbf16> to tensor<1x32x32x32xbf16> - %12 = affine.apply affine_map<()[s0, s1] -> (s0 + s1)>()[%arg8, %arg3] - /// CHECK: %[[PACK_OUT_1:.*]] = tensor.pack - /// CHECK: %[[PACK_OUT_2:.*]] = tensor.pack - /// CHECK: %[[PACK_OUT_3:.*]] = tensor.pack %[[PACK_OUT_2]] - %extracted_slice_4 = tensor.extract_slice %pack_1[%12, %arg5, 0, 0, 0] [1, 32, 16, 32, 2] [1, 1, 1, 1, 1] : tensor<128x128x16x32x2xbf16> to tensor<1x32x16x32x2xbf16> - %extracted_slice_5 = tensor.extract_slice %arg9[0, %arg8, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : tensor<1x16x32x32xf32> to tensor<1x1x32x32xf32> - %extracted_slice_6 = tensor.extract_slice %arg10[0, %arg8, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : tensor<1x16x32x32xbf16> to tensor<1x1x32x32xbf16> - /// CHECK: %[[COLLAPSE_OUT_1:.*]] = tensor.collapse_shape %[[PACK_OUT_1]] - /// CHECK: %[[COLLAPSE_OUT_2:.*]] = tensor.collapse_shape %[[PACK_OUT_3]] - %collapse_3 = tensor.collapse_shape %extracted_slice_3 [[0, 1], [2], [3]] : tensor<1x32x32x32xbf16> into tensor<32x32x32xbf16> - %collapse_4 = tensor.collapse_shape %extracted_slice_4 [[0, 1], [2], [3], [4]] : tensor<1x32x16x32x2xbf16> into tensor<32x16x32x2xbf16> - %collapse_5 = tensor.collapse_shape %extracted_slice_5 [[0, 1, 2], [3]] : tensor<1x1x32x32xf32> into tensor<32x32xf32> - %collapse_6 = tensor.collapse_shape %extracted_slice_6 [[0, 1, 2], [3]] : tensor<1x1x32x32xbf16> into tensor<32x32xbf16> - %13 = arith.cmpi eq, %arg5, %c0 : index - /// CHECK: %[[IF_RESULT_1:.*]] = scf.if - /// CHECK: linalg.generic - /// CHECK-SAME: ins(%[[COLLAPSE_OUT_1]], %[[COLLAPSE_OUT_2]] : - /// CHECK: } else { - /// CHECK: linalg.generic - /// CHECK-SAME: ins(%[[COLLAPSE_OUT_1]], %[[COLLAPSE_OUT_2]] : - /// CHECK: } - %14 = scf.if %13 -> (tensor<32x32xf32>) { - %18 = linalg.fill ins(%cst : bf16) outs(%collapse_5 : tensor<32x32xf32>) -> tensor<32x32xf32> - %19 = linalg.generic { - indexing_maps = [affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d3 * 2 + d4)>, - affine_map<(d0, d1, d2, d3, d4) -> (d0, d3, d2, d4)>, - affine_map<(d0, d1, d2, d3, d4) -> (d1, d2)>], - iterator_types = ["reduction", "parallel", "parallel", "reduction", "reduction"]} - ins(%collapse_3, %collapse_4 : tensor<32x32x32xbf16>, tensor<32x16x32x2xbf16>) - outs(%18 : tensor<32x32xf32>) { - ^bb0(%in: bf16, %in_0: bf16, %out: f32): - %b1 = arith.extf %in : bf16 to f32 - %b2 = arith.extf %in_0 : bf16 to f32 - %b3 = arith.mulf %b1, %b2 : f32 - %b4 = arith.addf %out, %b3 : f32 - linalg.yield %b4 : f32 - } -> tensor<32x32xf32> - scf.yield %19 : tensor<32x32xf32> - } else { - %18 = linalg.generic { - indexing_maps = [affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d3 * 2 + d4)>, - affine_map<(d0, d1, d2, d3, d4) -> (d0, d3, d2, d4)>, - affine_map<(d0, d1, d2, d3, d4) -> (d1, d2)>], - iterator_types = ["reduction", "parallel", "parallel", "reduction", "reduction"]} - ins(%collapse_3, %collapse_4 : tensor<32x32x32xbf16>, tensor<32x16x32x2xbf16>) - outs(%collapse_5 : tensor<32x32xf32>) { - ^bb0(%in: bf16, %in_0: bf16, %out: f32): - %b1 = arith.extf %in : bf16 to f32 - %b2 = arith.extf %in_0 : bf16 to f32 - %b3 = arith.mulf %b1, %b2 : f32 - %b4 = arith.addf %out, %b3 : f32 - linalg.yield %b4 : f32 - } -> tensor<32x32xf32> - scf.yield %18 : tensor<32x32xf32> - } - %15 = arith.addi %arg5, %c32 : index - %16 = arith.cmpi sge, %15, %c128 : index - /// CHECK: %[[IF_RESULT_2:.*]] = scf.if - %17 = scf.if %16 -> (tensor<32x32xbf16>) { - %18 = linalg.copy ins(%14 : tensor<32x32xf32>) outs(%collapse_6 : tensor<32x32xbf16>) -> tensor<32x32xbf16> - scf.yield %18 : tensor<32x32xbf16> - } else { - scf.yield %collapse_6 : tensor<32x32xbf16> - } - /// CHECK: %[[EXPAND_OUT_1:.*]] = tensor.expand_shape %[[IF_RESULT_1]] - /// CHECK: %[[EXPAND_OUT_2:.*]] = tensor.expand_shape %[[IF_RESULT_2]] - %expand_14 = tensor.expand_shape %14 [[0, 1, 2], [3]] output_shape [1, 1, 32, 32] : tensor<32x32xf32> into tensor<1x1x32x32xf32> - %expand_17 = tensor.expand_shape %17 [[0, 1, 2], [3]] output_shape [1, 1, 32, 32] : tensor<32x32xbf16> into tensor<1x1x32x32xbf16> - /// CHECK: %[[PACK_OUT_4:.*]] = tensor.pack - /// CHECK: %[[ADD_OUT:.*]] = linalg.add ins(%[[EXPAND_OUT_2]], %[[PACK_OUT_4]] : - /// CHECK: %[[UNPACK_OUT:.*]] = tensor.unpack %[[ADD_OUT]] - %inserted_slice_7 = tensor.insert_slice %expand_14 into %arg9[0, %arg8, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : tensor<1x1x32x32xf32> into tensor<1x16x32x32xf32> - %inserted_slice_8 = tensor.insert_slice %expand_17 into %arg10[0, %arg8, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : tensor<1x1x32x32xbf16> into tensor<1x16x32x32xbf16> - /// CHECK: scf.yield {{.*}}, {{.*}}, {{.*}}, {{.*}} : tensor<1x16x32x32xf32>, tensor<1x16x32x32xbf16>, tensor<1x16x32x32xbf16>, tensor<32x512xbf16> - scf.yield %inserted_slice_7, %inserted_slice_8 : tensor<1x16x32x32xf32>, tensor<1x16x32x32xbf16> - } - /// CHECK: scf.yield {{.*}}, {{.*}}, {{.*}}, {{.*}} : tensor<1x16x32x32xf32>, tensor<1x16x32x32xbf16>, tensor<1x16x32x32xbf16>, tensor<32x512xbf16> - scf.yield %11#0, %11#1 : tensor<1x16x32x32xf32>, tensor<1x16x32x32xbf16> - } - /// CHECK: scf.yield {{.*}}, {{.*}}, {{.*}}: tensor<1x128x32x32xbf16>, tensor<1x128x32x32xbf16>, tensor<32x4096xbf16> - %inserted_slice = tensor.insert_slice %10#1 into %arg4[0, %arg3, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : tensor<1x16x32x32xbf16> into tensor<1x128x32x32xbf16> - scf.yield %inserted_slice : tensor<1x128x32x32xbf16> - } - %5 = tensor.empty() : tensor<32x4096xbf16> - %6 = tensor.empty() : tensor<1x128x32x32xbf16> - %pack_2 = tensor.pack %arg2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [32, 32] into %6 : tensor<32x4096xbf16> -> tensor<1x128x32x32xbf16> - %7 = tensor.empty() : tensor<1x128x32x32xbf16> - %8 = linalg.add ins(%4, %pack_2 : tensor<1x128x32x32xbf16>, tensor<1x128x32x32xbf16>) outs(%7 : tensor<1x128x32x32xbf16>) -> tensor<1x128x32x32xbf16> - %unpack = tensor.unpack %8 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [32, 32] into %5 : tensor<1x128x32x32xbf16> -> tensor<32x4096xbf16> - /// CHECK: return %[[FINAL_RESULT]]#2 - return %unpack : tensor<32x4096xbf16> - } -} - -// ----- - -module { - /// CHECK-LABEL: @fuse_residual_pattern - func.func @fuse_residual_pattern(%arg0: tensor<128x256x256xf32>, %arg1: tensor<128x256x256xf32>) -> tensor<128x256x256xf32> { - %dest0 = tensor.empty() : tensor<128x256x256xf32> - /// CHECK: %[[FINAL_RESULT:.*]]:3 = scf.forall (%{{.*}}) = (0, 0, 0) to (128, 256, 256) step (1, 32, 32) - /// CHECK: %[[ADD_OUT:.*]] = linalg.add - /// CHECK: %[[EXP_OUT:.*]] = linalg.exp ins(%[[ADD_OUT:.*]] : - /// CHECK: %[[MUL_OUT:.*]] = linalg.mul ins(%[[ADD_OUT:.*]], %[[EXP_OUT:.*]] : - %0 = linalg.add ins(%arg0, %arg1 : tensor<128x256x256xf32>, tensor<128x256x256xf32>) outs(%dest0 : tensor<128x256x256xf32>) -> tensor<128x256x256xf32> - %1 = linalg.exp ins(%0 : tensor<128x256x256xf32>) outs(%dest0 : tensor<128x256x256xf32>) -> tensor<128x256x256xf32> - %2 = linalg.mul ins(%0, %1 : tensor<128x256x256xf32>, tensor<128x256x256xf32>) outs(%dest0 : tensor<128x256x256xf32>) -> tensor<128x256x256xf32> - /// CHECK: scf.forall.in_parallel - /// CHECK: tensor.parallel_insert_slice - /// CHECK: tensor.parallel_insert_slice - /// CHECK: tensor.parallel_insert_slice - /// CHECK: return %[[FINAL_RESULT]]#2 - return %2 : tensor<128x256x256xf32> - } -} - -// ----- - -module { - /// CHECK-LABEL: @not_fuse_pack - func.func @not_fuse_pack(%arg0: tensor<1x35x4096xbf16>, %arg1: tensor<1x35x4096xbf16>) -> tensor<1x2x128x32x32xbf16> { - %dest0 = tensor.empty() : tensor<1x35x4096xbf16> - /// CHECK: scf.forall - /// CHECK: linalg.add - %add = linalg.add ins(%arg0, %arg1 : tensor<1x35x4096xbf16>, tensor<1x35x4096xbf16>) outs(%dest0 : tensor<1x35x4096xbf16>) -> tensor<1x35x4096xbf16> - /// CHECK: } - %dest1 = tensor.empty() : tensor<1x2x128x32x32xbf16> - %pad = arith.constant 0.000000e+00 : bf16 - /// CHECK: %[[PACK_OUT:.*]] = scf.forall - /// CHECK: tensor.pack - %pack = tensor.pack %add padding_value(%pad : bf16) outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [32, 32] into %dest1 : tensor<1x35x4096xbf16> -> tensor<1x2x128x32x32xbf16> - /// CHECK: } - /// CHECK: return %[[PACK_OUT]] - return %pack : tensor<1x2x128x32x32xbf16> - } -} - -// ----- - -module { - /// CHECK-LABEL: @fuse_pack - func.func @fuse_pack(%arg0: tensor<1x32x4096xbf16>, %arg1: tensor<1x32x4096xbf16>) -> tensor<1x1x128x32x32xbf16> { - %dest0 = tensor.empty() : tensor<1x32x4096xbf16> - /// CHECK: %[[FINAL_RESULT:.*]]:2 = scf.forall (%{{.*}}) = (0, 0) to (1, 4096) step (1, 32) - /// CHECK: linalg.add - %add = linalg.add ins(%arg0, %arg1 : tensor<1x32x4096xbf16>, tensor<1x32x4096xbf16>) outs(%dest0 : tensor<1x32x4096xbf16>) -> tensor<1x32x4096xbf16> - %dest1 = tensor.empty() : tensor<1x1x128x32x32xbf16> - /// CHECK-NEXT: affine.apply - /// CHECK-NEXT: tensor.extract_slice - /// CHECK-NEXT: tensor.pack - %pack = tensor.pack %add outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [32, 32] into %dest1 : tensor<1x32x4096xbf16> -> tensor<1x1x128x32x32xbf16> - /// CHECK: scf.forall.in_parallel - /// CHECK: tensor.parallel_insert_slice - /// CHECK: tensor.parallel_insert_slice - /// CHECK: return %[[FINAL_RESULT]]#1 - return %pack : tensor<1x1x128x32x32xbf16> - } -} - -// ----- - -module { - /// CHECK-LABEL: @fuse_generic_matmul - /// CHECK-SAME: %[[ARG0:[a-zA-Z0-9]+]]: tensor<32x32xf32> - /// CHECK-SAME: %[[ARG1:[a-zA-Z0-9]+]]: tensor<2x16x16xf32> - /// CHECK-SAME: %[[ARG2:[a-zA-Z0-9]+]]: tensor<4x16x16xf32> - func.func @fuse_generic_matmul(%arg0: tensor<32x32xf32>, %arg1: tensor<2x16x16xf32>, %arg2: tensor<4x16x16xf32>) -> tensor<32x64xf32> { - /// CHECK: %[[EMPTY_OUT_0:.*]] = tensor.empty - %0 = tensor.empty() : tensor<2x2x16x16xf32> - %pack = tensor.pack %arg0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [16, 16] into %0 : tensor<32x32xf32> -> tensor<2x2x16x16xf32> - /// CHECK: %[[EMPTY_OUT_1:.*]] = tensor.empty - %1 = tensor.empty() : tensor<2x16x16xf32> - /// CHECK: %[[FIRST_MATMUL_OUT:.*]] = scf.forall (%{{.*}}) in (2) - /// CHECK: %[[EXTRACT_SLICE_0:.*]] = tensor.extract_slice %[[ARG0]]{{.*}} [16, 32] - /// CHECK: %[[EXTRACT_SLICE_1:.*]] = tensor.extract_slice %[[EMPTY_OUT_0]]{{.*}} [1, 2, 16, 16] - /// CHECK: %[[PACK_OUT:.*]] = tensor.pack %[[EXTRACT_SLICE_0]] - /// CHECK: %[[EXTRACT_SLICE_2:.*]] = tensor.extract_slice %[[ARG1]]{{.*}} [2, 16, 16] - /// CHECK: %[[MATMUL_OUT_0:.*]] = linalg.generic {{.*}} ins(%[[PACK_OUT]], %[[EXTRACT_SLICE_2]] : - %2 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d2, d4)>, affine_map<(d0, d1, d2, d3, d4) -> (d1, d4, d3)>, affine_map<(d0, d1, d2, d3, d4) -> (d0, d2, d3)>], iterator_types = ["parallel", "reduction", "parallel", "parallel", "reduction"]} ins(%pack, %arg1 : tensor<2x2x16x16xf32>, tensor<2x16x16xf32>) outs(%1 : tensor<2x16x16xf32>) { - ^bb0(%in: f32, %in_3: f32, %out: f32): - %9 = arith.mulf %in, %in_3 : f32 - %10 = arith.addf %out, %9 : f32 - linalg.yield %10 : f32 - } -> tensor<2x16x16xf32> - /// CHECK: scf.forall.in_parallel - /// CHECK: tensor.parallel_insert_slice - /// CHECK: } - /// CHECK: %[[EMPTY_OUT_2:.*]] = tensor.empty - /// CHECK: %[[EMPTY_OUT_3:.*]] = tensor.empty - %3 = tensor.empty() : tensor<2x4x16x16xf32> - /// CHECK: %[[FINAL_RESULT:.*]]:2 = scf.forall (%{{.*}}) in (2, 4) - /// CHECK: %[[EXTRACT_SLICE_3:.*]] = tensor.extract_slice %[[FIRST_MATMUL_OUT]]{{.*}} [1, 16, 16] - /// CHECK: %[[EXTRACT_SLICE_4:.*]] = tensor.extract_slice %[[ARG2]]{{.*}} [1, 16, 16] - /// CHECK: %[[MATMUL_OUT_1:.*]] = linalg.generic {{.*}} ins(%[[EXTRACT_SLICE_3]], %[[EXTRACT_SLICE_4]] : - /// CHECK: %[[UNPACK_OUT:.*]] = tensor.unpack %[[MATMUL_OUT_1]] - %4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2, d3, d4) -> (d0, d2, d4)>, affine_map<(d0, d1, d2, d3, d4) -> (d1, d4, d3)>, affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d2, d3)>], iterator_types = ["parallel", "parallel", "parallel", "parallel", "reduction"]} ins(%2, %arg2 : tensor<2x16x16xf32>, tensor<4x16x16xf32>) outs(%3 : tensor<2x4x16x16xf32>) { - ^bb0(%in: f32, %in_3: f32, %out: f32): - %9 = arith.mulf %in, %in_3 : f32 - %10 = arith.addf %out, %9 : f32 - linalg.yield %10 : f32 - } -> tensor<2x4x16x16xf32> - /// CHECK: scf.forall.in_parallel - /// CHECK: tensor.parallel_insert_slice - /// CHECK: tensor.parallel_insert_slice - /// CHECK: } - %5 = tensor.empty() : tensor<32x64xf32> - %unpack = tensor.unpack %4 inner_dims_pos = [0, 1] inner_tiles = [16, 16] into %5 : tensor<2x4x16x16xf32> -> tensor<32x64xf32> - /// CHECK: return %[[FINAL_RESULT]]#1 - return %unpack : tensor<32x64xf32> - } -} - -// ----- - -module { - /// CHECK-LABEL: @yield_fused_producer - func.func @yield_fused_producer(%arg0: tensor<16x32x32xf32>) -> (tensor<16x32x32xf32>, tensor<16x32xf32>) { - /// CHECK: arith.constant - %cst_0 = arith.constant dense<2.000000e+00> : tensor<16x32x32xf32> - /// CHECK-NEXT: tensor.empty - %dest0 = tensor.empty() : tensor<16x32x32xf32> - %0 = linalg.powf ins(%arg0, %cst_0 : tensor<16x32x32xf32>, tensor<16x32x32xf32>) outs(%dest0 : tensor<16x32x32xf32>) -> tensor<16x32x32xf32> - /// CHECK-NEXT: tensor.empty - %dest1 = tensor.empty() : tensor<16x32xf32> - /// CHECK-NEXT: %[[FINAL_RESULT:.*]]:2 = scf.forall (%{{.*}}) in (16) - /// CHECK-NEXT: tensor.extract_slice - /// CHECK-NEXT: tensor.extract_slice - /// CHECK-NEXT: tensor.extract_slice - /// CHECK-NEXT: linalg.powf - /// CHECK-NEXT: tensor.extract_slice - /// CHECK-NEXT: linalg.reduce - %1 = linalg.reduce { arith.addf } ins(%0 : tensor<16x32x32xf32>) outs(%dest1 : tensor<16x32xf32>) dimensions = [2] - /// CHECK-NEXT: scf.forall.in_parallel - /// CHECK-NEXT: tensor.parallel_insert_slice - /// CHECK-NEXT: tensor.parallel_insert_slice - /// CHECK-NEXT: } - /// CHECK: return %[[FINAL_RESULT]]#1, %[[FINAL_RESULT]]#0 - return %0, %1 : tensor<16x32x32xf32>, tensor<16x32xf32> - } -} - -// ----- - -#map = affine_map<(d0) -> (d0 * 128)> -module { - /// CHECK-LABEL: @fuse_tiled_producer - func.func @fuse_tiled_producer(%arg0: tensor<256x512xf32>, %arg1: tensor<512x256xf32>) -> tensor<256x256xf32> { - %c0 = arith.constant 0 : index - %c64 = arith.constant 64 : index - %c128 = arith.constant 128 : index - %cst = arith.constant 0.000000e+00 : f32 - %dest0 = tensor.empty() : tensor<256x256xf32> - /// CHECK: %[[FINAL_RESULT:.*]]:2 = scf.forall (%{{.*}}) in (2, 2) - %1 = scf.forall (%arg4, %arg5) in (2, 2) shared_outs(%arg6 = %dest0) -> tensor<256x256xf32> { - %iv0 = affine.apply #map(%arg4) - %iv1 = affine.apply #map(%arg5) - %extracted_slice_1 = tensor.extract_slice %arg6[%iv0, %iv1] [128, 128] [1, 1] : tensor<256x256xf32> to tensor<128x128xf32> - %dest1 = linalg.fill ins(%cst : f32) outs(%extracted_slice_1 : tensor<128x128xf32>) -> tensor<128x128xf32> - %extracted_slice_2 = tensor.extract_slice %arg0[%iv0, 0] [128, 512] [1, 1] : tensor<256x512xf32> to tensor<128x512xf32> - %extracted_slice_3 = tensor.extract_slice %arg1[0, %iv1] [512, 128] [1, 1] : tensor<512x256xf32> to tensor<512x128xf32> - /// CHECK: scf.for - /// CHECK: scf.for - %2 = scf.for %arg7 = %c0 to %c128 step %c64 iter_args(%arg8 = %dest1) -> (tensor<128x128xf32>) { - %3 = scf.for %arg9 = %c0 to %c128 step %c64 iter_args(%arg10 = %arg8) -> (tensor<128x128xf32>) { - %extracted_slice_4 = tensor.extract_slice %arg10[%arg7, %arg9] [64, 64] [1, 1] : tensor<128x128xf32> to tensor<64x64xf32> - %extracted_slice_5 = tensor.extract_slice %extracted_slice_2[%arg7, 0] [64, 512] [1, 1] : tensor<128x512xf32> to tensor<64x512xf32> - %extracted_slice_6 = tensor.extract_slice %extracted_slice_3[0, %arg9] [512, 64] [1, 1] : tensor<512x128xf32> to tensor<512x64xf32> - /// CHECK: %[[FILL_OUT:.*]] = linalg.fill - /// CHECK: %[[MATMUL_OUT:.*]] = linalg.matmul - /// CHECK: %[[EXP_OUT:.*]] = linalg.exp - %4 = linalg.matmul ins(%extracted_slice_5, %extracted_slice_6 : tensor<64x512xf32>, tensor<512x64xf32>) outs(%extracted_slice_4 : tensor<64x64xf32>) -> tensor<64x64xf32> - %insert_slice = tensor.insert_slice %4 into %arg10[%arg7, %arg9] [64, 64] [1, 1] : tensor<64x64xf32> into tensor<128x128xf32> - /// CHECK: scf.yield {{.*}}, {{.*}} : tensor<128x128xf32>, tensor<128x128xf32> - scf.yield %insert_slice : tensor<128x128xf32> - } - /// CHECK: scf.yield {{.*}}, {{.*}} : tensor<128x128xf32>, tensor<128x128xf32> - scf.yield %3 : tensor<128x128xf32> - } - scf.forall.in_parallel { - /// CHECK: tensor.parallel_insert_slice - /// CHECK: tensor.parallel_insert_slice - tensor.parallel_insert_slice %2 into %arg6[%iv0, %iv1] [128, 128] [1, 1] : tensor<128x128xf32> into tensor<256x256xf32> - } - } - %2 = linalg.exp ins(%1 : tensor<256x256xf32>) outs(%dest0 : tensor<256x256xf32>) -> tensor<256x256xf32> - /// CHECK: return %[[FINAL_RESULT]]#1 - return %2 : tensor<256x256xf32> - } -} \ No newline at end of file diff --git a/test/mlir/test/gc/Transforms/mergeNestedForall.mlir b/test/mlir/test/gc/Transforms/mergeNestedForall.mlir deleted file mode 100644 index d878739c8..000000000 --- a/test/mlir/test/gc/Transforms/mergeNestedForall.mlir +++ /dev/null @@ -1,93 +0,0 @@ -// RUN: gc-opt --split-input-file --merge-nested-forall %s | FileCheck %s - -// ----- - -#map = affine_map<(d0) -> (d0 * 1024)> -#map1 = affine_map<(d0) -> (d0 * 2048)> -#map2 = affine_map<(d0)[s0, s1] -> (d0 * 2048 + s0 + s1)> -#map3 = affine_map<(d0)[s0, s1] -> (d0 * 1024 + s0 + s1)> -module { - func.func @matmul_2Dx2D_f32(%arg0: memref<4096x4096xf32>, %arg1: memref<4096x4096xf32>, %arg2: memref<4096x4096xf32>) { - // CHECK: scf.forall {{.*}} (4, 2) - scf.forall (%arg3) in (4) { - scf.forall (%arg4) in (2) { - %c256 = arith.constant 256 : index - %c1024 = arith.constant 1024 : index - %c0 = arith.constant 0 : index - %alloc = memref.alloc() {alignment = 64 : i64} : memref<8x32x32xf32> - scf.for %arg5 = %c0 to %c1024 step %c256 { - %c2048 = arith.constant 2048 : index - scf.for %arg6 = %c0 to %c2048 step %c256 { - %c4096 = arith.constant 4096 : index - scf.for %arg7 = %c0 to %c4096 step %c256 { - %c32 = arith.constant 32 : index - scf.for %arg8 = %c0 to %c256 step %c32 { - scf.for %arg9 = %c0 to %c256 step %c32 { - %0 = affine.apply #map(%arg3) - %1 = affine.apply #map1(%arg4) - %subview = memref.subview %arg2[%0, 0] [1024, 4096] [1, 1] : memref<4096x4096xf32> to memref<1024x4096xf32, strided<[4096, 1], offset: ?>> - %subview_0 = memref.subview %subview[0, %1] [1024, 2048] [1, 1] : memref<1024x4096xf32, strided<[4096, 1], offset: ?>> to memref<1024x2048xf32, strided<[4096, 1], offset: ?>> - %subview_1 = memref.subview %subview_0[%arg5, 0] [256, 2048] [1, 1] : memref<1024x2048xf32, strided<[4096, 1], offset: ?>> to memref<256x2048xf32, strided<[4096, 1], offset: ?>> - %subview_2 = memref.subview %subview_1[0, %arg6] [256, 256] [1, 1] : memref<256x2048xf32, strided<[4096, 1], offset: ?>> to memref<256x256xf32, strided<[4096, 1], offset: ?>> - %subview_3 = memref.subview %subview_2[%arg8, 0] [32, 256] [1, 1] : memref<256x256xf32, strided<[4096, 1], offset: ?>> to memref<32x256xf32, strided<[4096, 1], offset: ?>> - %c1 = arith.constant 1 : index - %c8 = arith.constant 8 : index - %2 = arith.cmpi eq, %arg7, %c0 : index - %3 = affine.apply #map2(%arg4)[%arg9, %arg6] - %subview_4 = memref.subview %arg1[%arg7, %3] [256, 32] [1, 1] : memref<4096x4096xf32> to memref<256x32xf32, strided<[4096, 1], offset: ?>> - %subview_5 = memref.subview %subview_3[0, %arg9] [32, 32] [1, 1] : memref<32x256xf32, strided<[4096, 1], offset: ?>> to memref<32x32xf32, strided<[4096, 1], offset: ?>> - scf.parallel (%arg10, %arg11, %arg12) = (%c0, %c0, %c0) to (%c8, %c32, %c32) step (%c1, %c1, %c1) { - %4 = affine.apply #map3(%arg3)[%arg8, %arg5] - %subview_6 = memref.subview %arg0[%4, %arg7] [32, 256] [1, 1] : memref<4096x4096xf32> to memref<32x256xf32, strided<[4096, 1], offset: ?>> - %expand_shape_7 = memref.expand_shape %subview_6 [[0], [1, 2]] output_shape [32, 8, 32] : memref<32x256xf32, strided<[4096, 1], offset: ?>> into memref<32x8x32xf32, strided<[4096, 32, 1], offset: ?>> - %5 = memref.load %expand_shape_7[%arg11, %arg10, %arg12] : memref<32x8x32xf32, strided<[4096, 32, 1], offset: ?>> - memref.store %5, %alloc[%arg10, %arg11, %arg12] : memref<8x32x32xf32> - scf.reduce - } - %expand_shape = memref.expand_shape %subview_4 [[0, 1], [2]] output_shape [8, 32, 32] : memref<256x32xf32, strided<[4096, 1], offset: ?>> into memref<8x32x32xf32, strided<[131072, 4096, 1], offset: ?>> - scf.if %2 { - scf.parallel (%arg10, %arg11) = (%c0, %c0) to (%c32, %c32) step (%c1, %c1) { - %cst = arith.constant 0.000000e+00 : f32 - memref.store %cst, %subview_5[%arg10, %arg11] : memref<32x32xf32, strided<[4096, 1], offset: ?>> - scf.reduce - } - scf.for %arg10 = %c0 to %c8 step %c1 { - scf.parallel (%arg11, %arg12) = (%c0, %c0) to (%c32, %c32) step (%c1, %c1) { - scf.for %arg13 = %c0 to %c32 step %c1 { - %4 = memref.load %alloc[%arg10, %arg11, %arg13] : memref<8x32x32xf32> - %5 = memref.load %expand_shape[%arg10, %arg13, %arg12] : memref<8x32x32xf32, strided<[131072, 4096, 1], offset: ?>> - %6 = memref.load %subview_5[%arg11, %arg12] : memref<32x32xf32, strided<[4096, 1], offset: ?>> - %7 = arith.mulf %4, %5 : f32 - %8 = arith.addf %6, %7 : f32 - memref.store %8, %subview_5[%arg11, %arg12] : memref<32x32xf32, strided<[4096, 1], offset: ?>> - } - scf.reduce - } - } - } else { - scf.for %arg10 = %c0 to %c8 step %c1 { - scf.parallel (%arg11, %arg12) = (%c0, %c0) to (%c32, %c32) step (%c1, %c1) { - scf.for %arg13 = %c0 to %c32 step %c1 { - %4 = memref.load %alloc[%arg10, %arg11, %arg13] : memref<8x32x32xf32> - %5 = memref.load %expand_shape[%arg10, %arg13, %arg12] : memref<8x32x32xf32, strided<[131072, 4096, 1], offset: ?>> - %6 = memref.load %subview_5[%arg11, %arg12] : memref<32x32xf32, strided<[4096, 1], offset: ?>> - %7 = arith.mulf %4, %5 : f32 - %8 = arith.addf %6, %7 : f32 - memref.store %8, %subview_5[%arg11, %arg12] : memref<32x32xf32, strided<[4096, 1], offset: ?>> - } - scf.reduce - } - } - } - } - } - } - } - } - memref.dealloc %alloc : memref<8x32x32xf32> - } - } - return - } -} - diff --git a/test/mlir/test/gc/Transforms/sinkOpIntoInnerLoop.mlir b/test/mlir/test/gc/Transforms/sinkOpIntoInnerLoop.mlir deleted file mode 100644 index 908d08883..000000000 --- a/test/mlir/test/gc/Transforms/sinkOpIntoInnerLoop.mlir +++ /dev/null @@ -1,46 +0,0 @@ -// RUN: gc-opt --split-input-file --sink-op-into-inner-loop %s | FileCheck %s - -func.func @matmul_2Dx2D_f32(%arg0: memref<4096x4096xf32>, %arg1: memref<4096x4096xf32>, %arg2: memref<4096x4096xf32>) { - %c1 = arith.constant 1 : index - %c8 = arith.constant 8 : index - %cst = arith.constant 0.000000e+00 : f32 - %c0 = arith.constant 0 : index - %c1024 = arith.constant 1024 : index - %c256 = arith.constant 256 : index - %c2048 = arith.constant 2048 : index - %c4096 = arith.constant 4096 : index - %c32 = arith.constant 32 : index - // CHECK: scf.forall - // CHECK-NOT: affine.apply - // CHECK-NOT: memref.subview - // CHECK-NEXT: scf.forall - scf.forall (%arg3) in (4) { - %0 = affine.apply affine_map<(d0) -> (d0 * 1024)>(%arg3) - %subview = memref.subview %arg2[%0, 0] [1024, 4096] [1, 1] : memref<4096x4096xf32> to memref<1024x4096xf32, strided<[4096, 1], offset: ?>> - scf.forall (%arg4) in (2) { - %1 = affine.apply affine_map<(d0) -> (d0 * 2048)>(%arg4) - %subview_0 = memref.subview %subview[0, %1] [1024, 2048] [1, 1] : memref<1024x4096xf32, strided<[4096, 1], offset: ?>> to memref<1024x2048xf32, strided<[4096, 1], offset: ?>> - %alloc = memref.alloc() {alignment = 64 : i64} : memref<8x32x32xf32> - scf.for %arg5 = %c0 to %c1024 step %c256 { - %subview_1 = memref.subview %subview_0[%arg5, 0] [256, 2048] [1, 1] : memref<1024x2048xf32, strided<[4096, 1], offset: ?>> to memref<256x2048xf32, strided<[4096, 1], offset: ?>> - scf.for %arg6 = %c0 to %c2048 step %c256 { - %subview_2 = memref.subview %subview_1[0, %arg6] [256, 256] [1, 1] : memref<256x2048xf32, strided<[4096, 1], offset: ?>> to memref<256x256xf32, strided<[4096, 1], offset: ?>> - scf.for %arg7 = %c0 to %c4096 step %c256 { - %2 = arith.cmpi eq, %arg7, %c0 : index - scf.for %arg8 = %c0 to %c256 step %c32 { - %3 = affine.apply affine_map<(d0)[s0, s1] -> (d0 * 1024 + s0 + s1)>(%arg3)[%arg8, %arg5] - %subview_3 = memref.subview %arg0[%3, %arg7] [32, 256] [1, 1] : memref<4096x4096xf32> to memref<32x256xf32, strided<[4096, 1], offset: ?>> - %subview_4 = memref.subview %subview_2[%arg8, 0] [32, 256] [1, 1] : memref<256x256xf32, strided<[4096, 1], offset: ?>> to memref<32x256xf32, strided<[4096, 1], offset: ?>> - %expand_shape = memref.expand_shape %subview_3 [[0], [1, 2]] output_shape [32, 8, 32] : memref<32x256xf32, strided<[4096, 1], offset: ?>> into memref<32x8x32xf32, strided<[4096, 32, 1], offset: ?>> - scf.for %arg9 = %c0 to %c256 step %c32 { - - } - } - } - } - } - memref.dealloc %alloc : memref<8x32x32xf32> - } - } - return -} \ No newline at end of file diff --git a/test/mlir/test/gc/Transforms/vectorization.mlir b/test/mlir/test/gc/Transforms/vectorization.mlir deleted file mode 100644 index 2f87dffc3..000000000 --- a/test/mlir/test/gc/Transforms/vectorization.mlir +++ /dev/null @@ -1,218 +0,0 @@ -// RUN: gc-opt --split-input-file --fold-tensor-operation --lower-to-tile-vector -- %s | FileCheck %s - -// CHECK-LABEL: @add_tensor_test0 -// CHECK: %[[CST:.*]] = arith.constant 0.000000e+00 : f32 -// CHECK: %[[C0:.*]] = arith.constant 0 : index -// CHECK: %[[EMPTY:.*]] = tensor.empty() : tensor<4x8x16xf32> -// CHECK: %[[READ0:.*]] = vector.transfer_read %{{.*}} {in_bounds = [true, true, true]} : tensor<4x8x16xf32>, vector<4x8x16xf32> -// CHECK: %[[READ1:.*]] = vector.transfer_read %{{.*}} {in_bounds = [true, true, true]} : tensor<4x8x16xf32>, vector<4x8x16xf32> -// CHECK: %[[ADD0:.*]] = arith.addf %[[READ0]], %[[READ1]] : vector<4x8x16xf32> -// CHECK: %[[WRITE0:.*]] = vector.transfer_write %{{.*}} {in_bounds = [true, true, true]} : vector<4x8x16xf32>, tensor<4x8x16xf32> -func.func @add_tensor_test0(%arg0: tensor<4x8x16xf32>, %arg1: tensor<4x8x16xf32>) -> tensor<4x8x16xf32> { - %0 = tensor.empty() : tensor<4x8x16xf32> - %1 = linalg.add ins(%arg0, %arg1 : tensor<4x8x16xf32>, tensor<4x8x16xf32>) outs(%0: tensor<4x8x16xf32>) -> tensor<4x8x16xf32> - return %1 : tensor<4x8x16xf32> -} - -// CHECK-LABEL: @add_tensor_test1 -// CHECK: %[[CST:.*]] = arith.constant 0.000000e+00 : f32 -// CHECK: %[[C0:.*]] = arith.constant 0 : index -// CHECK: %[[EMPTY:.*]] = tensor.empty() : tensor<1x8x8xf32> -// CHECK: %[[EXTRACT0:.*]] = tensor.extract_slice %arg0[0, 0, 0] [1, 8, 8] [1, 1, 1] : tensor<4x8x16xf32> to tensor<1x8x8xf32> -// CHECK: %[[EXTRACT1:.*]] = tensor.extract_slice %arg1[0, 0, 0] [1, 8, 8] [1, 1, 1] : tensor<4x8x16xf32> to tensor<1x8x8xf32> -// CHECK: %[[READ0:.*]] = vector.transfer_read %{{.*}} {in_bounds = [true, true, true]} : tensor<1x8x8xf32>, vector<1x8x8xf32> -// CHECK: %[[READ1:.*]] = vector.transfer_read %{{.*}} {in_bounds = [true, true, true]} : tensor<1x8x8xf32>, vector<1x8x8xf32> -// CHECK: %[[ADD0:.*]] = arith.addf %[[READ0]], %[[READ1]] : vector<1x8x8xf32> -// CHECK: %[[WRITE0:.*]] = vector.transfer_write %{{.*}} {in_bounds = [true, true, true]} : vector<1x8x8xf32>, tensor<1x8x8xf32> -func.func @add_tensor_test1(%arg0: tensor<4x8x16xf32>, %arg1: tensor<4x8x16xf32>) -> tensor<1x8x8xf32> { - %0 = tensor.empty() : tensor<1x8x8xf32> - %1 = tensor.extract_slice %arg0[0, 0, 0] [1, 8, 8] [1, 1, 1] : tensor<4x8x16xf32> to tensor<1x8x8xf32> - %2 = tensor.extract_slice %arg1[0, 0, 0] [1, 8, 8] [1, 1, 1] : tensor<4x8x16xf32> to tensor<1x8x8xf32> - %3 = linalg.add ins(%1, %2 : tensor<1x8x8xf32>, tensor<1x8x8xf32>) outs(%0: tensor<1x8x8xf32>) -> tensor<1x8x8xf32> - return %3 : tensor<1x8x8xf32> -} - -// CHECK-LABEL: @add_tensor_pack_test2 -// CHECK: %[[CST:.*]] = arith.constant 0.000000e+00 : f32 -// CHECK: %[[C0:.*]] = arith.constant 0 : index -// CHECK: %[[READ0:.*]] = vector.transfer_read %{{.*}} {in_bounds = [true, true, true]} : tensor<4x16x16xf32>, vector<4x16x16xf32> -// CHECK: %[[SHAPECAST0:.*]] = vector.shape_cast %[[READ0]] : vector<4x16x16xf32> to vector<4x4x4x4x4xf32> -// CHECK: %[[READ1:.*]] = vector.transfer_read %{{.*}} {in_bounds = [true, true, true]} : tensor<4x16x16xf32>, vector<4x16x16xf32> -// CHECK: %[[SHAPECAST1:.*]] = vector.shape_cast %[[READ1]] : vector<4x16x16xf32> to vector<4x4x4x4x4xf32> -// CHECK: %[[EMPTY:.*]] = tensor.empty() : tensor<4x4x4x4x4xf32> -// CHECK: %[[ADD0:.*]] = arith.addf %[[SHAPECAST0]], %[[SHAPECAST1]] : vector<4x4x4x4x4xf32> -// CHECK: %[[TRANSPOSE1:.*]] = vector.transpose %[[ADD0]], [1, 0, 3, 2, 4] : vector<4x4x4x4x4xf32> to vector<4x4x4x4x4xf32> -// CHECK: %[[WRITE0:.*]] = vector.transfer_write %{{.*}} {in_bounds = [true, true, true, true, true]} : vector<4x4x4x4x4xf32>, tensor<4x4x4x4x4xf32> -func.func @add_tensor_pack_test2(%arg0: tensor<4x16x16xf32>, %arg1: tensor<4x16x16xf32>) -> tensor<4x4x4x4x4xf32> { - %0 = tensor.empty() : tensor<4x4x4x4x4xf32> - %1 = tensor.empty() : tensor<4x4x4x4x4xf32> - %2 = tensor.pack %arg0 outer_dims_perm = [1, 0, 2] inner_dims_pos = [1, 2] inner_tiles = [4, 4] into %0 : tensor<4x16x16xf32> -> tensor<4x4x4x4x4xf32> - %3 = tensor.pack %arg1 outer_dims_perm = [1, 0, 2] inner_dims_pos = [1, 2] inner_tiles = [4, 4] into %1 : tensor<4x16x16xf32> -> tensor<4x4x4x4x4xf32> - %4 = tensor.empty() : tensor<4x4x4x4x4xf32> - %6 = linalg.add ins(%2, %3 : tensor<4x4x4x4x4xf32>, tensor<4x4x4x4x4xf32>) outs(%4: tensor<4x4x4x4x4xf32>) -> tensor<4x4x4x4x4xf32> - return %6 : tensor<4x4x4x4x4xf32> -} - -// CHECK-LABEL: @add_tensor_pad_test3 -// CHECK: %[[cst:.*]] = arith.constant dense<0.000000e+00> : vector<4x16x16xf32> -// CHECK: %[[C0:.*]] = arith.constant 0 : index -// CHECK: %[[CST_0:.*]] = arith.constant 0.000000e+00 : f32 -// CHECK: %[[EMPTY0:.*]] = tensor.empty() : tensor<4x16x16xf32> -// CHECK: %[[FILL0:.*]] = vector.transfer_write %[[cst]], %[[EMPTY0]][%c0, %c0, %c0] {in_bounds = [true, true, true]} : vector<4x16x16xf32>, tensor<4x16x16xf32> -// CHECK: %[[READ0:.*]] = vector.transfer_read %{{.*}} {in_bounds = [true, true, true]} : tensor<4x16x15xf32>, vector<4x16x15xf32> -// CHECK: %[[WRITE0:.*]] = vector.transfer_write %[[READ0]], %[[FILL0]][%c0, %c0, %c0] {in_bounds = [true, true, true]} : vector<4x16x15xf32>, tensor<4x16x16xf32> -// CHECK: %[[EMPTY1:.*]] = tensor.empty() : tensor<4x16x16xf32> -// CHECK: %[[FILL1:.*]] = vector.transfer_write %[[cst]], %[[EMPTY1]][%c0, %c0, %c0] {in_bounds = [true, true, true]} : vector<4x16x16xf32>, tensor<4x16x16xf32> -// CHECK: %[[READ1:.*]] = vector.transfer_read %{{.*}} {in_bounds = [true, true, true]} : tensor<4x16x15xf32>, vector<4x16x15xf32> -// CHECK: %[[WRITE1:.*]] = vector.transfer_write %[[READ1]], %[[FILL1]][%c0, %c0, %c0] {in_bounds = [true, true, true]} : vector<4x16x15xf32>, tensor<4x16x16xf32> -// CHECK: %[[EMPTY2:.*]] = tensor.empty() : tensor<4x16x16xf32> -// CHECK: %[[READ2:.*]] = vector.transfer_read %{{.*}} {in_bounds = [true, true, true]} : tensor<4x16x16xf32>, vector<4x16x16xf32> -// CHECK: %[[READ3:.*]] = vector.transfer_read %{{.*}} {in_bounds = [true, true, true]} : tensor<4x16x16xf32>, vector<4x16x16xf32> -// CHECK: %[[ADD0:.*]] = arith.addf %[[READ2]], %[[READ3]] : vector<4x16x16xf32> -// CHECK: %[[WRITE2:.*]] = vector.transfer_write %{{.*}} {in_bounds = [true, true, true]} : vector<4x16x16xf32>, tensor<4x16x16xf32> -func.func @add_tensor_pad_test3(%arg0: tensor<4x16x15xf32>, %arg1: tensor<4x16x15xf32>) -> tensor<4x16x16xf32> { - %cst = arith.constant 0.000000e+00 : f32 - %0 = tensor.pad %arg0 low[0, 0, 0] high[0, 0, 1] { - ^bb0(%arg2: index, %arg3: index, %arg4: index): - tensor.yield %cst : f32 - } : tensor<4x16x15xf32> to tensor<4x16x16xf32> - %1 = tensor.pad %arg1 low[0, 0, 0] high[0, 0, 1] { - ^bb0(%arg5: index, %arg6: index, %arg7: index): - tensor.yield %cst : f32 - } : tensor<4x16x15xf32> to tensor<4x16x16xf32> - %2 = tensor.empty() : tensor<4x16x16xf32> - %3 = linalg.add ins(%0, %1 : tensor<4x16x16xf32>, tensor<4x16x16xf32>) outs(%2: tensor<4x16x16xf32>) -> tensor<4x16x16xf32> - return %3 : tensor<4x16x16xf32> -} - -// CHECK-LABEL: @add_tensor_test4 -// CHECK: %[[CST:.*]] = arith.constant 0.000000e+00 : f32 -// CHECK: %[[C0:.*]] = arith.constant 0 : index -// CHECK: %[[READ0:.*]] = vector.transfer_read %{{.*}} {in_bounds = [true, true, true, true, true]} : tensor<12x2x56x56x32xf32>, vector<12x2x56x56x32xf32> -// CHECK: %[[TRANSPOSE0:.*]] = vector.transpose %[[READ0]], [0, 2, 3, 1, 4] : vector<12x2x56x56x32xf32> to vector<12x56x56x2x32xf32> -// CHECK: %[[SHAPECAST0:.*]] = vector.shape_cast %[[TRANSPOSE0]] : vector<12x56x56x2x32xf32> to vector<12x56x56x64xf32> -// CHECK: %[[READ1:.*]] = vector.transfer_read %{{.*}} {in_bounds = [true, true, true, true, true]} : tensor<12x2x56x56x32xf32>, vector<12x2x56x56x32xf32> -// CHECK: %[[TRANSPOSE1:.*]] = vector.transpose %[[READ1]], [0, 2, 3, 1, 4] : vector<12x2x56x56x32xf32> to vector<12x56x56x2x32xf32> -// CHECK: %[[SHAPECAST1:.*]] = vector.shape_cast %[[TRANSPOSE1]] : vector<12x56x56x2x32xf32> to vector<12x56x56x64xf32> -// CHECK: %[[EMPTY:.*]] = tensor.empty() : tensor<12x56x56x64xf32> -// CHECK: %[[ADD0:.*]] = arith.addf %[[SHAPECAST0]], %[[SHAPECAST1]] : vector<12x56x56x64xf32> -// CHECK: %[[WRITE0:.*]] = vector.transfer_write %{{.*}} {in_bounds = [true, true, true, true]} : vector<12x56x56x64xf32>, tensor<12x56x56x64xf32> -func.func @add_tensor_test4(%arg0: tensor<12x2x56x56x32xf32>, %arg1: tensor<12x2x56x56x32xf32>) -> tensor<12x56x56x64xf32> { - %0 = tensor.empty() : tensor<12x56x56x64xf32> - %1 = tensor.unpack %arg0 outer_dims_perm = [0, 3, 1, 2] inner_dims_pos = [3] inner_tiles = [32] into %0 : tensor<12x2x56x56x32xf32> -> tensor<12x56x56x64xf32> - %2 = tensor.empty() : tensor<12x56x56x64xf32> - %3 = tensor.unpack %arg1 outer_dims_perm = [0, 3, 1, 2] inner_dims_pos = [3] inner_tiles = [32] into %2 : tensor<12x2x56x56x32xf32> -> tensor<12x56x56x64xf32> - %4 = tensor.empty() : tensor<12x56x56x64xf32> - %5 = linalg.add ins(%1, %3 : tensor<12x56x56x64xf32>, tensor<12x56x56x64xf32>) outs(%4: tensor<12x56x56x64xf32>) -> tensor<12x56x56x64xf32> - return %5 : tensor<12x56x56x64xf32> -} - -// CHECK-LABEL: @add_tensor_test5 -// CHECK: %[[CST:.*]] = arith.constant 0.000000e+00 : f32 -// CHECK: %[[CST_0:.*]] = arith.constant dense<1.000000e+00> : vector<1x8xf32> -// CHECK: %[[C0:.*]] = arith.constant 0 : index -// CHECK: %[[EMPTY0:.*]] tensor.empty() : tensor<1x8xf32> -// CHECK: %[[WRITE0:.*]] = vector.transfer_write %{{.*}} {in_bounds = [true, true]} : vector<1x8xf32>, tensor<1x8xf32> -// CHECK: %[[extracted_slice:.*]] = tensor.extract_slice %1[0, 0] [1, 8] [1, 1] : tensor<1x8xf32> to tensor<8xf32> -// CHECK: %[[READ1:.*]] = vector.transfer_read %{{.*}} {in_bounds = [true]} : tensor<8xf32>, vector<8xf32> -// CHECK: %[[SHAPECAST1:.*]] = vector.shape_cast %[[READ1]] : vector<8xf32> to vector<1x1x1x8xf32> -// CHECK: %[[EMPTY1:.*]] = tensor.empty() : tensor<1x1x1x8xf32> -// CHECK: %[[WRITE0:.*]] = vector.transfer_write %{{.*}} {in_bounds = [true, true, true, true]} : vector<1x1x1x8xf32>, tensor<1x1x1x8xf32> -func.func @add_tensor_test5() -> tensor<1x1x1x8xf32> { - %cst = arith.constant 1.000000e+00 : f32 - %init = tensor.empty() : tensor<1x8xf32> - %fill = linalg.fill ins(%cst : f32) outs(%init : tensor<1x8xf32>) -> tensor<1x8xf32> - %slice = tensor.extract_slice %fill[0, 0] [1, 8] [1, 1] : tensor<1x8xf32> to tensor<8xf32> - %expand = tensor.expand_shape %slice [[0, 1, 2, 3]] output_shape [1, 1, 1, 8] : tensor<8xf32> into tensor<1x1x1x8xf32> - return %expand : tensor<1x1x1x8xf32> -} - -// CHECK-LABEL: @tensor_collapse_shape_test6 -// CHECK: %[[CST:.*]] = arith.constant 0.000000e+00 : f32 -// CHECK: %[[C0:.*]] = arith.constant 0 : index -// CHECK: %[[READ1:.*]] = vector.transfer_read %{{.*}} {in_bounds = [true, true]} : tensor<2x3xf32>, vector<2x3xf32> -// CHECK: %[[SHAPECAST1:.*]] = vector.shape_cast %[[READ1]] : vector<2x3xf32> to vector<6xf32> -// CHECK: %[[EMPTY1:.*]] = tensor.empty() : tensor<6xf32> -// CHECK: %[[WRITE0:.*]] = vector.transfer_write %{{.*}} {in_bounds = [true]} : vector<6xf32>, tensor<6xf32> -func.func @tensor_collapse_shape_test6(%arg0: tensor<2x3xf32>) -> tensor<6xf32> { - %0 = tensor.collapse_shape %arg0 [[0, 1]] : tensor<2x3xf32> into tensor<6xf32> - return %0 : tensor<6xf32> -} - -// CHECK-LABEL: @tensor_static_concat_test7 -// CHECK: %[[C64:.*]] = arith.constant 64 : index -// CHECK: %[[C0:.*]] = arith.constant 0 : index -// CHECK: %[[CST:.*]] = arith.constant 0.000000e+00 : f32 -// CHECK: %[[EMPTY0:.*]] = tensor.empty() : tensor<1x1x128xf32> -// CHECK: %[[READ0:.*]] = vector.transfer_read %{{.*}} {in_bounds = [true, true, true]} : tensor<1x1x64xf32>, vector<1x1x64xf32> -// CHECK: %[[WRITE0:.*]] = vector.transfer_write %[[READ0]], %[[EMPTY0]][%[[C0]], %[[C0]], %[[C0]]] {in_bounds = [true, true, true]} : vector<1x1x64xf32>, tensor<1x1x128xf32> -// CHECK: %[[READ1:.*]] = vector.transfer_read %{{.*}} {in_bounds = [true, true, true]} : tensor<1x1x64xf32>, vector<1x1x64xf32> -// CHECK: %[[WRITE1:.*]] = vector.transfer_write %[[READ1]], %[[WRITE0]][%[[C0]], %[[C0]], %[[C64]]] {in_bounds = [true, true, true]} : vector<1x1x64xf32>, tensor<1x1x128xf32> -func.func @tensor_static_concat_test7(%arg0 : tensor<1x1x64xf32>, - %arg1: tensor<1x1x64xf32>) -> tensor<1x1x128xf32> { - %0 = tensor.concat dim(2) %arg0, %arg1 - : (tensor<1x1x64xf32>, tensor<1x1x64xf32>) -> tensor<1x1x128xf32> - return %0 : tensor<1x1x128xf32> -} - -// CHECK-LABEL: @fc_relu_test8 -// CHECK: %[[CST:.*]] = arith.constant dense<0.000000e+00> : vector<512x512xf32> -// CHECK: %[[CST_0:.*]] = arith.constant 0.000000e+00 : f32 -// CHECK: %[[C0:.*]] = arith.constant 0 : index -// CHECK: %[[MATMUL:.*]] = linalg.matmul -// CHECK: %[[READ0:.*]] = vector.transfer_read %{{.*}} {in_bounds = [true, true]} : tensor<512x512xf32>, vector<512x512xf32> -// CHECK: %[[READ1:.*]] = vector.transfer_read %{{.*}} {in_bounds = [true, true]} : tensor<512x512xf32>, vector<512x512xf32> -// CHECK: %[[ADD0:.*]] = arith.addf %[[READ0]], %[[READ1]] : vector<512x512xf32> -// CHECK: %[[MAX0:.*]] = arith.maximumf %[[ADD0]], %[[CST]] : vector<512x512xf32> -// CHECK: %[[WRITE0:.*]] = vector.transfer_write %{{.*}} {in_bounds = [true, true]} : vector<512x512xf32>, tensor<512x512xf32> -func.func @fc_relu_test8(%lhs: tensor<512x512xf32>, %rhs: tensor<512x512xf32>, - %bias: tensor<512x512xf32>, %output: tensor<512x512xf32>) - -> tensor<512x512xf32> { - %matmul = linalg.matmul ins(%lhs, %rhs: tensor<512x512xf32>, tensor<512x512xf32>) - outs(%output: tensor<512x512xf32>) -> tensor<512x512xf32> - - %biased = linalg.elemwise_binary { fun = #linalg.binary_fn } - ins(%matmul, %bias : tensor<512x512xf32>, tensor<512x512xf32>) - outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32> - - %c0f = arith.constant 0.0 : f32 - %relued = linalg.elemwise_binary { fun = #linalg.binary_fn } - ins(%biased, %c0f : tensor<512x512xf32>, f32) - outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32> - func.return %relued : tensor<512x512xf32> -} - - -// CHECK-LABEL: @test_pad_dynamic_shape_test9 -// CHECK: %[[C0:.*]] = arith.constant 0 : index -// CHECK: %[[CST_0:.*]] = arith.constant 0.000000e+00 : f32 -// CHECK: %[[EMPTY0:.*]] = tensor.empty() : tensor<4x16xf32> -// CHECK: %[[READ0:.*]] = vector.transfer_read %{{.*}} : tensor, vector<4x16xf32> -// CHECK: %[[WRITE0:.*]] = vector.transfer_write %{{.*}} {in_bounds = [true, true]} : vector<4x16xf32>, tensor<4x16xf32> -func.func @test_pad_dynamic_shape_test9(%arg0: tensor) -> tensor<4x16xf32> { - %f0 = arith.constant 0.0 : f32 - %pad = tensor.pad %arg0 low[0, 0] high[1,1] { - ^bb0(%arg3: index, %arg4: index): - tensor.yield %f0 : f32 - } : tensor to tensor<4x16xf32> - return %pad : tensor<4x16xf32> -} - -func.func @test_add_dynamic_shape_test11(%arg0: tensor, %arg1: tensor<4x16xf32>) -> tensor<4x16xf32> { - %0 = tensor.empty() : tensor<4x16xf32> - // expected-error @+1 {{Fail to vectorize.}} - %1 = linalg.add ins(%arg0, %arg1 : tensor, tensor<4x16xf32>) outs(%0: tensor<4x16xf32>) -> tensor<4x16xf32> - return %1 : tensor<4x16xf32> -} - -func.func @test_collapse_dynamic_shape_test12(%arg0: tensor) -> tensor { - // expected-error @+1 {{Fail to vectorize.}} - %0 = tensor.collapse_shape %arg0 [[0], [1, 2]]: tensor into tensor - return %0 : tensor -} - -func.func @test_expand_dynamic_shape_test13(%arg0 : tensor<2x?x5xf32>, %sz0: index) ->tensor<2x6x?x5xf32> { - // expected-error @+1 {{Fail to vectorize.}} - %0 = tensor.expand_shape %arg0 [[0], [1, 2], [3]] output_shape [2, 6, %sz0, 5] : tensor<2x?x5xf32> into tensor<2x6x?x5xf32> - return %0 : tensor<2x6x?x5xf32> -} diff --git a/test/mlir/test/gc/Unit/lit.cfg.py b/test/mlir/test/gc/Unit/lit.cfg.py deleted file mode 100644 index dac55589e..000000000 --- a/test/mlir/test/gc/Unit/lit.cfg.py +++ /dev/null @@ -1,50 +0,0 @@ -# -*- Python -*- - -# Configuration file for the 'lit' test runner. - -import os -import subprocess - -import lit.formats - -# name: The name of this test suite. -config.name = "GC-Unit" - -# suffixes: A list of file extensions to treat as test files. -config.suffixes = [] - -# test_source_root: The root path where tests are located. -# test_exec_root: The root path where tests should be run. -config.test_exec_root = os.path.join(config.gc_obj_root, "test", "mlir", "unittests") -config.test_source_root = config.test_exec_root - -# testFormat: The test format to use to interpret tests. -config.test_format = lit.formats.GoogleTest(config.llvm_build_mode, "Tests") - -# Propagate the temp directory. Windows requires this because it uses \Windows\ -# if none of these are present. -if "TMP" in os.environ: - config.environment["TMP"] = os.environ["TMP"] -if "TEMP" in os.environ: - config.environment["TEMP"] = os.environ["TEMP"] - -# Propagate HOME as it can be used to override incorrect homedir in passwd -# that causes the tests to fail. -if "HOME" in os.environ: - config.environment["HOME"] = os.environ["HOME"] - -# Propagate sanitizer options. -for var in [ - "ASAN_SYMBOLIZER_PATH", - "HWASAN_SYMBOLIZER_PATH", - "MSAN_SYMBOLIZER_PATH", - "TSAN_SYMBOLIZER_PATH", - "UBSAN_SYMBOLIZER_PATH", - "ASAN_OPTIONS", - "HWASAN_OPTIONS", - "MSAN_OPTIONS", - "TSAN_OPTIONS", - "UBSAN_OPTIONS", -]: - if var in os.environ: - config.environment[var] = os.environ[var] \ No newline at end of file diff --git a/test/mlir/test/gc/Unit/lit.site.cfg.py.in b/test/mlir/test/gc/Unit/lit.site.cfg.py.in deleted file mode 100644 index d05280c7d..000000000 --- a/test/mlir/test/gc/Unit/lit.site.cfg.py.in +++ /dev/null @@ -1,12 +0,0 @@ -@LIT_SITE_CFG_IN_HEADER@ - -import sys -config.llvm_tools_dir = lit_config.substitute("@LLVM_TOOLS_DIR@") -config.llvm_build_mode = lit_config.substitute("@LLVM_BUILD_MODE@") -config.mlir_obj_root = "@MLIR_BINARY_DIR@" -config.gc_src_root = "@CMAKE_SOURCE_DIR@" -config.gc_obj_root = "@PROJECT_BINARY_DIR@" -config.gc_lib_dir = os.path.join(config.gc_obj_root, "lib") - -# Let the main config do the real work. -lit_config.load_config(config, "@CMAKE_SOURCE_DIR@/test/mlir/test/gc/Unit/lit.cfg.py") \ No newline at end of file diff --git a/test/mlir/test/gc/cpu-runner/GPU/lit.local.cfg b/test/mlir/test/gc/cpu-runner/GPU/lit.local.cfg deleted file mode 100644 index 37d7602a3..000000000 --- a/test/mlir/test/gc/cpu-runner/GPU/lit.local.cfg +++ /dev/null @@ -1,2 +0,0 @@ -if not config.gc_use_gpu: - config.unsupported = True diff --git a/test/mlir/test/gc/cpu-runner/GPU/xevm_block_dpas.mlir b/test/mlir/test/gc/cpu-runner/GPU/xevm_block_dpas.mlir deleted file mode 100644 index 3f28e68bc..000000000 --- a/test/mlir/test/gc/cpu-runner/GPU/xevm_block_dpas.mlir +++ /dev/null @@ -1,117 +0,0 @@ -// RUN: gc-opt %s --convert-xevm-to-llvm --xevm-attach-target --convert-scf-to-cf --convert-cf-to-llvm --convert-arith-to-llvm --convert-gpu-to-llvm-spv --gpu-to-llvm --reconcile-unrealized-casts --cse --gpu-module-to-binary | gc-cpu-runner -e main -entry-point-result=void --shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%opencl_runtime | FileCheck %s - -module @gemm attributes {gpu.container_module} { - gpu.module @kernel { - // - Sets of `matrix_mad` intrinsics can differ based on device's *minimal* supported sub-group size. - // The *minimum supported* sub-group size should be used to call `matrix_mad` intrinsics. - // https://registry.khronos.org/OpenCL/extensions/intel/cl_intel_subgroup_matrix_multiply_accumulate.html - - gpu.func @block_dpas(%a: !llvm.ptr<1>, %b: !llvm.ptr<1>, %c: !llvm.ptr<1>) kernel { - %base_width_a = arith.constant 32 : i32 - %base_height_a = arith.constant 8 : i32 - %base_pitch_a = arith.constant 32 : i32 - %x = arith.constant 0 : i32 - %y = arith.constant 0 : i32 - %loaded_a = xevm.blockload2d %a, %base_width_a, %base_height_a, %base_pitch_a, %x, %y {elem_size_in_bits=16, tile_width=16, tile_height=8, v_blocks=1, transpose=false, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<8xi16> - - %base_width_b = arith.constant 32 : i32 - %base_height_b = arith.constant 16 : i32 - %base_pitch_b = arith.constant 32 : i32 - %loaded_b1 = xevm.blockload2d %b, %base_width_b, %base_height_b, %base_pitch_b, %x, %y {elem_size_in_bits=16, tile_width=16, tile_height=16, v_blocks=1, transpose=false, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<16xi16> - %loaded_b_casted = vector.bitcast %loaded_b1 : vector<16xi16> to vector<8xi32> - - %base_width_c = arith.constant 64 : i32 - %base_height_c = arith.constant 8 : i32 - %base_pitch_c = arith.constant 64 : i32 - %loaded_c = xevm.blockload2d %c, %base_width_c, %base_height_c, %base_pitch_c, %x, %y {elem_size_in_bits=32, tile_width=16, tile_height=8, v_blocks=1, transpose=false, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<8xi32> - - %loaded_c_casted = vector.bitcast %loaded_c : vector<8xi32> to vector<8xf32> - %c_result = xevm.dpas %loaded_c_casted, %loaded_a, %loaded_b_casted {pa = f16, pb = f16, rc = 8} : (vector<8xf32>, vector<8xi16>, vector<8xi32>) -> vector<8xf32> - %c_result_casted = vector.bitcast %c_result : vector<8xf32> to vector<8xi32> - - xevm.blockstore2d %c, %base_width_c, %base_height_c, %base_pitch_c, %x, %y, %c_result_casted {elem_size_in_bits=32, tile_width=16, tile_height=8, v_blocks=1, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32, vector<8xi32>) - gpu.return - } - } - - func.func @test(%a : memref<8x16xf16>, %b : memref<16x16xf16>, %c : memref<8x16xf32>) -> memref<8x16xf32> attributes {llvm.emit_c_interface} { - %c1 = arith.constant 1 : index - %c16 = arith.constant 16 : index - - %memref_a = gpu.alloc host_shared () : memref<8x16xf16> - memref.copy %a, %memref_a : memref<8x16xf16> to memref<8x16xf16> - %a_ptr_as_idx = memref.extract_aligned_pointer_as_index %memref_a : memref<8x16xf16> -> index - %a_ptr_as_i64 = arith.index_cast %a_ptr_as_idx : index to i64 - %a_ptr = llvm.inttoptr %a_ptr_as_i64 : i64 to !llvm.ptr - %a_ptr_casted = llvm.addrspacecast %a_ptr : !llvm.ptr to !llvm.ptr<1> - - %memref_b = gpu.alloc host_shared () : memref<16x16xf16> - memref.copy %b, %memref_b : memref<16x16xf16> to memref<16x16xf16> - %b_ptr_as_idx = memref.extract_aligned_pointer_as_index %memref_b : memref<16x16xf16> -> index - %b_ptr_as_i64 = arith.index_cast %b_ptr_as_idx : index to i64 - %b_ptr = llvm.inttoptr %b_ptr_as_i64 : i64 to !llvm.ptr - %b_ptr_casted = llvm.addrspacecast %b_ptr : !llvm.ptr to !llvm.ptr<1> - - %memref_c = gpu.alloc host_shared () : memref<8x16xf32> - memref.copy %c, %memref_c : memref<8x16xf32> to memref<8x16xf32> - %c_ptr_as_idx = memref.extract_aligned_pointer_as_index %memref_c : memref<8x16xf32> -> index - %c_ptr_as_i64 = arith.index_cast %c_ptr_as_idx : index to i64 - %c_ptr = llvm.inttoptr %c_ptr_as_i64 : i64 to !llvm.ptr - %c_ptr_casted = llvm.addrspacecast %c_ptr : !llvm.ptr to !llvm.ptr<1> - - gpu.launch_func @kernel::@block_dpas blocks in (%c1, %c1, %c1) threads in (%c16, %c1, %c1) args(%a_ptr_casted : !llvm.ptr<1>, %b_ptr_casted : !llvm.ptr<1>, %c_ptr_casted : !llvm.ptr<1>) - return %memref_c : memref<8x16xf32> - } - - func.func @main() attributes {llvm.emit_c_interface} { - %A = memref.alloc() : memref<8x16xf16> - %c0 = arith.constant 0 : index - %c1 = arith.constant 1 : index - %c8 = arith.constant 8 : index - %c16 = arith.constant 16 : index - - scf.for %i = %c0 to %c8 step %c1 { - scf.for %j = %c0 to %c16 step %c1 { - %row_idx = arith.index_cast %i : index to i32 - %row = arith.sitofp %row_idx : i32 to f16 - memref.store %row, %A[%i, %j] : memref<8x16xf16> - } - } - %B = memref.alloc() : memref<16x16xf16> - scf.for %i = %c0 to %c16 step %c1 { - scf.for %j = %c0 to %c16 step %c1 { - %col_idx = arith.index_cast %j : index to i32 - %col = arith.sitofp %col_idx : i32 to f16 - memref.store %col, %B[%i, %j] : memref<16x16xf16> - } - } - - %C = memref.alloc() : memref<8x16xf32> - %c0_f16 = arith.constant 0.0 : f32 - scf.for %i = %c0 to %c8 step %c1 { - scf.for %j = %c0 to %c16 step %c1 { - memref.store %c0_f16, %C[%i, %j] : memref<8x16xf32> - } - } - - %C_res = call @test(%A, %B, %C) : (memref<8x16xf16>, memref<16x16xf16>, memref<8x16xf32>) -> memref<8x16xf32> - %C_cast = memref.cast %C_res : memref<8x16xf32> to memref<*xf32> - %A_cast = memref.cast %A : memref<8x16xf16> to memref<*xf16> - call @printMemrefF32(%C_cast) : (memref<*xf32>) -> () - - // CHECK: Unranked Memref base@ = 0x{{[0-9a-f]+}} - // CHECK-NEXT: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] - // CHECK-NEXT: [0, 16, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240] - // CHECK-NEXT: [0, 32, 64, 96, 128, 160, 192, 224, 256, 288, 320, 352, 384, 416, 448, 480] - // CHECK-NEXT: [0, 48, 96, 144, 192, 240, 288, 336, 384, 432, 480, 528, 576, 624, 672, 720] - // CHECK-NEXT: [0, 64, 128, 192, 256, 320, 384, 448, 512, 576, 640, 704, 768, 832, 896, 960] - // CHECK-NEXT: [0, 80, 160, 240, 320, 400, 480, 560, 640, 720, 800, 880, 960, 1040, 1120, 1200] - // CHECK-NEXT: [0, 96, 192, 288, 384, 480, 576, 672, 768, 864, 960, 1056, 1152, 1248, 1344, 1440] - // CHECK-NEXT: [0, 112, 224, 336, 448, 560, 672, 784, 896, 1008, 1120, 1232, 1344, 1456, 1568, 1680] - - return - } - func.func private @printMemrefF16(%ptr : memref<*xf16>) attributes { llvm.emit_c_interface } - func.func private @printMemrefF32(%ptr : memref<*xf32>) attributes { llvm.emit_c_interface } - -} diff --git a/test/mlir/test/gc/cpu-runner/GPU/xevm_block_load_store.mlir b/test/mlir/test/gc/cpu-runner/GPU/xevm_block_load_store.mlir deleted file mode 100644 index f4bb29f2a..000000000 --- a/test/mlir/test/gc/cpu-runner/GPU/xevm_block_load_store.mlir +++ /dev/null @@ -1,88 +0,0 @@ -// RUN: gc-opt %s --convert-xevm-to-llvm --xevm-attach-target --convert-scf-to-cf --convert-cf-to-llvm --convert-arith-to-llvm --convert-gpu-to-llvm-spv --gpu-to-llvm --reconcile-unrealized-casts --cse --gpu-module-to-binary | gc-cpu-runner -e main -entry-point-result=void --shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%opencl_runtime | FileCheck %s - -module @gemm attributes {gpu.container_module} { - - gpu.module @kernel { - // - `cl_intel_subgroups` block load/store intrinsics operate at the *maximum* sub-group size, - // regardless of the active sub-group size. Make sure `clGetKernelSubGroupInfo` meets your expectations. - // - The attribute `intel_reqd_sub_group_size` establishes the maximum sub-group size for a kernel. - // - // Note: launching 16 threads without explicit `intel_reqd_sub_group_size = 16` may still use - // the default sub-group size of 32. - // - // https://registry.khronos.org/OpenCL/extensions/intel/cl_intel_required_subgroup_size.html - // https://registry.khronos.org/OpenCL/extensions/intel/cl_intel_subgroups.html - - gpu.func @block_load_store(%src: !llvm.ptr<1>, %dst: !llvm.ptr<1>) kernel { - %base_width = arith.constant 64 : i32 // bytewidth of the block - %base_height = arith.constant 8 : i32 // number of rows - %base_pitch = arith.constant 64 : i32 // bytewidth of the base row - %x = arith.constant 0 : i32 - %y = arith.constant 0 : i32 - // If `intel_reqd_sub_group_size = 16` is not set, the default (32) is used and this `blockload2d` would only load 4 elements into vector<8xi32> - %loaded = xevm.blockload2d %src, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=32, tile_width=16, tile_height=8, v_blocks=1, transpose=false, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<8xi32> - %loaded_f32 = vector.bitcast %loaded : vector<8xi32> to vector<8xf32> - %c0 = arith.constant 0 : i32 - %thread_x = gpu.thread_id x - %thread_x_i32 = arith.index_cast %thread_x : index to i32 - %thread_x_f32 = arith.sitofp %thread_x_i32 : i32 to f32 - %loaded_f32_modified = vector.insertelement %thread_x_f32, %loaded_f32[%c0 : i32] : vector<8xf32> - %loaded_modified = vector.bitcast %loaded_f32_modified : vector<8xf32> to vector<8xi32> - xevm.blockstore2d %dst, %base_width, %base_height, %base_pitch, %x, %y, %loaded_modified {elem_size_in_bits=32, tile_width=16, tile_height=8, v_blocks=1, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32, vector<8xi32>) - gpu.return - } - } - - func.func @test(%src : memref<8x16xf32>) -> memref<8x16xf32> attributes {llvm.emit_c_interface} { - %c1 = arith.constant 1 : index - %c16 = arith.constant 16 : index // Multiple of the *maximum sub-group size* (see `intel_reqd_sub_group_size`) - %memref_src = gpu.alloc host_shared () : memref<8x16xf32> - memref.copy %src, %memref_src : memref<8x16xf32> to memref<8x16xf32> - %src_ptr_as_idx = memref.extract_aligned_pointer_as_index %memref_src : memref<8x16xf32> -> index - %src_ptr_as_i64 = arith.index_cast %src_ptr_as_idx : index to i64 - %src_ptr = llvm.inttoptr %src_ptr_as_i64 : i64 to !llvm.ptr - %src_ptr_casted = llvm.addrspacecast %src_ptr : !llvm.ptr to !llvm.ptr<1> - - %memref_dst = gpu.alloc host_shared () : memref<8x16xf32> - %dst_ptr_as_idx = memref.extract_aligned_pointer_as_index %memref_dst : memref<8x16xf32> -> index - %dst_ptr_as_i64 = arith.index_cast %dst_ptr_as_idx : index to i64 - %dst_ptr = llvm.inttoptr %dst_ptr_as_i64 : i64 to !llvm.ptr - %dst_ptr_casted = llvm.addrspacecast %dst_ptr : !llvm.ptr to !llvm.ptr<1> - - gpu.launch_func @kernel::@block_load_store blocks in (%c1, %c1, %c1) threads in (%c16, %c1, %c1) args(%src_ptr_casted : !llvm.ptr<1>, %dst_ptr_casted : !llvm.ptr<1>) - return %memref_dst : memref<8x16xf32> - } - - func.func @main() attributes {llvm.emit_c_interface} { - %A = memref.alloc() : memref<8x16xf32> - %c0 = arith.constant 0 : index - %c1 = arith.constant 1 : index - %c8 = arith.constant 8 : index - %c16 = arith.constant 16 : index - %c11_f32 = arith.constant 11.11 : f32 - scf.for %i = %c0 to %c8 step %c1 { - scf.for %j = %c0 to %c16 step %c1 { - memref.store %c11_f32, %A[%i, %j] : memref<8x16xf32> - } - } - %B = call @test(%A) : (memref<8x16xf32>) -> memref<8x16xf32> - %B_cast = memref.cast %B : memref<8x16xf32> to memref<*xf32> - %A_cast = memref.cast %A : memref<8x16xf32> to memref<*xf32> - call @printMemrefF32(%A_cast) : (memref<*xf32>) -> () - call @printMemrefF32(%B_cast) : (memref<*xf32>) -> () - - // CHECK: Unranked Memref base@ = 0x{{[0-9a-f]+}} - // CHECK-NEXT: [11.11{{.*}}] - // CHECK-COUNT-96: 11.11 - // CHECK-NEXT: [11.11{{.*}}] - - // CHECK-NEXT: Unranked Memref base@ = 0x{{[0-9a-f]+}} - // CHECK: 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 - // CHECK-COUNT-96: 11.11 - // CHECK-NEXT: [11.11{{.*}}] - - memref.dealloc %A : memref<8x16xf32> - return - } - func.func private @printMemrefF32(%ptr : memref<*xf32>) -} diff --git a/test/mlir/test/gc/cpu-runner/brgemm-multilevel-for.mlir b/test/mlir/test/gc/cpu-runner/brgemm-multilevel-for.mlir deleted file mode 100644 index 759f22776..000000000 --- a/test/mlir/test/gc/cpu-runner/brgemm-multilevel-for.mlir +++ /dev/null @@ -1,148 +0,0 @@ -// RUN: gc-opt %s --early-dispatch-microkernel --convert-microkernel-to-dnnl-func --cse --microkernel-invariant-code-motion --convert-linalg-to-loops --convert-scf-to-cf --expand-strided-metadata --lower-affine -finalize-memref-to-llvm --convert-cpuruntime-to-llvm --convert-func-to-llvm --convert-arith-to-llvm --convert-cf-to-llvm --convert-complex-to-llvm --canonicalize --cse --reconcile-unrealized-casts --symbol-dce | gc-cpu-runner -e main -entry-point-result=void - -#map = affine_map<(d0, d1) -> (d0, d1)> -module { - func.func @simple_brgemm() { - %c0_i64 = arith.constant 0 : i64 - %c2_i64 = arith.constant 2 : i64 - %c0_index = arith.constant 0 : index - %c1_index = arith.constant 1 : index - %c4_index = arith.constant 4 : index - %c8_index = arith.constant 8 : index - %c32_index = arith.constant 32 : index - %c16_i64 = arith.constant 16 : i64 - %cst0f = arith.constant 0.000000e+00 : f32 - %cstn64f = arith.constant -64.000000e+00 : f32 - %cst1f = arith.constant 1.000000e+00 : bf16 - %cst2f = arith.constant 2.000000e+00 : bf16 - %cst3f = arith.constant 3.000000e+00 : f32 - %alloc = memref.alloc() {alignment = 64 : i64} : memref<4x16x32x32xbf16> - linalg.fill ins(%cst1f : bf16) outs(%alloc : memref<4x16x32x32xbf16>) - %alloc_0 = memref.alloc() {alignment = 64 : i64} : memref<8x16x16x32x2xbf16> - linalg.fill ins(%cst2f : bf16) outs(%alloc_0 : memref<8x16x16x32x2xbf16>) - %alloc_1 = memref.alloc() {alignment = 64 : i64} : memref<4x8x32x32xf32> - linalg.fill ins(%cst3f : f32) outs(%alloc_1 : memref<4x8x32x32xf32>) - %alloc_2 = memref.alloc() {alignment = 64 : i64} : memref<4x8x32x32xf32> - linalg.fill ins(%cst0f : f32) outs(%alloc_2 : memref<4x8x32x32xf32>) - scf.for %arg0 = %c0_index to %c4_index step %c1_index { - scf.for %arg1 = %c0_index to %c8_index step %c1_index { - %alloc_3 = memref.alloc() {alignment = 64 : i64} : memref<32x32xf32> - %arg0i = arith.index_castui %arg0 : index to i64 - %arg1i = arith.index_castui %arg1 : index to i64 - %argmulti = arith.muli %arg0i, %arg1i : i64 - %v = arith.uitofp %argmulti : i64 to f32 - %v1 = arith.mulf %v, %cstn64f : f32 - linalg.fill ins(%v1 : f32) outs(%alloc_3 : memref<32x32xf32>) - %subview = memref.subview %alloc[%arg0, 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : memref<4x16x32x32xbf16> to memref<16x32x32xbf16, strided<[1024, 32, 1], offset: ?>> - %subview_4 = memref.subview %alloc_0[%arg1, 0, 0, 0, 0] [1, 16, 16, 32, 2] [1, 1, 1, 1, 1] : memref<8x16x16x32x2xbf16> to memref<16x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>> - %0 = microkernel.brgemm.dispatch [32, 32, 32, 32, 32, 32, 1024, 1024] flags(stride) data_type(bf16, bf16) - microkernel.brgemm.prologue(%0) : (i64) -> () - microkernel.brgemm.execute(%0, %subview, %subview_4, %alloc_3, %c16_i64, %c0_i64) : (i64, memref<16x32x32xbf16, strided<[1024, 32, 1], offset: ?>>, memref<16x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>>, memref<32x32xf32>, i64, i64) -> () - microkernel.brgemm.epilogue(%0) : (i64) -> () - %subview_5 = memref.subview %alloc_1[%arg0, %arg1, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : memref<4x8x32x32xf32> to memref<32x32xf32, strided<[32, 1], offset: ?>> - linalg.generic {indexing_maps = [#map, #map, #map], iterator_types = ["parallel", "parallel"]} ins(%alloc_3, %subview_5 : memref<32x32xf32>, memref<32x32xf32, strided<[32, 1], offset: ?>>) outs(%alloc_3 : memref<32x32xf32>) { - ^bb0(%in: f32, %in_7: f32, %out: f32): - %1 = arith.addf %in, %in_7 : f32 - linalg.yield %1 : f32 - } - %subview_6 = memref.subview %alloc_2[%arg0, %arg1, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : memref<4x8x32x32xf32> to memref<32x32xf32, strided<[32, 1], offset: ?>> - linalg.generic {indexing_maps = [#map, #map], iterator_types = ["parallel", "parallel"]} ins(%alloc_3 : memref<32x32xf32>) outs(%subview_6 : memref<32x32xf32, strided<[32, 1], offset: ?>>) { - ^bb0(%in: f32, %out: f32): - %1 = arith.maximumf %in, %cst0f : f32 - linalg.yield %1 : f32 - } - memref.dealloc %alloc_3 : memref<32x32xf32> - } - %subview_c = memref.subview %alloc_2[%arg0, 0, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : memref<4x8x32x32xf32> to memref<32x32xf32, strided<[32, 1], offset: ?>> - %subview_7 = memref.subview %alloc[%arg0, 0, 0, 0] [1, 2, 32, 32] [1, 1, 1, 1] : memref<4x16x32x32xbf16> to memref<2x32x32xbf16, strided<[1024, 32, 1], offset: ?>> - %subview_8 = memref.subview %alloc_0[%arg0, 0, 0, 0, 0] [1, 2, 16, 32, 2] [1, 1, 1, 1, 1] : memref<8x16x16x32x2xbf16> to memref<2x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>> - %10 = microkernel.brgemm.dispatch [32, 32, 32, 32, 32, 32, 1024, 1024] flags(stride) data_type(bf16, bf16) - microkernel.brgemm.prologue(%10) : (i64) -> () - microkernel.brgemm.execute(%10, %subview_7, %subview_8, %subview_c, %c2_i64, %c0_i64) : (i64, memref<2x32x32xbf16, strided<[1024, 32, 1], offset: ?>>, memref<2x16x32x2xbf16, strided<[1024, 64, 2, 1], offset: ?>>, memref<32x32xf32, strided<[32, 1], offset: ?>>, i64, i64) -> () - microkernel.brgemm.epilogue(%10) : (i64) -> () - } - scf.for %arg0 = %c0_index to %c4_index step %c1_index { - scf.for %arg1 = %c0_index to %c8_index step %c1_index { - scf.for %arg2 = %c0_index to %c32_index step %c1_index { - scf.for %arg3 = %c0_index to %c32_index step %c1_index { - %elem = memref.load %alloc_2[%arg0, %arg1, %arg2, %arg3] : memref<4x8x32x32xf32> - cpuruntime.printf "%f, " %elem : f32 - } - cpuruntime.printf "\n" - } - cpuruntime.printf "==================================\n" - } - } - return - } - - func.func @main() { - call @simple_brgemm() : ()->() - cpuruntime.printf "BRGEMM DONE\n" - return - } - - // CHECK: 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, - // CHECK: ================================== - // CHECK: 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, - // CHECK: ================================== - // CHECK: 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, - // CHECK: ================================== - // CHECK: 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, - // CHECK: ================================== - // CHECK: 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, - // CHECK: ================================== - // CHECK: 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, - // CHECK: ================================== - // CHECK: 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, - // CHECK: ================================== - // CHECK: 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, - // CHECK: ================================== - // CHECK: 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, - // CHECK: ================================== - // CHECK: 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, - // CHECK: ================================== - // CHECK: 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, - // CHECK: ================================== - // CHECK: 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, - // CHECK: ================================== - // CHECK: 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, - // CHECK: ================================== - // CHECK: 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, - // CHECK: ================================== - // CHECK: 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, - // CHECK: ================================== - // CHECK: 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, - // CHECK: ================================== - // CHECK: 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, - // CHECK: ================================== - // CHECK: 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, - // CHECK: ================================== - // CHECK: 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, - // CHECK: ================================== - // CHECK: 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, - // CHECK: ================================== - // CHECK: 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, - // CHECK: ================================== - // CHECK: 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, - // CHECK: ================================== - // CHECK: 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, - // CHECK: ================================== - // CHECK: 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, 1155.000000, - // CHECK: ================================== - // CHECK: 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, - // CHECK: ================================== - // CHECK: 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, - // CHECK: ================================== - // CHECK: 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, - // CHECK: ================================== - // CHECK: 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, - // CHECK: ================================== - // CHECK: 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, - // CHECK: ================================== - // CHECK: 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, - // CHECK: ================================== - // CHECK: 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, - // CHECK: ================================== - // CHECK: BRGEMM DONE -} diff --git a/test/mlir/test/gc/cpu-runner/brgemm-parallel.mlir b/test/mlir/test/gc/cpu-runner/brgemm-parallel.mlir deleted file mode 100644 index 38cf0dbf0..000000000 --- a/test/mlir/test/gc/cpu-runner/brgemm-parallel.mlir +++ /dev/null @@ -1,138 +0,0 @@ -// RUN: gc-opt %s --early-dispatch-microkernel --convert-microkernel-to-dnnl-func --cse --microkernel-invariant-code-motion --convert-linalg-to-loops --convert-scf-to-cf --expand-strided-metadata --lower-affine -finalize-memref-to-llvm --convert-cpuruntime-to-llvm --convert-func-to-llvm --convert-arith-to-llvm --convert-cf-to-llvm --convert-complex-to-llvm --canonicalize --cse --reconcile-unrealized-casts --symbol-dce | gc-cpu-runner -e main -entry-point-result=void - -#map = affine_map<(d0, d1) -> (d0, d1)> -module { - func.func @simple_brgemm() { - %c0_i64 = arith.constant 0 : i64 - %c16_i64 = arith.constant 16 : i64 - %c0_index = arith.constant 0 : index - %c1_index = arith.constant 1 : index - %c4_index = arith.constant 4 : index - %c8_index = arith.constant 8 : index - %c32_index = arith.constant 32 : index - %cst0f = arith.constant 0.000000e+00 : f32 - %cstn64f = arith.constant -64.000000e+00 : f32 - %cst1f = arith.constant 1.000000e+00 : f32 - %cst2f = arith.constant 2.000000e+00 : f32 - %cst3f = arith.constant 3.000000e+00 : f32 - %alloc = memref.alloc() {alignment = 64 : i64} : memref<4x16x32x32xf32> - linalg.fill ins(%cst1f : f32) outs(%alloc : memref<4x16x32x32xf32>) - %alloc_0 = memref.alloc() {alignment = 64 : i64} : memref<8x16x32x32xf32> - linalg.fill ins(%cst2f : f32) outs(%alloc_0 : memref<8x16x32x32xf32>) - %alloc_1 = memref.alloc() {alignment = 64 : i64} : memref<4x8x32x32xf32> - linalg.fill ins(%cst3f : f32) outs(%alloc_1 : memref<4x8x32x32xf32>) - %alloc_2 = memref.alloc() {alignment = 64 : i64} : memref<4x8x32x32xf32> - linalg.fill ins(%cst0f : f32) outs(%alloc_2 : memref<4x8x32x32xf32>) - scf.forall (%arg0, %arg1) in (4, 8) { - %alloc_3 = memref.alloc() {alignment = 64 : i64} : memref<32x32xf32> - %arg0i = arith.index_castui %arg0 : index to i64 - %arg1i = arith.index_castui %arg1 : index to i64 - %argmulti = arith.muli %arg0i, %arg1i : i64 - %v = arith.uitofp %argmulti : i64 to f32 - %v1 = arith.mulf %v, %cstn64f : f32 - linalg.fill ins(%v1 : f32) outs(%alloc_3 : memref<32x32xf32>) - %subview = memref.subview %alloc[%arg0, 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : memref<4x16x32x32xf32> to memref<16x32x32xf32, strided<[1024, 32, 1], offset: ?>> - %subview_4 = memref.subview %alloc_0[%arg1, 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : memref<8x16x32x32xf32> to memref<16x32x32xf32, strided<[1024, 32, 1], offset: ?>> - %0 = microkernel.brgemm.dispatch [32, 32, 32, 32, 32, 32, 1024, 1024] flags(stride) data_type(f32, f32) - microkernel.brgemm.prologue(%0) : (i64) -> () - microkernel.brgemm.execute(%0, %subview, %subview_4, %alloc_3, %c16_i64, %c0_i64) : (i64, memref<16x32x32xf32, strided<[1024, 32, 1], offset: ?>>, memref<16x32x32xf32, strided<[1024, 32, 1], offset: ?>>, memref<32x32xf32>, i64, i64) -> () - microkernel.brgemm.epilogue(%0) : (i64) -> () - %subview_5 = memref.subview %alloc_1[%arg0, %arg1, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : memref<4x8x32x32xf32> to memref<32x32xf32, strided<[32, 1], offset: ?>> - linalg.generic {indexing_maps = [#map, #map, #map], iterator_types = ["parallel", "parallel"]} ins(%alloc_3, %subview_5 : memref<32x32xf32>, memref<32x32xf32, strided<[32, 1], offset: ?>>) outs(%alloc_3 : memref<32x32xf32>) { - ^bb0(%in: f32, %in_7: f32, %out: f32): - %1 = arith.addf %in, %in_7 : f32 - linalg.yield %1 : f32 - } - %subview_6 = memref.subview %alloc_2[%arg0, %arg1, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : memref<4x8x32x32xf32> to memref<32x32xf32, strided<[32, 1], offset: ?>> - linalg.generic {indexing_maps = [#map, #map], iterator_types = ["parallel", "parallel"]} ins(%alloc_3 : memref<32x32xf32>) outs(%subview_6 : memref<32x32xf32, strided<[32, 1], offset: ?>>) { - ^bb0(%in: f32, %out: f32): - %1 = arith.maximumf %in, %cst0f : f32 - linalg.yield %1 : f32 - } - memref.dealloc %alloc_3 : memref<32x32xf32> - } - scf.for %arg0 = %c0_index to %c4_index step %c1_index { - scf.for %arg1 = %c0_index to %c8_index step %c1_index { - scf.for %arg2 = %c0_index to %c32_index step %c1_index { - scf.for %arg3 = %c0_index to %c32_index step %c1_index { - %elem = memref.load %alloc_2[%arg0, %arg1, %arg2, %arg3] : memref<4x8x32x32xf32> - cpuruntime.printf "%f, " %elem : f32 - } - cpuruntime.printf "\n" - } - cpuruntime.printf "==================================\n" - } - } - return - } - - func.func @main() { - call @simple_brgemm() : ()->() - // CHECK: 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, - // CHECK: ================================== - // CHECK: 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, - // CHECK: ================================== - // CHECK: 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, - // CHECK: ================================== - // CHECK: 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, - // CHECK: ================================== - // CHECK: 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, - // CHECK: ================================== - // CHECK: 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, - // CHECK: ================================== - // CHECK: 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, - // CHECK: ================================== - // CHECK: 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, - // CHECK: ================================== - // CHECK: 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, - // CHECK: ================================== - // CHECK: 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, - // CHECK: ================================== - // CHECK: 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, - // CHECK: ================================== - // CHECK: 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, - // CHECK: ================================== - // CHECK: 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, - // CHECK: ================================== - // CHECK: 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, - // CHECK: ================================== - // CHECK: 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, - // CHECK: ================================== - // CHECK: 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, - // CHECK: ================================== - // CHECK: 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, - // CHECK: ================================== - // CHECK: 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, - // CHECK: ================================== - // CHECK: 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, - // CHECK: ================================== - // CHECK: 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, - // CHECK: ================================== - // CHECK: 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, - // CHECK: ================================== - // CHECK: 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, - // CHECK: ================================== - // CHECK: 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, - // CHECK: ================================== - // CHECK: 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, - // CHECK: ================================== - // CHECK: 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, - // CHECK: ================================== - // CHECK: 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, - // CHECK: ================================== - // CHECK: 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, - // CHECK: ================================== - // CHECK: 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, - // CHECK: ================================== - // CHECK: 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, - // CHECK: ================================== - // CHECK: 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, - // CHECK: ================================== - // CHECK: 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, - // CHECK: ================================== - cpuruntime.printf "BRGEMM DONE\n" - return - } - - // CHECK: BRGEMM DONE -} diff --git a/test/mlir/test/gc/cpu-runner/brgemm-simple-for.mlir b/test/mlir/test/gc/cpu-runner/brgemm-simple-for.mlir deleted file mode 100644 index b7f3ca6a7..000000000 --- a/test/mlir/test/gc/cpu-runner/brgemm-simple-for.mlir +++ /dev/null @@ -1,140 +0,0 @@ -// RUN: gc-opt %s --early-dispatch-microkernel --convert-microkernel-to-dnnl-func --cse --microkernel-invariant-code-motion --convert-linalg-to-loops --convert-scf-to-cf --expand-strided-metadata --lower-affine -finalize-memref-to-llvm --convert-cpuruntime-to-llvm --convert-func-to-llvm --convert-arith-to-llvm --convert-cf-to-llvm --convert-complex-to-llvm --canonicalize --cse --reconcile-unrealized-casts --symbol-dce | gc-cpu-runner -e main -entry-point-result=void - -#map = affine_map<(d0, d1) -> (d0, d1)> -module { - func.func @simple_brgemm() { - %c0_index = arith.constant 0 : index - %c1_index = arith.constant 1 : index - %c4_index = arith.constant 4 : index - %c8_index = arith.constant 8 : index - %c32_index = arith.constant 32 : index - %c0_i64 = arith.constant 0 : i64 - %c16_i64 = arith.constant 16 : i64 - %cst0f = arith.constant 0.000000e+00 : f32 - %cstn64f = arith.constant -64.000000e+00 : f32 - %cst1f = arith.constant 1.000000e+00 : f32 - %cst2f = arith.constant 2.000000e+00 : f32 - %cst3f = arith.constant 3.000000e+00 : f32 - %alloc = memref.alloc() {alignment = 64 : i64} : memref<4x16x32x32xf32> - linalg.fill ins(%cst1f : f32) outs(%alloc : memref<4x16x32x32xf32>) - %alloc_0 = memref.alloc() {alignment = 64 : i64} : memref<8x16x32x32xf32> - linalg.fill ins(%cst2f : f32) outs(%alloc_0 : memref<8x16x32x32xf32>) - %alloc_1 = memref.alloc() {alignment = 64 : i64} : memref<4x8x32x32xf32> - linalg.fill ins(%cst3f : f32) outs(%alloc_1 : memref<4x8x32x32xf32>) - %alloc_2 = memref.alloc() {alignment = 64 : i64} : memref<4x8x32x32xf32> - linalg.fill ins(%cst0f : f32) outs(%alloc_2 : memref<4x8x32x32xf32>) - scf.for %arg0 = %c0_index to %c4_index step %c1_index { - scf.for %arg1 = %c0_index to %c8_index step %c1_index { - %alloc_3 = memref.alloc() {alignment = 64 : i64} : memref<32x32xf32> - %arg0i = arith.index_castui %arg0 : index to i64 - %arg1i = arith.index_castui %arg1 : index to i64 - %argmulti = arith.muli %arg0i, %arg1i : i64 - %v = arith.uitofp %argmulti : i64 to f32 - %v1 = arith.mulf %v, %cstn64f : f32 - linalg.fill ins(%v1 : f32) outs(%alloc_3 : memref<32x32xf32>) - %subview = memref.subview %alloc[%arg0, 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : memref<4x16x32x32xf32> to memref<16x32x32xf32, strided<[1024, 32, 1], offset: ?>> - %subview_4 = memref.subview %alloc_0[%arg1, 0, 0, 0] [1, 16, 32, 32] [1, 1, 1, 1] : memref<8x16x32x32xf32> to memref<16x32x32xf32, strided<[1024, 32, 1], offset: ?>> - %0 = microkernel.brgemm.dispatch [32, 32, 32, 32, 32, 32, 1024, 1024] flags(stride) data_type(f32, f32) - microkernel.brgemm.prologue(%0) : (i64) -> () - microkernel.brgemm.execute(%0, %subview, %subview_4, %alloc_3, %c16_i64, %c0_i64) : (i64, memref<16x32x32xf32, strided<[1024, 32, 1], offset: ?>>, memref<16x32x32xf32, strided<[1024, 32, 1], offset: ?>>, memref<32x32xf32>, i64, i64) -> () - microkernel.brgemm.epilogue(%0) : (i64) -> () - %subview_5 = memref.subview %alloc_1[%arg0, %arg1, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : memref<4x8x32x32xf32> to memref<32x32xf32, strided<[32, 1], offset: ?>> - linalg.generic {indexing_maps = [#map, #map, #map], iterator_types = ["parallel", "parallel"]} ins(%alloc_3, %subview_5 : memref<32x32xf32>, memref<32x32xf32, strided<[32, 1], offset: ?>>) outs(%alloc_3 : memref<32x32xf32>) { - ^bb0(%in: f32, %in_7: f32, %out: f32): - %1 = arith.addf %in, %in_7 : f32 - linalg.yield %1 : f32 - } - %subview_6 = memref.subview %alloc_2[%arg0, %arg1, 0, 0] [1, 1, 32, 32] [1, 1, 1, 1] : memref<4x8x32x32xf32> to memref<32x32xf32, strided<[32, 1], offset: ?>> - linalg.generic {indexing_maps = [#map, #map], iterator_types = ["parallel", "parallel"]} ins(%alloc_3 : memref<32x32xf32>) outs(%subview_6 : memref<32x32xf32, strided<[32, 1], offset: ?>>) { - ^bb0(%in: f32, %out: f32): - %1 = arith.maximumf %in, %cst0f : f32 - linalg.yield %1 : f32 - } - memref.dealloc %alloc_3 : memref<32x32xf32> - } - } - scf.for %arg0 = %c0_index to %c4_index step %c1_index { - scf.for %arg1 = %c0_index to %c8_index step %c1_index { - scf.for %arg2 = %c0_index to %c32_index step %c1_index { - scf.for %arg3 = %c0_index to %c32_index step %c1_index { - %elem = memref.load %alloc_2[%arg0, %arg1, %arg2, %arg3] : memref<4x8x32x32xf32> - cpuruntime.printf "%f, " %elem : f32 - } - cpuruntime.printf "\n" - } - cpuruntime.printf "==================================\n" - } - } - return - } - - func.func @main() { - call @simple_brgemm() : ()->() - cpuruntime.printf "BRGEMM DONE\n" - return - } - - // CHECK: 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, - // CHECK: ================================== - // CHECK: 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, - // CHECK: ================================== - // CHECK: 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, - // CHECK: ================================== - // CHECK: 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, - // CHECK: ================================== - // CHECK: 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, - // CHECK: ================================== - // CHECK: 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, - // CHECK: ================================== - // CHECK: 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, - // CHECK: ================================== - // CHECK: 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, - // CHECK: ================================== - // CHECK: 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, - // CHECK: ================================== - // CHECK: 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, 963.000000, - // CHECK: ================================== - // CHECK: 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, - // CHECK: ================================== - // CHECK: 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, - // CHECK: ================================== - // CHECK: 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, 707.000000, - // CHECK: ================================== - // CHECK: 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, - // CHECK: ================================== - // CHECK: 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, 579.000000, - // CHECK: ================================== - // CHECK: 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, - // CHECK: ================================== - // CHECK: 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, 899.000000, - // CHECK: ================================== - // CHECK: 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, 771.000000, - // CHECK: ================================== - // CHECK: 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, - // CHECK: ================================== - // CHECK: 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, 515.000000, - // CHECK: ================================== - // CHECK: 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, 387.000000, - // CHECK: ================================== - // CHECK: 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, - // CHECK: ================================== - // CHECK: 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, 131.000000, - // CHECK: ================================== - // CHECK: 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, 1027.000000, - // CHECK: ================================== - // CHECK: 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, 835.000000, - // CHECK: ================================== - // CHECK: 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, 643.000000, - // CHECK: ================================== - // CHECK: 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, 451.000000, - // CHECK: ================================== - // CHECK: 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, 259.000000, - // CHECK: ================================== - // CHECK: 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, 67.000000, - // CHECK: ================================== - // CHECK: 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, - // CHECK: ================================== - // CHECK: 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, - // CHECK: ================================== - // CHECK: BRGEMM DONE -} diff --git a/test/mlir/test/gc/cpu-runner/run.mlir b/test/mlir/test/gc/cpu-runner/run.mlir deleted file mode 100644 index 406bf68d9..000000000 --- a/test/mlir/test/gc/cpu-runner/run.mlir +++ /dev/null @@ -1,5 +0,0 @@ -// RUN: gc-opt %s --convert-func-to-llvm --convert-cf-to-llvm | gc-cpu-runner -e main -entry-point-result=void | FileCheck --allow-empty %s -func.func @main() { - return -} -// CHECK-NOT: any \ No newline at end of file diff --git a/test/mlir/test/gc/cpu-runner/tid.mlir b/test/mlir/test/gc/cpu-runner/tid.mlir deleted file mode 100644 index aedcc0a20..000000000 --- a/test/mlir/test/gc/cpu-runner/tid.mlir +++ /dev/null @@ -1,37 +0,0 @@ -// RUN: gc-opt %s --convert-cpuruntime-to-llvm --convert-openmp-to-llvm --convert-func-to-llvm --convert-arith-to-llvm --convert-cf-to-llvm --reconcile-unrealized-casts | gc-cpu-runner -e main -entry-point-result=void | FileCheck %s -module { - func.func private @omp_get_thread_num() -> i32 - - func.func @check_parallel() { - %c64 = arith.constant 64 : index - %c1 = arith.constant 1 : index - %c0 = arith.constant 0 : index - %c8 = arith.constant 8 : index - %0 = llvm.mlir.constant(1 : i64) : i64 - omp.parallel num_threads(%c8: index) { - omp.wsloop { - omp.loop_nest (%arg1, %arg2) : index = (%c0, %c0) to (%c1, %c64) step (%c1, %c1) { - cpuruntime.printf "ITR %zu\n" %arg2 : index - omp.yield - } - omp.terminator - } - %tid = func.call @omp_get_thread_num() : () -> i32 - cpuruntime.printf "EXIT %d\n" %tid : i32 - omp.terminator - } - return - } - - func.func @main() { - %0 = func.call @omp_get_thread_num() : () -> i32 - cpuruntime.printf "TID %d\n" %0 : i32 - call @check_parallel() : ()->() - return - } - // CHECK: TID 0 - // CHECK-COUNT-64: ITR {{[0-9]+}} - // CHECK-NOT: ITR - // CHECK-COUNT-8: EXIT {{[0-9]+}} - // CHECK-NOT: EXIT -} \ No newline at end of file diff --git a/test/mlir/test/gc/gpu-runner/IMEX/XeGPU/f16_matmul_128x64_transpose.mlir b/test/mlir/test/gc/gpu-runner/IMEX/XeGPU/f16_matmul_128x64_transpose.mlir deleted file mode 100644 index 1302d6809..000000000 --- a/test/mlir/test/gc/gpu-runner/IMEX/XeGPU/f16_matmul_128x64_transpose.mlir +++ /dev/null @@ -1,32 +0,0 @@ -// RUN: gc-gpu-runner --shared-libs=%mlir_runner_utils %s | FileCheck %s - -module{ - -func.func @linalg_matmul(%arg0: tensor<128x256xf16>, - %arg1: tensor<64x256xf16>, - %arg2: tensor<128x64xf16>) -> tensor<128x64xf16> { - %0 = linalg.matmul_transpose_b ins(%arg0, %arg1 : tensor<128x256xf16>, tensor<64x256xf16>) - outs(%arg2 : tensor<128x64xf16>) -> tensor<128x64xf16> - return %0 : tensor<128x64xf16> -} - -func.func @main() { - %0 = arith.constant dense<0.1> : tensor<128x256xf16> - %1 = arith.constant dense<0.2> : tensor<64x256xf16> - %2 = arith.constant dense<0.0> : tensor<128x64xf16> - %gpu_res = call @linalg_matmul(%0, %1, %2) : (tensor<128x256xf16>, tensor<64x256xf16>, tensor<128x64xf16>) -> tensor<128x64xf16> - - %slice = tensor.extract_slice %gpu_res[0, 0][32, 1][1, 1] : tensor<128x64xf16> to tensor<32xf16> - %cast = tensor.cast %slice : tensor<32xf16> to tensor<*xf16> - call @printMemrefF16(%cast) : (tensor<*xf16>) -> () - - return -} - -func.func private @printMemrefF16(%ptr : tensor<*xf16>) -} - -// CHECK: Unranked Memref base@{{(0x)?[-0-9a-fA-F]*}} -// CHECK-SAME: rank = 1 offset = 0 sizes = [32] strides = [64] data = -// Computed using numpy: -// CHECK-NEXT: [5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719] diff --git a/test/mlir/test/gc/gpu-runner/IMEX/XeGPU/f16_matmul_128x64_transpose_sep.mlir b/test/mlir/test/gc/gpu-runner/IMEX/XeGPU/f16_matmul_128x64_transpose_sep.mlir deleted file mode 100644 index 0a06bda8e..000000000 --- a/test/mlir/test/gc/gpu-runner/IMEX/XeGPU/f16_matmul_128x64_transpose_sep.mlir +++ /dev/null @@ -1,33 +0,0 @@ -// RUN: gc-gpu-runner --shared-libs=%mlir_runner_utils %s | FileCheck %s - -module{ - -func.func @linalg_matmul(%arg0: tensor<128x256xf16>, - %arg1: tensor<64x256xf16>, - %arg2: tensor<128x64xf16>) -> tensor<128x64xf16> { - %t = tensor.empty() : tensor<256x64xf16> - %0 = linalg.transpose ins(%arg1 : tensor<64x256xf16>) outs(%t: tensor<256x64xf16>) permutation = [1, 0] - %1 = linalg.matmul ins(%arg0, %0 : tensor<128x256xf16>, tensor<256x64xf16>) - outs(%arg2 : tensor<128x64xf16>) -> tensor<128x64xf16> - return %1 : tensor<128x64xf16> -} - -func.func @main() { - %0 = arith.constant dense<0.1> : tensor<128x256xf16> - %1 = arith.constant dense<0.2> : tensor<64x256xf16> - %2 = arith.constant dense<0.0> : tensor<128x64xf16> - %gpu_res = call @linalg_matmul(%0, %1, %2) : (tensor<128x256xf16>, tensor<64x256xf16>, tensor<128x64xf16>) -> tensor<128x64xf16> - - %slice = tensor.extract_slice %gpu_res[0, 0][32, 1][1, 1] : tensor<128x64xf16> to tensor<32xf16> - %cast = tensor.cast %slice : tensor<32xf16> to tensor<*xf16> - call @printMemrefF16(%cast) : (tensor<*xf16>) -> () - return -} - -func.func private @printMemrefF16(%ptr : tensor<*xf16>) -} - -// CHECK: Unranked Memref base@{{(0x)?[-0-9a-fA-F]*}} -// CHECK-SAME: rank = 1 offset = 0 sizes = [32] strides = [64] data = -// Computed using numpy: -// CHECK-NEXT: [5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719, 5.11719] diff --git a/test/mlir/test/gc/gpu-runner/IMEX/XeGPU/f16_matmul_64x128_slm.mlir b/test/mlir/test/gc/gpu-runner/IMEX/XeGPU/f16_matmul_64x128_slm.mlir deleted file mode 100644 index 4cc15c911..000000000 --- a/test/mlir/test/gc/gpu-runner/IMEX/XeGPU/f16_matmul_64x128_slm.mlir +++ /dev/null @@ -1,96 +0,0 @@ -// RUN: gc-gpu-runner --shared-libs=%mlir_runner_utils %s | FileCheck %s - -module @fragment_name { - // This kernel requires using SLM - func.func @entry(%0: tensor<64x128xf16>, %1: tensor<128x128xf16>, %2: tensor<64x128xf16>, %res: tensor<64x128xf16>) -> tensor<64x128xf16> { - %3 = tensor.empty() : tensor<128x128xf16> - %4 = tensor.empty() : tensor<64x128xf16> - %cst = arith.constant 0.000000e+00 : f16 - %5 = linalg.fill ins(%cst : f16) outs(%4 : tensor<64x128xf16>) -> tensor<64x128xf16> - %6 = linalg.matmul ins(%0, %1 : tensor<64x128xf16>, tensor<128x128xf16>) outs(%5 : tensor<64x128xf16>) -> tensor<64x128xf16> - %7 = tensor.empty() : tensor<64x128xf16> - %8 = linalg.add ins(%6, %2 : tensor<64x128xf16>, tensor<64x128xf16>) outs(%7 : tensor<64x128xf16>) -> tensor<64x128xf16> - %9 = tensor.empty() : tensor<64x128xf16> - %cst_0 = arith.constant 0.000000e+00 : f16 - %10 = linalg.fill ins(%cst_0 : f16) outs(%9 : tensor<64x128xf16>) -> tensor<64x128xf16> - %11 = linalg.max ins(%8, %10 : tensor<64x128xf16>, tensor<64x128xf16>) outs(%res : tensor<64x128xf16>) -> tensor<64x128xf16> - return %11 : tensor<64x128xf16> - } - - func.func @get_value(%i : index, %j : index, %even_val : f16, %odd_val : f16) -> f16 { - %int0 = arith.index_cast %i : index to i32 - - %c2 = arith.constant 2 : i32 - %remeinder = arith.remui %int0, %c2 : i32 - %c0i = arith.constant 0 : i32 - %is_even = arith.cmpi eq, %remeinder, %c0i : i32 - - %val = scf.if %is_even -> (f16) { - scf.yield %even_val : f16 - } else { - scf.yield %odd_val : f16 - } - return %val : f16 - } - - // generates asymmetric tensor - func.func @generate_t(%even_val : f16, %odd_val : f16) -> tensor<64x128xf16> { - %0 = tensor.generate { - ^bb0(%i : index, %j : index): - %val = func.call @get_value(%i, %j, %even_val, %odd_val) : (index, index, f16, f16) -> f16 - tensor.yield %val : f16 - } : tensor<64x128xf16> - return %0 : tensor<64x128xf16> - } - - func.func @generate_t_wide(%even_val : f16, %odd_val : f16) -> tensor<128x128xf16> { - %0 = tensor.generate { - ^bb0(%i : index, %j : index): - %val = func.call @get_value(%i, %j, %even_val, %odd_val) : (index, index, f16, f16) -> f16 - tensor.yield %val : f16 - } : tensor<128x128xf16> - return %0 : tensor<128x128xf16> - } - - func.func @main() { - %a0 = arith.constant 0.1 : f16 - %b0 = arith.constant 0.2 : f16 - %0 = call @generate_t(%a0, %b0) : (f16, f16) -> tensor<64x128xf16> - - %a1 = arith.constant 0.3 : f16 - %b1 = arith.constant 0.4 : f16 - %1 = call @generate_t_wide(%a1, %b1) : (f16, f16) -> tensor<128x128xf16> - - %a2 = arith.constant 0.5 : f16 - %b2 = arith.constant 0.6 : f16 - %2 = call @generate_t(%a2, %b2) : (f16, f16) -> tensor<64x128xf16> - - %3 = arith.constant dense<0.0> : tensor<64x128xf16> - %gpu_res = call @entry(%0, %1, %2, %3) : (tensor<64x128xf16>, tensor<128x128xf16>, tensor<64x128xf16>, tensor<64x128xf16>) -> (tensor<64x128xf16>) - %slice = tensor.extract_slice %gpu_res[0, 0][16, 16][1, 1] : tensor<64x128xf16> to tensor<16x16xf16> - %cast = tensor.cast %slice : tensor<16x16xf16> to tensor<*xf16> - call @printMemrefF16(%cast) : (tensor<*xf16>) -> () - return -} - -func.func private @printMemrefF16(%ptr : tensor<*xf16>) -} - -// CHECK: Unranked Memref base@{{(0x)?[-0-9a-fA-F]*}} -// CHECK-SAME: rank = 2 offset = 0 sizes = [16, 16] strides = [128, 1] data = -// CHECK-NEXT: [4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047], -// CHECK-NEXT: [9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625], -// CHECK-NEXT: [4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047], -// CHECK-NEXT: [9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625], -// CHECK-NEXT: [4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047], -// CHECK-NEXT: [9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625], -// CHECK-NEXT: [4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047], -// CHECK-NEXT: [9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625], -// CHECK-NEXT: [4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047], -// CHECK-NEXT: [9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625], -// CHECK-NEXT: [4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047], -// CHECK-NEXT: [9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625], -// CHECK-NEXT: [4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047], -// CHECK-NEXT: [9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625], -// CHECK-NEXT: [4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047, 4.98047], -// CHECK-NEXT: [9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625, 9.5625] diff --git a/test/mlir/test/gc/gpu-runner/IMEX/XeGPU/f16_matmul_64x64.mlir b/test/mlir/test/gc/gpu-runner/IMEX/XeGPU/f16_matmul_64x64.mlir deleted file mode 100644 index 8b5fbbdea..000000000 --- a/test/mlir/test/gc/gpu-runner/IMEX/XeGPU/f16_matmul_64x64.mlir +++ /dev/null @@ -1,128 +0,0 @@ -// RUN: gc-gpu-runner --shared-libs=%mlir_runner_utils %s | FileCheck %s - -module{ - -func.func @linalg_matmul(%arg0: tensor<64x64xf16>, - %arg1: tensor<64x64xf16>, - %arg2: tensor<64x64xf16>) -> tensor<64x64xf16> { - %0 = linalg.matmul ins(%arg0, %arg1 : tensor<64x64xf16>, tensor<64x64xf16>) - outs(%arg2 : tensor<64x64xf16>) -> tensor<64x64xf16> - return %0 : tensor<64x64xf16> -} - -func.func @generate_t(%min : f16, %max : f16) -> tensor<64x64xf16> { - %c32 = arith.constant 32.0 : f16 - %c1023 = arith.constant 1023.0 : f16 - %tmp = arith.subf %max, %min : f16 - %step = arith.divf %tmp, %c1023 : f16 - - // Generate the values - // for i in range(n): - // for j in range(n): - // index = i * n + j - // value = min_value + index * step - %0 = tensor.generate { - ^bb0(%i : index, %j : index): - %cst64 = arith.constant 64.0 : f16 - %int0 = arith.index_cast %i : index to i16 - %int1 = arith.index_cast %j : index to i16 - %fp1 = arith.uitofp %int0 : i16 to f16 - %fp2 = arith.uitofp %int1 : i16 to f16 - - %tmp1 = arith.mulf %fp1, %cst64 : f16 - %res = arith.addf %tmp1, %fp2 : f16 - - %tmp2 = arith.mulf %res, %step : f16 - %val = arith.addf %min, %tmp2 : f16 - tensor.yield %val : f16 - } : tensor<64x64xf16> - return %0 : tensor<64x64xf16> -} - -func.func @main() { - %a0 = arith.constant 0.0 : f16 - %b0 = arith.constant 12.0 : f16 - %0 = call @generate_t(%a0, %b0) : (f16, f16) -> tensor<64x64xf16> - - %a1 = arith.constant 0.0 : f16 - %b1 = arith.constant 10.0 : f16 - %1 = call @generate_t(%a1, %b1) : (f16, f16) -> tensor<64x64xf16> - - %2 = arith.constant dense<0.0> : tensor<64x64xf16> - %gpu_res = call @linalg_matmul(%0, %1, %2) : (tensor<64x64xf16>, tensor<64x64xf16>, tensor<64x64xf16>) -> tensor<64x64xf16> - - %cast = tensor.cast %gpu_res : tensor<64x64xf16> to tensor<*xf16> - call @printMemrefF16(%cast) : (tensor<*xf16>) -> () - return -} - -func.func private @printMemrefF16(%ptr : tensor<*xf16>) -} - -// CHECK: Unranked Memref base@{{(0x)?[-0-9a-fA-F]*}} -// CHECK-SAME: rank = 2 offset = 0 sizes = [64, 64] strides = [64, 1] data = -// Computed using numpy: -// CHECK-NEXT: [626.5, 626.5, 627, 627.5, 627.5, 627.5, 628, 628, 628, 628, 628.5, 629.5, 629.5, 629.5, 629.5, 630, 630, 630, 630.5, 631, 631, 631, 631.5, 632, 632, 632, 632.5, 633, 633, 633, 633.5, 634, 634, 634, 634.5, 634.5, 634.5, 635, 635.5, 635.5, 635.5, 635.5, 636, 636.5, 636.5, 637, 637, 637.5, 637.5, 637.5, 638, 638.5, 638.5, 638.5, 639, 639, 639, 639.5, 640, 640.5, 640.5, 640.5, 640.5, 641], -// CHECK-NEXT: [1574, 1574, 1575, 1576, 1576, 1577, 1578, 1579, 1579, 1579, 1581, 1582, 1582, 1582, 1583, 1584, 1585, 1585, 1586, 1587, 1587, 1588, 1589, 1590, 1590, 1590, 1592, 1593, 1593, 1594, 1594, 1596, 1596, 1596, 1597, 1598, 1599, 1599, 1600, 1601, 1601, 1602, 1603, 1604, 1605, 1605, 1606, 1607, 1607, 1607, 1609, 1610, 1610, 1610, 1611, 1612, 1612, 1613, 1614, 1616, 1616, 1616, 1617, 1618], -// CHECK-NEXT: [2520, 2520, 2522, 2524, 2524, 2524, 2526, 2528, 2528, 2530, 2532, 2534, 2534, 2534, 2536, 2538, 2538, 2540, 2540, 2542, 2544, 2544, 2546, 2546, 2548, 2548, 2550, 2552, 2552, 2554, 2554, 2556, 2558, 2558, 2560, 2562, 2562, 2562, 2564, 2566, 2566, 2566, 2568, 2572, 2572, 2572, 2574, 2576, 2576, 2576, 2578, 2580, 2580, 2582, 2584, 2584, 2584, 2586, 2588, 2590, 2590, 2590, 2592, 2594], -// CHECK-NEXT: [3468, 3468, 3472, 3474, 3474, 3476, 3478, 3480, 3480, 3482, 3484, 3488, 3488, 3488, 3490, 3494, 3494, 3494, 3498, 3500, 3500, 3502, 3504, 3506, 3506, 3508, 3510, 3514, 3514, 3514, 3516, 3520, 3520, 3522, 3524, 3526, 3526, 3528, 3530, 3532, 3532, 3534, 3536, 3540, 3540, 3542, 3542, 3546, 3546, 3548, 3550, 3552, 3552, 3554, 3556, 3558, 3558, 3560, 3562, 3566, 3566, 3568, 3570, 3572], -// CHECK-NEXT: [4412, 4416, 4416, 4420, 4420, 4424, 4428, 4428, 4428, 4432, 4436, 4440, 4440, 4440, 4444, 4448, 4448, 4448, 4452, 4456, 4456, 4456, 4460, 4464, 4464, 4464, 4468, 4472, 4472, 4476, 4476, 4480, 4480, 4484, 4484, 4488, 4488, 4492, 4496, 4496, 4496, 4500, 4504, 4508, 4508, 4508, 4512, 4512, 4516, 4516, 4520, 4524, 4524, 4524, 4528, 4528, 4532, 4532, 4536, 4540, 4540, 4540, 4544, 4548], -// CHECK-NEXT: [5360, 5360, 5364, 5368, 5368, 5372, 5376, 5376, 5380, 5380, 5384, 5392, 5392, 5392, 5396, 5400, 5400, 5404, 5408, 5408, 5412, 5412, 5416, 5420, 5420, 5420, 5428, 5432, 5432, 5432, 5436, 5440, 5440, 5444, 5448, 5452, 5452, 5452, 5456, 5460, 5460, 5464, 5468, 5472, 5472, 5476, 5476, 5480, 5484, 5484, 5488, 5492, 5492, 5496, 5500, 5500, 5504, 5504, 5508, 5512, 5516, 5516, 5520, 5524], -// CHECK-NEXT: [6308, 6312, 6316, 6320, 6320, 6324, 6328, 6332, 6332, 6336, 6340, 6348, 6348, 6348, 6352, 6356, 6360, 6360, 6364, 6368, 6372, 6372, 6376, 6380, 6384, 6384, 6388, 6396, 6396, 6396, 6400, 6404, 6408, 6408, 6412, 6420, 6420, 6420, 6428, 6428, 6432, 6432, 6436, 6444, 6444, 6448, 6448, 6456, 6456, 6456, 6464, 6468, 6468, 6468, 6476, 6480, 6480, 6480, 6488, 6492, 6492, 6496, 6500, 6504], -// CHECK-NEXT: [7256, 7256, 7264, 7268, 7268, 7272, 7276, 7280, 7284, 7284, 7292, 7296, 7300, 7300, 7304, 7312, 7312, 7312, 7320, 7324, 7324, 7328, 7332, 7336, 7340, 7340, 7348, 7352, 7356, 7356, 7360, 7368, 7368, 7368, 7376, 7380, 7384, 7384, 7392, 7392, 7396, 7396, 7404, 7412, 7412, 7412, 7416, 7424, 7424, 7428, 7432, 7436, 7440, 7440, 7448, 7448, 7452, 7452, 7460, 7468, 7468, 7468, 7472, 7480], -// CHECK-NEXT: [8200, 8200, 8208, 8216, 8216, 8216, 8224, 8232, 8232, 8232, 8240, 8248, 8248, 8256, 8256, 8264, 8264, 8264, 8272, 8280, 8280, 8280, 8288, 8296, 8296, 8296, 8304, 8312, 8312, 8320, 8320, 8328, 8328, 8328, 8336, 8344, 8344, 8344, 8352, 8360, 8360, 8360, 8368, 8376, 8376, 8384, 8384, 8392, 8392, 8392, 8400, 8408, 8408, 8408, 8416, 8424, 8424, 8424, 8432, 8440, 8440, 8440, 8448, 8456], -// CHECK-NEXT: [9144, 9152, 9160, 9160, 9168, 9168, 9176, 9176, 9184, 9184, 9192, 9200, 9200, 9208, 9208, 9216, 9216, 9224, 9224, 9232, 9232, 9240, 9248, 9248, 9256, 9256, 9264, 9272, 9272, 9272, 9280, 9288, 9288, 9288, 9296, 9304, 9304, 9312, 9320, 9320, 9320, 9328, 9336, 9344, 9344, 9344, 9352, 9360, 9360, 9360, 9368, 9376, 9376, 9384, 9392, 9392, 9392, 9400, 9408, 9416, 9416, 9416, 9424, 9432], -// CHECK-NEXT: [10096, 10096, 10104, 10112, 10112, 10120, 10128, 10128, 10136, 10136, 10144, 10152, 10160, 10160, 10168, 10176, 10176, 10176, 10184, 10192, 10192, 10200, 10208, 10208, 10216, 10216, 10224, 10232, 10232, 10240, 10240, 10248, 10256, 10256, 10264, 10272, 10272, 10280, 10288, 10288, 10288, 10296, 10304, 10312, 10312, 10320, 10320, 10328, 10336, 10336, 10344, 10352, 10352, 10352, 10360, 10368, 10368, 10376, 10384, 10392, 10392, 10392, 10400, 10408], -// CHECK-NEXT: [11040, 11040, 11056, 11064, 11064, 11064, 11072, 11080, 11080, 11088, 11096, 11104, 11104, 11112, 11112, 11128, 11128, 11128, 11136, 11144, 11152, 11152, 11160, 11168, 11168, 11168, 11184, 11192, 11192, 11200, 11200, 11208, 11216, 11216, 11224, 11232, 11232, 11240, 11248, 11256, 11256, 11256, 11264, 11280, 11280, 11280, 11288, 11296, 11304, 11304, 11312, 11320, 11320, 11328, 11336, 11336, 11344, 11344, 11352, 11368, 11368, 11368, 11376, 11384], -// CHECK-NEXT: [11992, 11992, 12008, 12016, 12016, 12016, 12024, 12032, 12040, 12040, 12048, 12064, 12064, 12064, 12072, 12080, 12088, 12088, 12096, 12104, 12112, 12112, 12120, 12128, 12128, 12136, 12144, 12152, 12160, 12160, 12168, 12176, 12176, 12184, 12192, 12200, 12200, 12208, 12216, 12224, 12224, 12224, 12240, 12248, 12248, 12256, 12264, 12272, 12272, 12280, 12288, 12296, 12296, 12296, 12312, 12312, 12320, 12320, 12328, 12344, 12344, 12352, 12352, 12368], -// CHECK-NEXT: [12936, 12944, 12952, 12960, 12960, 12968, 12976, 12984, 12984, 12992, 13000, 13016, 13016, 13016, 13024, 13032, 13040, 13040, 13048, 13064, 13064, 13064, 13080, 13088, 13088, 13088, 13104, 13112, 13120, 13120, 13128, 13136, 13144, 13144, 13152, 13160, 13168, 13168, 13176, 13184, 13192, 13192, 13200, 13216, 13216, 13224, 13224, 13240, 13240, 13248, 13256, 13264, 13264, 13272, 13280, 13288, 13288, 13296, 13304, 13320, 13320, 13320, 13328, 13336], -// CHECK-NEXT: [13880, 13888, 13896, 13904, 13912, 13912, 13928, 13936, 13936, 13936, 13952, 13968, 13968, 13968, 13976, 13992, 13992, 13992, 14008, 14016, 14016, 14024, 14032, 14040, 14048, 14048, 14056, 14072, 14072, 14080, 14088, 14096, 14104, 14104, 14112, 14128, 14128, 14128, 14144, 14152, 14152, 14160, 14168, 14184, 14184, 14192, 14192, 14208, 14208, 14216, 14224, 14232, 14240, 14240, 14248, 14256, 14264, 14264, 14280, 14288, 14296, 14296, 14304, 14312], -// CHECK-NEXT: [14832, 14832, 14848, 14856, 14856, 14864, 14872, 14880, 14888, 14888, 14904, 14912, 14920, 14920, 14928, 14944, 14944, 14952, 14960, 14968, 14976, 14976, 14992, 15000, 15000, 15008, 15016, 15032, 15032, 15040, 15048, 15056, 15064, 15064, 15080, 15088, 15088, 15096, 15104, 15112, 15120, 15120, 15136, 15152, 15152, 15152, 15160, 15176, 15176, 15184, 15192, 15200, 15208, 15208, 15224, 15232, 15232, 15240, 15248, 15264, 15264, 15272, 15280, 15288], -// CHECK-NEXT: [15776, 15784, 15792, 15808, 15808, 15816, 15824, 15832, 15840, 15840, 15856, 15872, 15872, 15880, 15888, 15896, 15904, 15904, 15920, 15928, 15936, 15936, 15952, 15960, 15960, 15968, 15984, 15992, 16000, 16000, 16008, 16024, 16024, 16032, 16040, 16056, 16056, 16064, 16072, 16080, 16088, 16088, 16104, 16120, 16120, 16128, 16136, 16144, 16152, 16152, 16168, 16176, 16184, 16184, 16200, 16208, 16208, 16216, 16232, 16240, 16248, 16248, 16256, 16272], -// CHECK-NEXT: [16720, 16736, 16736, 16752, 16752, 16768, 16784, 16784, 16784, 16800, 16800, 16816, 16832, 16832, 16832, 16848, 16864, 16864, 16880, 16880, 16896, 16896, 16912, 16912, 16912, 16928, 16944, 16960, 16960, 16960, 16976, 16976, 16992, 16992, 17008, 17024, 17024, 17024, 17040, 17040, 17056, 17056, 17072, 17088, 17088, 17088, 17104, 17120, 17120, 17120, 17136, 17152, 17152, 17152, 17168, 17184, 17184, 17184, 17200, 17216, 17216, 17232, 17232, 17248], -// CHECK-NEXT: [17664, 17680, 17696, 17696, 17712, 17712, 17728, 17728, 17744, 17744, 17760, 17776, 17776, 17776, 17792, 17808, 17808, 17808, 17824, 17840, 17840, 17856, 17856, 17872, 17872, 17888, 17904, 17920, 17920, 17920, 17936, 17952, 17952, 17952, 17968, 17984, 17984, 17984, 18000, 18016, 18016, 18016, 18032, 18048, 18048, 18064, 18064, 18080, 18080, 18096, 18112, 18112, 18128, 18128, 18144, 18144, 18160, 18160, 18176, 18192, 18192, 18192, 18208, 18224], -// CHECK-NEXT: [18624, 18624, 18640, 18656, 18656, 18656, 18672, 18688, 18688, 18688, 18704, 18720, 18736, 18736, 18736, 18752, 18768, 18768, 18784, 18800, 18800, 18800, 18816, 18832, 18832, 18832, 18848, 18880, 18880, 18880, 18896, 18912, 18912, 18912, 18928, 18944, 18944, 18944, 18960, 18976, 18976, 18992, 19008, 19024, 19024, 19024, 19040, 19056, 19056, 19056, 19072, 19088, 19088, 19104, 19120, 19120, 19120, 19136, 19152, 19168, 19168, 19168, 19184, 19200], -// CHECK-NEXT: [19568, 19568, 19584, 19600, 19600, 19616, 19632, 19632, 19648, 19648, 19664, 19680, 19680, 19696, 19696, 19712, 19728, 19728, 19744, 19760, 19760, 19760, 19776, 19792, 19792, 19808, 19824, 19840, 19840, 19840, 19856, 19872, 19872, 19888, 19888, 19904, 19920, 19920, 19936, 19952, 19952, 19952, 19968, 19984, 20000, 20000, 20016, 20032, 20032, 20032, 20048, 20064, 20064, 20080, 20096, 20096, 20096, 20112, 20128, 20144, 20144, 20160, 20160, 20176], -// CHECK-NEXT: [20512, 20512, 20528, 20544, 20560, 20560, 20576, 20592, 20592, 20592, 20608, 20640, 20640, 20640, 20656, 20672, 20672, 20688, 20688, 20704, 20720, 20720, 20736, 20752, 20752, 20752, 20768, 20800, 20800, 20800, 20816, 20832, 20832, 20848, 20864, 20880, 20880, 20880, 20896, 20912, 20912, 20912, 20944, 20960, 20960, 20960, 20976, 20992, 20992, 21008, 21024, 21040, 21040, 21040, 21056, 21072, 21072, 21088, 21104, 21120, 21120, 21120, 21136, 21152], -// CHECK-NEXT: [21456, 21472, 21488, 21504, 21504, 21504, 21520, 21536, 21536, 21552, 21568, 21584, 21584, 21600, 21600, 21616, 21632, 21632, 21648, 21664, 21664, 21680, 21696, 21696, 21712, 21712, 21728, 21760, 21760, 21760, 21776, 21792, 21792, 21808, 21824, 21840, 21840, 21840, 21856, 21872, 21872, 21888, 21904, 21920, 21920, 21936, 21936, 21968, 21968, 21968, 21984, 22000, 22000, 22016, 22032, 22048, 22048, 22048, 22064, 22096, 22096, 22096, 22112, 22128], -// CHECK-NEXT: [22400, 22416, 22432, 22448, 22448, 22448, 22480, 22480, 22496, 22496, 22512, 22544, 22544, 22544, 22560, 22576, 22576, 22592, 22608, 22624, 22624, 22624, 22656, 22656, 22672, 22672, 22688, 22720, 22720, 22720, 22736, 22752, 22752, 22768, 22784, 22800, 22800, 22800, 22832, 22832, 22848, 22848, 22864, 22896, 22896, 22896, 22912, 22928, 22928, 22944, 22960, 22976, 22976, 22976, 23008, 23008, 23024, 23024, 23040, 23072, 23072, 23072, 23088, 23104], -// CHECK-NEXT: [23360, 23360, 23376, 23392, 23408, 23408, 23424, 23440, 23440, 23456, 23472, 23488, 23504, 23504, 23520, 23536, 23536, 23552, 23568, 23584, 23584, 23584, 23616, 23616, 23632, 23632, 23648, 23680, 23680, 23680, 23696, 23712, 23728, 23728, 23744, 23760, 23776, 23776, 23792, 23808, 23808, 23824, 23840, 23856, 23872, 23872, 23888, 23904, 23904, 23920, 23936, 23952, 23952, 23952, 23984, 23984, 24000, 24000, 24016, 24048, 24048, 24048, 24064, 24080], -// CHECK-NEXT: [24304, 24304, 24320, 24336, 24352, 24352, 24368, 24384, 24400, 24400, 24416, 24448, 24448, 24448, 24464, 24480, 24496, 24496, 24512, 24528, 24544, 24544, 24560, 24576, 24592, 24592, 24608, 24640, 24640, 24640, 24656, 24672, 24688, 24688, 24704, 24720, 24736, 24736, 24752, 24768, 24768, 24784, 24800, 24832, 24832, 24832, 24848, 24864, 24880, 24880, 24896, 24912, 24928, 24928, 24944, 24960, 24960, 24976, 24992, 25024, 25024, 25024, 25040, 25056], -// CHECK-NEXT: [25248, 25248, 25280, 25296, 25296, 25296, 25328, 25344, 25344, 25344, 25376, 25392, 25392, 25408, 25424, 25440, 25440, 25456, 25472, 25488, 25488, 25504, 25520, 25536, 25536, 25552, 25568, 25600, 25600, 25600, 25616, 25632, 25648, 25648, 25664, 25696, 25696, 25696, 25728, 25728, 25744, 25744, 25776, 25792, 25792, 25808, 25824, 25840, 25840, 25856, 25872, 25888, 25888, 25904, 25920, 25936, 25936, 25952, 25968, 25984, 26000, 26000, 26016, 26032], -// CHECK-NEXT: [26192, 26192, 26224, 26240, 26240, 26256, 26272, 26288, 26288, 26304, 26320, 26352, 26352, 26352, 26368, 26400, 26400, 26400, 26432, 26448, 26448, 26464, 26480, 26496, 26496, 26512, 26528, 26560, 26560, 26560, 26576, 26592, 26608, 26608, 26640, 26656, 26656, 26656, 26688, 26704, 26704, 26704, 26736, 26752, 26768, 26768, 26784, 26800, 26816, 26816, 26832, 26864, 26864, 26864, 26896, 26912, 26912, 26912, 26944, 26960, 26976, 26976, 26992, 27008], -// CHECK-NEXT: [27136, 27152, 27168, 27184, 27200, 27200, 27232, 27232, 27248, 27248, 27280, 27296, 27312, 27312, 27328, 27344, 27360, 27360, 27392, 27408, 27408, 27424, 27440, 27456, 27456, 27472, 27488, 27520, 27520, 27536, 27536, 27568, 27568, 27584, 27600, 27616, 27616, 27632, 27648, 27664, 27680, 27680, 27712, 27728, 27728, 27744, 27760, 27776, 27792, 27792, 27808, 27840, 27840, 27840, 27872, 27888, 27888, 27888, 27920, 27952, 27952, 27952, 27968, 28000], -// CHECK-NEXT: [28096, 28096, 28112, 28144, 28144, 28144, 28176, 28192, 28192, 28208, 28224, 28256, 28256, 28272, 28288, 28304, 28304, 28320, 28336, 28352, 28368, 28368, 28400, 28416, 28416, 28416, 28448, 28480, 28480, 28496, 28496, 28528, 28528, 28544, 28560, 28576, 28592, 28592, 28624, 28624, 28640, 28640, 28672, 28704, 28704, 28704, 28720, 28752, 28752, 28768, 28784, 28800, 28800, 28816, 28832, 28848, 28864, 28864, 28896, 28912, 28928, 28928, 28944, 28976], -// CHECK-NEXT: [29040, 29040, 29072, 29088, 29088, 29104, 29120, 29136, 29152, 29152, 29184, 29200, 29216, 29216, 29232, 29264, 29264, 29264, 29296, 29312, 29328, 29328, 29360, 29360, 29376, 29376, 29408, 29440, 29440, 29440, 29456, 29488, 29488, 29504, 29520, 29536, 29552, 29552, 29584, 29600, 29600, 29616, 29632, 29664, 29664, 29680, 29696, 29712, 29728, 29728, 29744, 29776, 29776, 29792, 29808, 29824, 29824, 29840, 29872, 29888, 29904, 29904, 29920, 29936], -// CHECK-NEXT: [29984, 29984, 30016, 30032, 30032, 30048, 30064, 30080, 30096, 30096, 30128, 30160, 30160, 30176, 30192, 30208, 30224, 30224, 30240, 30272, 30272, 30288, 30304, 30320, 30336, 30336, 30368, 30400, 30400, 30400, 30416, 30448, 30448, 30464, 30480, 30512, 30512, 30512, 30544, 30560, 30560, 30576, 30608, 30624, 30640, 30640, 30656, 30688, 30688, 30704, 30720, 30736, 30752, 30752, 30784, 30800, 30800, 30816, 30832, 30864, 30864, 30880, 30896, 30912], -// CHECK-NEXT: [30928, 30944, 30960, 30992, 30992, 31008, 31024, 31040, 31056, 31056, 31088, 31120, 31120, 31120, 31136, 31168, 31168, 31184, 31200, 31232, 31232, 31248, 31264, 31280, 31296, 31296, 31328, 31360, 31360, 31376, 31392, 31408, 31424, 31424, 31456, 31472, 31472, 31488, 31520, 31536, 31536, 31552, 31568, 31600, 31600, 31616, 31632, 31648, 31664, 31664, 31696, 31712, 31728, 31728, 31760, 31776, 31776, 31792, 31808, 31840, 31856, 31856, 31872, 31904], -// CHECK-NEXT: [31872, 31888, 31904, 31936, 31936, 31952, 31968, 31984, 32000, 32000, 32032, 32064, 32064, 32080, 32096, 32128, 32128, 32144, 32160, 32192, 32192, 32192, 32224, 32240, 32256, 32256, 32288, 32320, 32320, 32336, 32352, 32368, 32384, 32384, 32416, 32432, 32448, 32448, 32480, 32496, 32496, 32512, 32544, 32560, 32576, 32576, 32592, 32624, 32640, 32640, 32672, 32688, 32688, 32704, 32736, 32752, 32752, 32768, 32800, 32832, 32832, 32832, 32864, 32864], -// CHECK-NEXT: [32832, 32832, 32864, 32896, 32896, 32896, 32928, 32928, 32960, 32960, 32992, 33024, 33024, 33024, 33056, 33088, 33088, 33088, 33120, 33152, 33152, 33152, 33184, 33184, 33216, 33216, 33248, 33280, 33280, 33280, 33312, 33344, 33344, 33344, 33376, 33408, 33408, 33408, 33440, 33472, 33472, 33472, 33504, 33536, 33536, 33536, 33568, 33600, 33600, 33600, 33632, 33664, 33664, 33664, 33696, 33728, 33728, 33728, 33760, 33792, 33792, 33792, 33824, 33856], -// CHECK-NEXT: [33760, 33792, 33792, 33824, 33824, 33856, 33856, 33888, 33888, 33920, 33920, 33952, 33984, 33984, 33984, 34016, 34048, 34048, 34080, 34080, 34112, 34112, 34144, 34144, 34176, 34176, 34208, 34240, 34240, 34240, 34272, 34304, 34304, 34304, 34336, 34368, 34368, 34368, 34400, 34432, 34432, 34432, 34464, 34496, 34496, 34528, 34528, 34560, 34560, 34592, 34592, 34624, 34624, 34656, 34656, 34688, 34688, 34720, 34720, 34752, 34784, 34784, 34784, 34816], -// CHECK-NEXT: [34720, 34720, 34752, 34784, 34784, 34784, 34816, 34848, 34848, 34848, 34880, 34912, 34912, 34944, 34944, 34976, 34976, 35008, 35008, 35040, 35040, 35072, 35104, 35104, 35136, 35136, 35168, 35200, 35200, 35200, 35232, 35264, 35264, 35264, 35296, 35328, 35328, 35328, 35360, 35392, 35392, 35392, 35424, 35456, 35488, 35488, 35488, 35520, 35552, 35552, 35584, 35584, 35616, 35616, 35648, 35648, 35680, 35680, 35712, 35744, 35744, 35744, 35776, 35808], -// CHECK-NEXT: [35648, 35680, 35712, 35712, 35744, 35744, 35776, 35776, 35808, 35808, 35840, 35872, 35872, 35872, 35904, 35936, 35936, 35936, 35968, 36000, 36000, 36032, 36064, 36064, 36064, 36096, 36128, 36160, 36160, 36160, 36192, 36224, 36224, 36224, 36256, 36288, 36288, 36288, 36320, 36352, 36352, 36352, 36384, 36448, 36448, 36448, 36480, 36480, 36512, 36512, 36544, 36576, 36576, 36576, 36608, 36640, 36640, 36640, 36672, 36704, 36736, 36736, 36736, 36768], -// CHECK-NEXT: [36608, 36608, 36640, 36672, 36672, 36672, 36704, 36736, 36736, 36768, 36800, 36832, 36832, 36832, 36864, 36896, 36896, 36896, 36928, 36960, 36960, 36992, 36992, 37024, 37024, 37056, 37088, 37120, 37120, 37120, 37152, 37184, 37184, 37184, 37216, 37248, 37248, 37280, 37312, 37312, 37312, 37344, 37376, 37408, 37408, 37408, 37440, 37472, 37472, 37472, 37504, 37536, 37536, 37568, 37600, 37600, 37600, 37632, 37664, 37696, 37696, 37696, 37728, 37760], -// CHECK-NEXT: [37536, 37568, 37600, 37632, 37632, 37632, 37664, 37696, 37696, 37696, 37728, 37760, 37792, 37792, 37824, 37824, 37856, 37856, 37888, 37920, 37920, 37920, 37952, 37984, 37984, 38016, 38048, 38080, 38080, 38080, 38112, 38144, 38144, 38144, 38176, 38208, 38208, 38240, 38272, 38272, 38272, 38304, 38336, 38368, 38368, 38368, 38400, 38432, 38432, 38464, 38464, 38496, 38528, 38528, 38560, 38560, 38592, 38592, 38624, 38656, 38656, 38688, 38688, 38720], -// CHECK-NEXT: [38528, 38528, 38560, 38592, 38592, 38592, 38624, 38656, 38656, 38656, 38688, 38752, 38752, 38752, 38784, 38816, 38816, 38816, 38848, 38880, 38880, 38912, 38944, 38944, 38976, 38976, 39008, 39040, 39040, 39072, 39072, 39104, 39136, 39136, 39168, 39200, 39200, 39200, 39232, 39264, 39264, 39264, 39296, 39360, 39360, 39360, 39392, 39424, 39424, 39424, 39456, 39488, 39488, 39520, 39552, 39552, 39584, 39584, 39616, 39648, 39648, 39680, 39680, 39712], -// CHECK-NEXT: [39456, 39456, 39488, 39520, 39520, 39552, 39584, 39584, 39616, 39616, 39648, 39680, 39712, 39712, 39744, 39744, 39776, 39776, 39808, 39840, 39840, 39872, 39904, 39904, 39904, 39936, 39968, 40000, 40000, 40032, 40032, 40064, 40096, 40096, 40128, 40160, 40160, 40160, 40192, 40224, 40224, 40256, 40288, 40320, 40320, 40320, 40352, 40384, 40384, 40416, 40448, 40448, 40480, 40480, 40512, 40544, 40544, 40544, 40576, 40640, 40640, 40640, 40672, 40704], -// CHECK-NEXT: [40416, 40416, 40448, 40480, 40480, 40480, 40512, 40544, 40544, 40576, 40608, 40640, 40640, 40672, 40672, 40704, 40704, 40736, 40768, 40800, 40800, 40800, 40832, 40864, 40864, 40896, 40928, 40960, 40960, 40960, 40992, 41024, 41024, 41056, 41088, 41120, 41120, 41120, 41152, 41184, 41184, 41216, 41248, 41280, 41280, 41280, 41312, 41344, 41344, 41376, 41408, 41440, 41440, 41440, 41472, 41504, 41504, 41536, 41568, 41600, 41600, 41600, 41632, 41664], -// CHECK-NEXT: [41344, 41344, 41376, 41408, 41440, 41440, 41472, 41504, 41504, 41504, 41536, 41600, 41600, 41600, 41632, 41664, 41664, 41696, 41728, 41760, 41760, 41760, 41792, 41824, 41824, 41824, 41888, 41920, 41920, 41920, 41952, 41984, 41984, 42016, 42048, 42080, 42080, 42080, 42112, 42144, 42144, 42176, 42208, 42240, 42240, 42272, 42272, 42304, 42336, 42336, 42368, 42400, 42400, 42432, 42464, 42464, 42496, 42496, 42528, 42560, 42560, 42592, 42592, 42656], -// CHECK-NEXT: [42304, 42304, 42336, 42368, 42368, 42400, 42432, 42432, 42464, 42464, 42496, 42528, 42560, 42560, 42592, 42624, 42624, 42624, 42656, 42688, 42720, 42720, 42752, 42784, 42784, 42784, 42848, 42880, 42880, 42880, 42912, 42944, 42944, 42976, 43008, 43040, 43040, 43040, 43104, 43104, 43104, 43136, 43168, 43200, 43200, 43232, 43264, 43296, 43296, 43296, 43328, 43360, 43360, 43392, 43424, 43456, 43456, 43456, 43488, 43552, 43552, 43552, 43584, 43616], -// CHECK-NEXT: [43232, 43232, 43296, 43328, 43328, 43328, 43360, 43392, 43392, 43424, 43456, 43488, 43488, 43520, 43520, 43552, 43584, 43584, 43616, 43648, 43648, 43680, 43712, 43744, 43744, 43744, 43776, 43840, 43840, 43840, 43872, 43904, 43904, 43936, 43968, 44000, 44000, 44000, 44064, 44064, 44096, 44096, 44128, 44160, 44192, 44192, 44224, 44256, 44256, 44288, 44320, 44352, 44352, 44352, 44384, 44416, 44416, 44448, 44480, 44512, 44512, 44544, 44544, 44608], -// CHECK-NEXT: [44192, 44192, 44224, 44256, 44256, 44288, 44320, 44352, 44352, 44352, 44416, 44448, 44448, 44448, 44480, 44512, 44544, 44544, 44576, 44608, 44608, 44640, 44672, 44672, 44704, 44704, 44736, 44800, 44800, 44800, 44832, 44864, 44896, 44896, 44928, 44960, 44960, 44992, 45024, 45024, 45056, 45056, 45088, 45152, 45152, 45152, 45184, 45216, 45216, 45248, 45280, 45312, 45312, 45312, 45376, 45376, 45408, 45408, 45440, 45504, 45504, 45504, 45536, 45568], -// CHECK-NEXT: [45120, 45152, 45184, 45216, 45216, 45216, 45280, 45280, 45312, 45312, 45344, 45408, 45408, 45408, 45440, 45472, 45472, 45504, 45536, 45568, 45568, 45600, 45632, 45632, 45664, 45664, 45696, 45760, 45760, 45760, 45792, 45824, 45856, 45856, 45888, 45920, 45920, 45952, 45984, 46016, 46016, 46016, 46080, 46112, 46112, 46144, 46144, 46176, 46208, 46208, 46240, 46272, 46272, 46304, 46336, 46368, 46368, 46368, 46432, 46464, 46464, 46496, 46496, 46560], -// CHECK-NEXT: [46080, 46112, 46144, 46176, 46176, 46176, 46240, 46240, 46272, 46272, 46304, 46368, 46368, 46368, 46400, 46432, 46464, 46464, 46496, 46528, 46528, 46560, 46592, 46624, 46624, 46624, 46688, 46720, 46720, 46752, 46752, 46816, 46816, 46816, 46848, 46912, 46912, 46912, 46944, 46976, 46976, 47008, 47040, 47072, 47104, 47104, 47136, 47168, 47168, 47200, 47232, 47264, 47264, 47296, 47328, 47328, 47360, 47360, 47424, 47456, 47456, 47488, 47488, 47520], -// CHECK-NEXT: [47040, 47040, 47072, 47104, 47136, 47136, 47168, 47200, 47200, 47232, 47264, 47296, 47328, 47328, 47360, 47392, 47392, 47424, 47456, 47488, 47488, 47520, 47552, 47584, 47584, 47584, 47648, 47680, 47680, 47712, 47712, 47776, 47776, 47776, 47840, 47872, 47872, 47872, 47936, 47936, 47968, 47968, 48000, 48064, 48064, 48064, 48096, 48128, 48160, 48160, 48192, 48224, 48256, 48256, 48288, 48320, 48320, 48352, 48384, 48416, 48448, 48448, 48480, 48512], -// CHECK-NEXT: [47968, 48000, 48032, 48064, 48064, 48096, 48128, 48160, 48160, 48160, 48224, 48256, 48256, 48288, 48320, 48352, 48352, 48384, 48416, 48448, 48448, 48480, 48512, 48544, 48544, 48544, 48608, 48640, 48640, 48672, 48672, 48736, 48736, 48736, 48800, 48832, 48832, 48832, 48896, 48896, 48928, 48928, 48960, 49024, 49024, 49056, 49056, 49120, 49120, 49120, 49152, 49216, 49216, 49216, 49280, 49280, 49312, 49312, 49344, 49408, 49408, 49408, 49440, 49472], -// CHECK-NEXT: [48928, 48928, 48960, 49024, 49024, 49024, 49088, 49088, 49120, 49120, 49152, 49216, 49216, 49248, 49248, 49312, 49312, 49312, 49376, 49408, 49408, 49408, 49472, 49472, 49504, 49504, 49568, 49600, 49600, 49632, 49632, 49696, 49696, 49696, 49760, 49792, 49792, 49792, 49856, 49856, 49888, 49888, 49952, 49984, 49984, 50016, 50048, 50080, 50080, 50112, 50144, 50176, 50176, 50208, 50240, 50272, 50272, 50272, 50336, 50368, 50368, 50400, 50432, 50464], -// CHECK-NEXT: [49856, 49888, 49920, 49952, 49952, 49984, 50016, 50048, 50048, 50080, 50112, 50176, 50176, 50176, 50208, 50240, 50272, 50272, 50304, 50368, 50368, 50368, 50432, 50432, 50464, 50464, 50496, 50560, 50560, 50592, 50592, 50656, 50656, 50688, 50720, 50752, 50752, 50784, 50816, 50848, 50848, 50848, 50912, 50944, 50976, 50976, 51008, 51040, 51040, 51072, 51104, 51136, 51136, 51168, 51200, 51232, 51232, 51264, 51296, 51360, 51360, 51360, 51392, 51424], -// CHECK-NEXT: [50816, 50816, 50880, 50912, 50912, 50944, 50976, 51008, 51008, 51008, 51072, 51104, 51136, 51136, 51168, 51200, 51232, 51232, 51264, 51296, 51328, 51328, 51360, 51392, 51424, 51424, 51456, 51520, 51520, 51552, 51552, 51616, 51616, 51648, 51680, 51712, 51712, 51744, 51776, 51808, 51808, 51840, 51872, 51904, 51936, 51936, 51968, 52000, 52032, 52032, 52064, 52096, 52128, 52128, 52192, 52192, 52224, 52224, 52288, 52320, 52320, 52352, 52384, 52416], -// CHECK-NEXT: [51776, 51776, 51808, 51840, 51872, 51872, 51936, 51936, 51968, 51968, 52032, 52064, 52064, 52096, 52128, 52160, 52160, 52192, 52224, 52256, 52288, 52288, 52320, 52352, 52384, 52384, 52416, 52480, 52480, 52512, 52512, 52576, 52576, 52608, 52640, 52672, 52672, 52704, 52736, 52768, 52768, 52800, 52832, 52896, 52896, 52896, 52928, 52992, 52992, 52992, 53056, 53088, 53088, 53120, 53152, 53184, 53184, 53184, 53248, 53280, 53312, 53312, 53344, 53376], -// CHECK-NEXT: [52704, 52736, 52768, 52800, 52800, 52832, 52864, 52896, 52896, 52928, 52960, 53024, 53024, 53056, 53056, 53120, 53120, 53152, 53184, 53216, 53216, 53248, 53280, 53312, 53312, 53344, 53376, 53440, 53440, 53472, 53472, 53536, 53536, 53568, 53600, 53632, 53632, 53664, 53696, 53728, 53728, 53760, 53792, 53856, 53856, 53888, 53888, 53952, 53952, 53984, 54016, 54048, 54048, 54080, 54112, 54144, 54144, 54176, 54208, 54272, 54272, 54304, 54304, 54368], -// CHECK-NEXT: [53664, 53664, 53728, 53760, 53760, 53792, 53824, 53856, 53856, 53888, 53920, 53984, 53984, 53984, 54016, 54080, 54080, 54112, 54144, 54176, 54176, 54208, 54240, 54272, 54272, 54304, 54336, 54400, 54400, 54432, 54464, 54496, 54496, 54528, 54560, 54592, 54624, 54624, 54656, 54688, 54720, 54720, 54784, 54816, 54848, 54848, 54880, 54912, 54944, 54944, 54976, 55040, 55040, 55040, 55104, 55136, 55136, 55136, 55200, 55232, 55264, 55264, 55296, 55328], -// CHECK-NEXT: [54592, 54624, 54656, 54688, 54720, 54720, 54784, 54784, 54816, 54816, 54880, 54912, 54944, 54944, 54976, 55008, 55040, 55040, 55104, 55136, 55136, 55168, 55200, 55232, 55232, 55264, 55296, 55360, 55360, 55392, 55424, 55456, 55456, 55488, 55520, 55552, 55584, 55584, 55648, 55648, 55680, 55680, 55744, 55776, 55808, 55808, 55840, 55872, 55904, 55904, 55968, 56000, 56000, 56032, 56064, 56096, 56096, 56128, 56160, 56224, 56224, 56256, 56288, 56320], -// CHECK-NEXT: [55552, 55552, 55616, 55648, 55648, 55680, 55712, 55744, 55776, 55776, 55840, 55872, 55872, 55904, 55936, 55968, 56000, 56000, 56032, 56096, 56096, 56128, 56160, 56192, 56192, 56224, 56256, 56320, 56320, 56352, 56384, 56416, 56416, 56448, 56480, 56512, 56544, 56544, 56608, 56640, 56640, 56640, 56704, 56768, 56768, 56768, 56800, 56864, 56864, 56896, 56928, 56960, 56960, 56992, 57024, 57056, 57088, 57088, 57152, 57184, 57216, 57216, 57248, 57280], -// CHECK-NEXT: [56512, 56512, 56544, 56608, 56608, 56608, 56672, 56704, 56704, 56736, 56768, 56832, 56832, 56864, 56896, 56928, 56928, 56960, 56992, 57056, 57056, 57056, 57120, 57152, 57152, 57184, 57216, 57280, 57280, 57312, 57344, 57376, 57376, 57408, 57440, 57504, 57504, 57504, 57568, 57600, 57600, 57632, 57664, 57728, 57728, 57760, 57792, 57824, 57824, 57856, 57888, 57920, 57952, 57952, 58016, 58048, 58048, 58080, 58112, 58176, 58176, 58208, 58208, 58272], -// CHECK-NEXT: [57440, 57472, 57504, 57536, 57568, 57568, 57632, 57632, 57664, 57664, 57728, 57792, 57792, 57792, 57824, 57888, 57888, 57920, 57952, 57984, 58016, 58016, 58080, 58112, 58112, 58144, 58176, 58240, 58240, 58272, 58304, 58336, 58336, 58368, 58400, 58464, 58464, 58464, 58528, 58560, 58560, 58592, 58624, 58688, 58688, 58720, 58752, 58784, 58816, 58816, 58848, 58912, 58912, 58944, 58976, 59008, 59008, 59040, 59072, 59136, 59168, 59168, 59200, 59232], -// CHECK-NEXT: [58400, 58400, 58432, 58496, 58496, 58528, 58560, 58592, 58624, 58624, 58688, 58720, 58752, 58752, 58784, 58848, 58848, 58880, 58912, 58944, 58976, 58976, 59040, 59040, 59072, 59072, 59136, 59200, 59200, 59232, 59264, 59296, 59296, 59328, 59360, 59424, 59424, 59456, 59488, 59520, 59520, 59552, 59584, 59648, 59680, 59680, 59712, 59744, 59776, 59776, 59840, 59872, 59872, 59904, 59936, 59968, 60000, 60000, 60064, 60128, 60128, 60128, 60160, 60224], -// CHECK-NEXT: [59328, 59360, 59392, 59424, 59456, 59456, 59520, 59552, 59552, 59584, 59616, 59680, 59680, 59712, 59744, 59776, 59808, 59808, 59872, 59904, 59904, 59936, 60000, 60000, 60032, 60032, 60096, 60160, 60160, 60192, 60224, 60256, 60288, 60288, 60320, 60384, 60384, 60416, 60448, 60480, 60512, 60512, 60576, 60608, 60640, 60640, 60672, 60736, 60736, 60768, 60800, 60832, 60864, 60864, 60928, 60960, 60960, 60992, 61024, 61088, 61088, 61120, 61152, 61184], -// CHECK-NEXT: [60288, 60288, 60352, 60384, 60384, 60416, 60448, 60480, 60512, 60512, 60576, 60640, 60640, 60672, 60704, 60736, 60768, 60768, 60832, 60864, 60864, 60896, 60928, 60960, 60992, 60992, 61056, 61120, 61120, 61152, 61184, 61216, 61248, 61248, 61280, 61344, 61344, 61376, 61408, 61440, 61472, 61472, 61536, 61600, 61600, 61600, 61632, 61696, 61696, 61728, 61760, 61824, 61824, 61856, 61888, 61920, 61952, 61952, 62016, 62048, 62080, 62080, 62112, 62176] diff --git a/test/mlir/test/gc/gpu-runner/IMEX/XeGPU/f16_mlp_32x4096x4096x4096.mlir b/test/mlir/test/gc/gpu-runner/IMEX/XeGPU/f16_mlp_32x4096x4096x4096.mlir deleted file mode 100644 index cb3f59728..000000000 --- a/test/mlir/test/gc/gpu-runner/IMEX/XeGPU/f16_mlp_32x4096x4096x4096.mlir +++ /dev/null @@ -1,55 +0,0 @@ -// RUN: gc-gpu-runner --shared-libs=%mlir_runner_utils %s | FileCheck %s - -module { - func.func @linalg_mlp(%arg0: tensor<32x4096xf16>, %arg1: tensor<4096x4096xf16>, %arg2 : tensor<32x4096xf16>, - %arg3: tensor<4096x4096xf16>, %arg4 : tensor<32x4096xf16>) { - %cst = arith.constant 0.000000e+00 : f16 - %0 = tensor.empty() : tensor<32x4096xf16> - %1 = linalg.fill ins(%cst : f16) outs(%0 : tensor<32x4096xf16>) -> tensor<32x4096xf16> - %2 = linalg.matmul ins(%arg0, %arg1 : tensor<32x4096xf16>, tensor<4096x4096xf16>) - outs(%1 : tensor<32x4096xf16>) -> (tensor<32x4096xf16>) - %3 = tensor.empty() : tensor<32x4096xf16> - %4 = linalg.add ins(%arg2, %2 : tensor<32x4096xf16>, tensor<32x4096xf16>) - outs(%3 : tensor<32x4096xf16>) -> tensor<32x4096xf16> - %5 = arith.constant dense<0.000000e+00> : tensor<32x4096xf16> - %6 = tensor.empty() : tensor<32x4096xf16> - %7 = linalg.max ins(%5, %4 : tensor<32x4096xf16>, tensor<32x4096xf16>) - outs(%6 : tensor<32x4096xf16>) -> tensor<32x4096xf16> - - %8 = tensor.empty() : tensor<32x4096xf16> - %9 = linalg.fill ins(%cst : f16) outs(%8 : tensor<32x4096xf16>) -> tensor<32x4096xf16> - %10 = linalg.matmul ins(%7, %arg3 : tensor<32x4096xf16>, tensor<4096x4096xf16>) - outs(%9 : tensor<32x4096xf16>) -> (tensor<32x4096xf16>) - %11 = tensor.empty() : tensor<32x4096xf16> - %12 = linalg.add ins(%arg4, %10 : tensor<32x4096xf16>, tensor<32x4096xf16>) - outs(%11 : tensor<32x4096xf16>) -> tensor<32x4096xf16> - %13 = arith.constant dense<0.000000e+00> : tensor<32x4096xf16> - %14 = tensor.empty() : tensor<32x4096xf16> - %15 = linalg.max ins(%13, %12 : tensor<32x4096xf16>, tensor<32x4096xf16>) - outs(%14 : tensor<32x4096xf16>) -> tensor<32x4096xf16> - - %slice = tensor.extract_slice %15[0, 0][32, 1][1, 1] : tensor<32x4096xf16> to tensor<32xf16> - %cast = tensor.cast %slice : tensor<32xf16> to tensor<*xf16> - call @printMemrefF16(%cast) : (tensor<*xf16>) -> () - - return - } - - func.func @main() { - %0 = arith.constant dense<0.01> : tensor<32x4096xf16> - %1 = arith.constant dense<0.01> : tensor<4096x4096xf16> - %2 = arith.constant dense<0.02> : tensor<32x4096xf16> - %3 = arith.constant dense<0.01> : tensor<4096x4096xf16> - %4 = arith.constant dense<0.02> : tensor<32x4096xf16> - - func.call @linalg_mlp(%0, %1, %2, %3, %4) : (tensor<32x4096xf16>, tensor<4096x4096xf16>, tensor<32x4096xf16>, - tensor<4096x4096xf16>, tensor<32x4096xf16>) -> () - return - } - - func.func private @printMemrefF16(%ptr : tensor<*xf16>) attributes { llvm.emit_c_interface } -} - -// CHECK: Unranked Memref base@{{(0x)?[-0-9a-fA-F]*}} -// CHECK-SAME: rank = 1 offset = 0 sizes = [32] strides = [4096] data = -// CHECK-NEXT: [17.625, 17.625, 17.625, 17.625, 17.625, 17.625, 17.625, 17.625, 17.625, 17.625, 17.625, 17.625, 17.625, 17.625, 17.625, 17.625, 17.625, 17.625, 17.625, 17.625, 17.625, 17.625, 17.625, 17.625, 17.625, 17.625, 17.625, 17.625, 17.625, 17.625, 17.625, 17.625] diff --git a/test/mlir/test/gc/gpu-runner/IMEX/XeGPU/f16_mlp_32x4096x4096x4096_transpose.mlir b/test/mlir/test/gc/gpu-runner/IMEX/XeGPU/f16_mlp_32x4096x4096x4096_transpose.mlir deleted file mode 100644 index ff88f71ec..000000000 --- a/test/mlir/test/gc/gpu-runner/IMEX/XeGPU/f16_mlp_32x4096x4096x4096_transpose.mlir +++ /dev/null @@ -1,120 +0,0 @@ -// RUN: gc-gpu-runner --shared-libs=%mlir_runner_utils %s | FileCheck %s - -module { - func.func @linalg_mlp(%arg0: tensor<32x4096xf16>, %arg1: tensor<4096x4096xf16>, %arg2 : tensor<32x4096xf16>, - %arg3: tensor<4096x4096xf16>, %arg4 : tensor<32x4096xf16>) { - %cst = arith.constant 0.000000e+00 : f16 - %0 = tensor.empty() : tensor<32x4096xf16> - %1 = linalg.fill ins(%cst : f16) outs(%0 : tensor<32x4096xf16>) -> tensor<32x4096xf16> - %2 = linalg.matmul_transpose_b ins(%arg0, %arg1 : tensor<32x4096xf16>, tensor<4096x4096xf16>) - outs(%1 : tensor<32x4096xf16>) -> (tensor<32x4096xf16>) - %3 = tensor.empty() : tensor<32x4096xf16> - %4 = linalg.add ins(%arg2, %2 : tensor<32x4096xf16>, tensor<32x4096xf16>) - outs(%3 : tensor<32x4096xf16>) -> tensor<32x4096xf16> - %5 = arith.constant dense<0.000000e+00> : tensor<32x4096xf16> - %6 = tensor.empty() : tensor<32x4096xf16> - %7 = linalg.max ins(%5, %4 : tensor<32x4096xf16>, tensor<32x4096xf16>) - outs(%6 : tensor<32x4096xf16>) -> tensor<32x4096xf16> - - %8 = tensor.empty() : tensor<32x4096xf16> - %9 = linalg.fill ins(%cst : f16) outs(%8 : tensor<32x4096xf16>) -> tensor<32x4096xf16> - %t = tensor.empty() : tensor<4096x4096xf16> - %transposed = linalg.transpose ins(%arg3 : tensor<4096x4096xf16>) outs(%t : tensor<4096x4096xf16>) permutation = [1, 0] - - %10 = linalg.matmul ins(%7, %transposed : tensor<32x4096xf16>, tensor<4096x4096xf16>) - outs(%9 : tensor<32x4096xf16>) -> (tensor<32x4096xf16>) - %11 = tensor.empty() : tensor<32x4096xf16> - %12 = linalg.add ins(%arg4, %10 : tensor<32x4096xf16>, tensor<32x4096xf16>) - outs(%11 : tensor<32x4096xf16>) -> tensor<32x4096xf16> - %13 = arith.constant dense<0.000000e+00> : tensor<32x4096xf16> - %14 = tensor.empty() : tensor<32x4096xf16> - %15 = linalg.max ins(%13, %12 : tensor<32x4096xf16>, tensor<32x4096xf16>) - outs(%14 : tensor<32x4096xf16>) -> tensor<32x4096xf16> - - %slice = tensor.extract_slice %15[0, 0][32, 2][1, 1] : tensor<32x4096xf16> to tensor<32x2xf16> - %cast = tensor.cast %slice : tensor<32x2xf16> to tensor<*xf16> - call @printMemrefF16(%cast) : (tensor<*xf16>) -> () - - return - } - - // generates asymmetric tensor - func.func @generate_t(%even_val : f16, %odd_val : f16) -> tensor<4096x4096xf16> { - %0 = tensor.generate { - ^bb0(%i : index, %j : index): - %int0 = arith.index_cast %i : index to i32 - %int1 = arith.index_cast %j : index to i32 - - %c2 = arith.constant 2 : i32 - %c0 = arith.constant 0 : i32 - %remeinder = arith.remui %int0, %c2 : i32 - %is_even = arith.cmpi eq, %remeinder, %c0 : i32 - - %val = scf.if %is_even -> (f16) { - scf.yield %even_val : f16 - } else { - scf.yield %odd_val : f16 - } - - tensor.yield %val : f16 - } : tensor<4096x4096xf16> - return %0 : tensor<4096x4096xf16> - } - - func.func @main() { - %0 = arith.constant dense<0.01> : tensor<32x4096xf16> - - %even_v1 = arith.constant 0.02 : f16 - %odd_v1 = arith.constant 0.01 : f16 - %1 = call @generate_t(%even_v1, %odd_v1) : (f16, f16) -> tensor<4096x4096xf16> - - %2 = arith.constant dense<0.02> : tensor<32x4096xf16> - - %even_v2 = arith.constant 0.06 : f16 - %odd_v2 = arith.constant 0.03 : f16 - %3 = call @generate_t(%even_v2, %odd_v2) : (f16, f16) -> tensor<4096x4096xf16> - - %4 = arith.constant dense<0.02> : tensor<32x4096xf16> - - func.call @linalg_mlp(%0, %1, %2, %3, %4) : (tensor<32x4096xf16>, tensor<4096x4096xf16>, tensor<32x4096xf16>, - tensor<4096x4096xf16>, tensor<32x4096xf16>) -> () - return - } - - func.func private @printMemrefF16(%ptr : tensor<*xf16>) attributes { llvm.emit_c_interface } -} - -// CHECK: Unranked Memref base@{{(0x)?[-0-9a-fA-F]*}} -// CHECK-SAME: rank = 2 offset = 0 sizes = [32, 2] strides = [4096, 1] data = -// CHECK-NEXT: [155.875, 77.9375], -// CHECK-NEXT: [155.875, 77.9375], -// CHECK-NEXT: [155.875, 77.9375], -// CHECK-NEXT: [155.875, 77.9375], -// CHECK-NEXT: [155.875, 77.9375], -// CHECK-NEXT: [155.875, 77.9375], -// CHECK-NEXT: [155.875, 77.9375], -// CHECK-NEXT: [155.875, 77.9375], -// CHECK-NEXT: [155.875, 77.9375], -// CHECK-NEXT: [155.875, 77.9375], -// CHECK-NEXT: [155.875, 77.9375], -// CHECK-NEXT: [155.875, 77.9375], -// CHECK-NEXT: [155.875, 77.9375], -// CHECK-NEXT: [155.875, 77.9375], -// CHECK-NEXT: [155.875, 77.9375], -// CHECK-NEXT: [155.875, 77.9375], -// CHECK-NEXT: [155.875, 77.9375], -// CHECK-NEXT: [155.875, 77.9375], -// CHECK-NEXT: [155.875, 77.9375], -// CHECK-NEXT: [155.875, 77.9375], -// CHECK-NEXT: [155.875, 77.9375], -// CHECK-NEXT: [155.875, 77.9375], -// CHECK-NEXT: [155.875, 77.9375], -// CHECK-NEXT: [155.875, 77.9375], -// CHECK-NEXT: [155.875, 77.9375], -// CHECK-NEXT: [155.875, 77.9375], -// CHECK-NEXT: [155.875, 77.9375], -// CHECK-NEXT: [155.875, 77.9375], -// CHECK-NEXT: [155.875, 77.9375], -// CHECK-NEXT: [155.875, 77.9375], -// CHECK-NEXT: [155.875, 77.9375], -// CHECK-NEXT: [155.875, 77.9375] diff --git a/test/mlir/test/gc/gpu-runner/IMEX/XeGPU/rope_sanity_test.mlir b/test/mlir/test/gc/gpu-runner/IMEX/XeGPU/rope_sanity_test.mlir deleted file mode 100644 index 98a0535c0..000000000 --- a/test/mlir/test/gc/gpu-runner/IMEX/XeGPU/rope_sanity_test.mlir +++ /dev/null @@ -1,95 +0,0 @@ -// RUN: gc-gpu-runner --shared-libs=%mlir_runner_utils,%mlir_c_runner_utils %s | FileCheck %s - -!dtype=f16 -!input_memref_type=memref<2x7x32x128x!dtype> -!input_tensor_type=tensor<2x7x32x128x!dtype> -!output_memref_type=memref<2x32x7x128x!dtype> -!output_tensor_type=tensor<2x32x7x128x!dtype> -!cos_sin_cache_memref_type=memref<1x1x2048x128x!dtype> -!cos_sin_cache_tensor_type=tensor<1x1x2048x128x!dtype> -!cos_sin_cache_tensor_shrink_type=tensor<1x1x7x128x!dtype> -!pos_ids_memref_type=memref<1x7xindex> -!pos_ids_tensor_type=tensor<1x7xindex> -#map = affine_map<(xi, yi, zi) -> ((xi * 3 * 4 + yi * 4 + zi) * 2)> -module @fragment_name { -memref.global "private" constant @_cos_cache : !cos_sin_cache_memref_type = dense<3.000000e+00> -memref.global "private" constant @_sin_cache : !cos_sin_cache_memref_type = dense<2.000000e+00> -memref.global "private" constant @_iinput_const : !input_memref_type = dense<3.000000e+00> -memref.global "private" constant @_ipos_ids_const : !pos_ids_memref_type = dense<1> -memref.global "private" constant @_ipos_id_end_const : memref<1xindex> = dense<1> -func.func @RoPE(%iinput: !input_memref_type, %ipos_ids: !pos_ids_memref_type, %ipos_id_end: memref<1xindex>, %out: !output_memref_type) { - %input = bufferization.to_tensor %iinput restrict : !input_memref_type - %cos_cache = memref.get_global @_cos_cache : !cos_sin_cache_memref_type - %sin_cache = memref.get_global @_sin_cache : !cos_sin_cache_memref_type - %cos_cache_tensor = bufferization.to_tensor %cos_cache restrict : !cos_sin_cache_memref_type - %sin_cache_tensor = bufferization.to_tensor %sin_cache restrict : !cos_sin_cache_memref_type - %pos_ids = bufferization.to_tensor %ipos_ids restrict : !pos_ids_memref_type - %pos_id_end = bufferization.to_tensor %ipos_id_end restrict : memref<1xindex> - %3 = tensor.empty(): !output_tensor_type - - %transpose_in = linalg.transpose ins(%input: !input_tensor_type) outs(%3:!output_tensor_type) permutation = [0, 2, 1, 3] - - %c0 = arith.constant 0 : index - %c3 = arith.constant 3 : index - %cos_cache_slice = tensor.extract_slice %cos_cache_tensor[0, 0, 0, 0] [1, 1, 7, 128] [1, 1, 1, 1] : !cos_sin_cache_tensor_type to !cos_sin_cache_tensor_shrink_type - %cos_cache_slice2 = tensor.collapse_shape %cos_cache_slice [[0, 1], [2],[3]] : tensor<1x1x7x128x!dtype> into tensor<1x7x128x!dtype> - %cos_cache_slice3 = tensor.collapse_shape %cos_cache_slice2 [[0, 1], [2]] : tensor<1x7x128x!dtype> into tensor<7x128x!dtype> - %pos_ids_index=tensor.expand_shape %pos_ids [[0],[1,2]] output_shape [1, 7, 1] : tensor<1x7xindex> into tensor<1x7x1xindex> - - %cos_cache_slice4 = tensor.gather %cos_cache_slice3[%pos_ids_index] gather_dims([0]) : (tensor<7x128x!dtype>, tensor<1x7x1xindex>) -> tensor<1x7x128x!dtype> - - %cos_cache_slice5 = tensor.expand_shape %cos_cache_slice4 [[0,1],[2],[3]] output_shape [1,1,7,128] : tensor<1x7x128x!dtype> into tensor<1x1x7x128x!dtype> - %cos_cache_slice6 = tensor.collapse_shape %cos_cache_slice5 [[0,1,2],[3]] : tensor<1x1x7x128x!dtype> into tensor<7x128x!dtype> - - %cos_cache_slice7 = linalg.broadcast ins(%cos_cache_slice6: tensor<7x128x!dtype>) outs(%3: !output_tensor_type) dimensions = [0, 1] - %input_apply_cos_cache = linalg.mul ins(%transpose_in, %cos_cache_slice7: !output_tensor_type, !output_tensor_type) outs(%3: !output_tensor_type) -> !output_tensor_type - - %head_dim = tensor.dim %transpose_in, %c3 : !output_tensor_type - %c2 = arith.constant 2 : index - %half_head_dim = arith.floordivsi %head_dim, %c2 : index - %transpose_input_first_half = tensor.extract_slice %transpose_in[0, 0, 0, 0][2, 32, 7, 64][1,1,1,1] : !output_tensor_type to tensor<2x32x7x64x!dtype> - %transpose_input_second_half = tensor.extract_slice %transpose_in[0, 0, 0, %half_head_dim][2, 32, 7, 64][1,1,1,1] : !output_tensor_type to tensor<2x32x7x64x!dtype> - %cnegative1 = arith.constant dense<-1.000000e+00> : tensor<2x32x7x64x!dtype> - %empty_tensor = tensor.empty() : tensor<2x32x7x64x!dtype> - %transpose_input_second_half_opposite = linalg.mul ins(%transpose_input_second_half, %cnegative1: tensor<2x32x7x64x!dtype>, tensor<2x32x7x64x!dtype>) outs(%empty_tensor: tensor<2x32x7x64x!dtype>) -> tensor<2x32x7x64x!dtype> - - %transformed_input = tensor.concat dim(3) %transpose_input_second_half_opposite, %transpose_input_first_half : (tensor<2x32x7x64x!dtype>, tensor<2x32x7x64x!dtype>) -> !output_tensor_type - - %sin_cache_slice = tensor.extract_slice %sin_cache_tensor[0, 0, 0, 0] [1, 1, 7, 128] [1, 1, 1, 1] : !cos_sin_cache_tensor_type to !cos_sin_cache_tensor_shrink_type - %sin_cache_slice2 = tensor.collapse_shape %sin_cache_slice [[0, 1], [2],[3]] : tensor<1x1x7x128x!dtype> into tensor<1x7x128x!dtype> - %sin_cache_slice3 = tensor.collapse_shape %sin_cache_slice2 [[0, 1], [2]] : tensor<1x7x128x!dtype> into tensor<7x128x!dtype> - %sin_cache_slice4 = tensor.gather %sin_cache_slice3[%pos_ids_index] gather_dims([0]) : (tensor<7x128x!dtype>, tensor<1x7x1xindex>) -> tensor<1x7x128x!dtype> - - %sin_cache_slice5 = tensor.expand_shape %sin_cache_slice4 [[0,1],[2],[3]] output_shape [1,1,7,128] : tensor<1x7x128x!dtype> into tensor<1x1x7x128x!dtype> - %sin_cache_slice6 = tensor.collapse_shape %sin_cache_slice5 [[0,1,2],[3]] : tensor<1x1x7x128x!dtype> into tensor<7x128x!dtype> - - %sin_cache_slice7 = linalg.broadcast ins(%sin_cache_slice6: tensor<7x128x!dtype>) outs(%3: !output_tensor_type) dimensions = [0, 1] - %input_apply_sin_cache = linalg.mul ins(%transformed_input, %sin_cache_slice7: !output_tensor_type, !output_tensor_type) outs(%3: !output_tensor_type) -> !output_tensor_type - - %result = linalg.add ins(%input_apply_cos_cache, %input_apply_sin_cache: !output_tensor_type, !output_tensor_type) outs(%3: !output_tensor_type) -> !output_tensor_type - bufferization.materialize_in_destination %result in restrict writable %out : (!output_tensor_type, !output_memref_type) -> () - return -} - -func.func @main() { - %inp = memref.get_global @_iinput_const : !input_memref_type - %ipos_ids = memref.get_global @_ipos_ids_const : !pos_ids_memref_type - %ipos_id_end = memref.get_global @_ipos_id_end_const : memref<1xindex> - - %out = memref.alloc() {alignment = 64 : i64} : !output_memref_type - - func.call @RoPE(%inp, %ipos_ids, %ipos_id_end, %out) : (!input_memref_type, !pos_ids_memref_type, memref<1xindex>, !output_memref_type) -> () - - %out_subview = memref.subview %out[0, 0, 0, 0] [2, 1, 1, 1] [1, 1, 1, 1] : !output_memref_type to memref<2xf16, strided<[28672]>> - %cast = memref.cast %out_subview : memref<2xf16, strided<[28672]>> to memref<*xf16> - call @printMemrefF16(%cast) : (memref<*xf16>) -> () - - return -} - -func.func private @printMemrefF16(%ptr : memref<*xf16>) -} - -// CHECK: Unranked Memref base@{{(0x)?[-0-9a-fA-F]*}} -// CHECK-SAME: rank = 1 offset = 0 sizes = [2] strides = [28672] data = -// CHECK-NEXT: [3, 3] diff --git a/test/mlir/test/gc/gpu-runner/IMEX/lit.local.cfg b/test/mlir/test/gc/gpu-runner/IMEX/lit.local.cfg deleted file mode 100644 index 958f39814..000000000 --- a/test/mlir/test/gc/gpu-runner/IMEX/lit.local.cfg +++ /dev/null @@ -1,2 +0,0 @@ -if not config.gc_use_imex: - config.unsupported = True diff --git a/test/mlir/test/gc/gpu-runner/IMEX/mlp.mlir b/test/mlir/test/gc/gpu-runner/IMEX/mlp.mlir deleted file mode 100644 index 6f914beb8..000000000 --- a/test/mlir/test/gc/gpu-runner/IMEX/mlp.mlir +++ /dev/null @@ -1,132 +0,0 @@ -// RUN: gc-gpu-runner --shared-libs=%mlir_runner_utils %s | FileCheck %s - -#map0 = affine_map<(d0, d1) -> (d1)> -#map1 = affine_map<(d0, d1) -> (d0, d1)> -#map2 = affine_map<(d0, d1, d2) -> (d0, d2)> -#map3 = affine_map<(d0, d1, d2) -> (d2, d1)> -#map4 = affine_map<(d0, d1, d2) -> (d0, d1)> -#map5 = affine_map<(d0, d1) -> ()> -#map6 = affine_map<(d0, d1) -> (d0, 0)> -module @mnist_mlp { - func.func @main() { - %0 = arith.constant dense<1.0> : tensor<1x784xf32> - %1 = arith.constant dense<2.0> : tensor<784x512xf32> - %2 = arith.constant dense<0.5> : tensor<512xf32> - %3 = arith.constant dense<0.8> : tensor<512x512xf32> - %4 = arith.constant dense<0.5> : tensor<512xf32> - %5 = arith.constant dense<1.5> : tensor<512x10xf32> - %6 = arith.constant dense<0.3> : tensor<10xf32> - %7 = call @test(%0,%1,%2,%3,%4,%5,%6) : (tensor<1x784xf32>, tensor<784x512xf32>, tensor<512xf32>, tensor<512x512xf32>, tensor<512xf32>, tensor<512x10xf32>, tensor<10xf32>) -> tensor<1x10xf32> - %unranked = tensor.cast %7 : tensor<1x10xf32>to tensor<*xf32> - call @printMemrefF32(%unranked) : (tensor<*xf32>) -> () - return - } - func.func private @printMemrefF32(tensor<*xf32>) - func.func @test(%arg0: tensor<1x784xf32>, %arg1: tensor<784x512xf32>, %arg2: tensor<512xf32>, %arg3: tensor<512x512xf32>, %arg4: tensor<512xf32>, %arg5: tensor<512x10xf32>, %arg6: tensor<10xf32>) -> tensor<1x10xf32> { - %cst = arith.constant 0xFF800000 : f32 - %cst_0 = arith.constant 0.000000e+00 : f32 - %0 = tensor.empty() : tensor<1x512xf32> - %1 = linalg.generic {indexing_maps = [#map0, #map1], iterator_types = ["parallel", "parallel"]} ins(%arg2 : tensor<512xf32>) outs(%0 : tensor<1x512xf32>) { - ^bb0(%arg7: f32, %arg8: f32): - linalg.yield %arg7 : f32 - } -> tensor<1x512xf32> - %2 = linalg.generic {indexing_maps = [#map2, #map3, #map4], iterator_types = ["parallel", "parallel", "reduction"]} ins(%arg0, %arg1 : tensor<1x784xf32>, tensor<784x512xf32>) outs(%1 : tensor<1x512xf32>) attrs = {iterator_ranges = [1, 512, 784]} { - ^bb0(%arg7: f32, %arg8: f32, %arg9: f32): - %29 = arith.mulf %arg7, %arg8 : f32 - %30 = arith.addf %arg9, %29 : f32 - linalg.yield %30 : f32 - } -> tensor<1x512xf32> - %3 = tensor.empty() : tensor<1x512xi1> - %4 = linalg.generic {indexing_maps = [#map1, #map5, #map1], iterator_types = ["parallel", "parallel"]} ins(%2, %cst_0 : tensor<1x512xf32>, f32) outs(%3 : tensor<1x512xi1>) { - ^bb0(%arg7: f32, %arg8: f32, %arg9: i1): - %29 = arith.cmpf olt, %arg7, %arg8 : f32 - linalg.yield %29 : i1 - } -> tensor<1x512xi1> - %5 = tensor.empty() : tensor<1x512xf32> - %6 = linalg.generic {indexing_maps = [#map1, #map5, #map1, #map1], iterator_types = ["parallel", "parallel"]} ins(%4, %cst_0, %2 : tensor<1x512xi1>, f32, tensor<1x512xf32>) outs(%5 : tensor<1x512xf32>) { - ^bb0(%arg7: i1, %arg8: f32, %arg9: f32, %arg10: f32): - %29 = arith.select %arg7, %arg8, %arg9 : f32 - linalg.yield %29 : f32 - } -> tensor<1x512xf32> - %7 = tensor.empty() : tensor<1x512xf32> - %8 = linalg.generic {indexing_maps = [#map0, #map1], iterator_types = ["parallel", "parallel"]} ins(%arg4 : tensor<512xf32>) outs(%7 : tensor<1x512xf32>) { - ^bb0(%arg7: f32, %arg8: f32): - linalg.yield %arg7 : f32 - } -> tensor<1x512xf32> - %9 = linalg.generic {indexing_maps = [#map2, #map3, #map4], iterator_types = ["parallel", "parallel", "reduction"]} ins(%6, %arg3 : tensor<1x512xf32>, tensor<512x512xf32>) outs(%8 : tensor<1x512xf32>) attrs = {iterator_ranges = [1, 512, 512]} { - ^bb0(%arg7: f32, %arg8: f32, %arg9: f32): - %29 = arith.mulf %arg7, %arg8 : f32 - %30 = arith.addf %arg9, %29 : f32 - linalg.yield %30 : f32 - } -> tensor<1x512xf32> - %10 = tensor.empty() : tensor<1x512xi1> - %11 = linalg.generic {indexing_maps = [#map1, #map5, #map1], iterator_types = ["parallel", "parallel"]} ins(%9, %cst_0 : tensor<1x512xf32>, f32) outs(%10 : tensor<1x512xi1>) { - ^bb0(%arg7: f32, %arg8: f32, %arg9: i1): - %29 = arith.cmpf olt, %arg7, %arg8 : f32 - linalg.yield %29 : i1 - } -> tensor<1x512xi1> - %12 = tensor.empty() : tensor<1x512xf32> - %13 = linalg.generic {indexing_maps = [#map1, #map5, #map1, #map1], iterator_types = ["parallel", "parallel"]} ins(%11, %cst_0, %9 : tensor<1x512xi1>, f32, tensor<1x512xf32>) outs(%12 : tensor<1x512xf32>) { - ^bb0(%arg7: i1, %arg8: f32, %arg9: f32, %arg10: f32): - %29 = arith.select %arg7, %arg8, %arg9 : f32 - linalg.yield %29 : f32 - } -> tensor<1x512xf32> - %14 = tensor.empty() : tensor<1x10xf32> - %15 = linalg.generic {indexing_maps = [#map0, #map1], iterator_types = ["parallel", "parallel"]} ins(%arg6 : tensor<10xf32>) outs(%14 : tensor<1x10xf32>) { - ^bb0(%arg7: f32, %arg8: f32): - linalg.yield %arg7 : f32 - } -> tensor<1x10xf32> - %16 = linalg.generic {indexing_maps = [#map2, #map3, #map4], iterator_types = ["parallel", "parallel", "reduction"]} ins(%13, %arg5 : tensor<1x512xf32>, tensor<512x10xf32>) outs(%15 : tensor<1x10xf32>) attrs = {iterator_ranges = [1, 10, 512]} { - ^bb0(%arg7: f32, %arg8: f32, %arg9: f32): - %29 = arith.mulf %arg7, %arg8 : f32 - %30 = arith.addf %arg9, %29 : f32 - linalg.yield %30 : f32 - } -> tensor<1x10xf32> - %17 = tensor.empty() : tensor<1x1xf32> - %18 = linalg.fill ins(%cst : f32) outs(%17 : tensor<1x1xf32>) -> tensor<1x1xf32> - %19 = linalg.generic {indexing_maps = [#map1, #map6], iterator_types = ["parallel", "reduction"]} ins(%16 : tensor<1x10xf32>) outs(%18 : tensor<1x1xf32>) attrs = {iterator_ranges = [1, 10]} { - ^bb0(%arg7: f32, %arg8: f32): - %29 = arith.cmpf ogt, %arg8, %arg7 : f32 - %30 = arith.select %29, %arg8, %arg7 : f32 - linalg.yield %30 : f32 - } -> tensor<1x1xf32> - %20 = tensor.empty() : tensor<1x10xf32> - %21 = linalg.generic {indexing_maps = [#map1, #map6, #map1], iterator_types = ["parallel", "parallel"]} ins(%16, %19 : tensor<1x10xf32>, tensor<1x1xf32>) outs(%20 : tensor<1x10xf32>) { - ^bb0(%arg7: f32, %arg8: f32, %arg9: f32): - %29 = arith.subf %arg7, %arg8 : f32 - linalg.yield %29 : f32 - } -> tensor<1x10xf32> - %22 = tensor.empty() : tensor<1x10xf32> - %23 = linalg.generic {indexing_maps = [#map1, #map1], iterator_types = ["parallel", "parallel"]} ins(%21 : tensor<1x10xf32>) outs(%22 : tensor<1x10xf32>) { - ^bb0(%arg7: f32, %arg8: f32): - %29 = math.exp %arg7 : f32 - linalg.yield %29 : f32 - } -> tensor<1x10xf32> - %24 = tensor.empty() : tensor<1x1xf32> - %25 = linalg.fill ins(%cst_0 : f32) outs(%24 : tensor<1x1xf32>) -> tensor<1x1xf32> - %26 = linalg.generic {indexing_maps = [#map1, #map6], iterator_types = ["parallel", "reduction"]} ins(%23 : tensor<1x10xf32>) outs(%25 : tensor<1x1xf32>) attrs = {iterator_ranges = [1, 10]} { - ^bb0(%arg7: f32, %arg8: f32): - %29 = arith.addf %arg8, %arg7 : f32 - linalg.yield %29 : f32 - } -> tensor<1x1xf32> - %27 = tensor.empty() : tensor<1x10xf32> - %28 = linalg.generic {indexing_maps = [#map1, #map6, #map1], iterator_types = ["parallel", "parallel"]} ins(%23, %26 : tensor<1x10xf32>, tensor<1x1xf32>) outs(%27 : tensor<1x10xf32>) { - ^bb0(%arg7: f32, %arg8: f32, %arg9: f32): - %29 = arith.divf %arg7, %arg8 : f32 - linalg.yield %29 : f32 - } -> tensor<1x10xf32> - return %28 : tensor<1x10xf32> - } -} -// CHECK: Unranked Memref base@ = {{0x[-9a-f]*}} -// CHECK-SAME: rank = {{.}} offset = {{.}} sizes = [1, 10] strides = {{.*}} data = -// CHECK: 0.1 -// CHECK: 0.1 -// CHECK: 0.1 -// CHECK: 0.1 -// CHECK: 0.1 -// CHECK: 0.1 -// CHECK: 0.1 -// CHECK: 0.1 -// CHECK: 0.1 -// CHECK: 0.1 diff --git a/test/mlir/test/gc/gpu-runner/XeVM/lit.local.cfg b/test/mlir/test/gc/gpu-runner/XeVM/lit.local.cfg deleted file mode 100644 index 736deccc1..000000000 --- a/test/mlir/test/gc/gpu-runner/XeVM/lit.local.cfg +++ /dev/null @@ -1,2 +0,0 @@ -if not config.gc_use_gpu or config.gc_use_imex: - config.unsupported = True \ No newline at end of file diff --git a/test/mlir/test/gc/gpu-runner/XeVM/xevm_dpas.mlir b/test/mlir/test/gc/gpu-runner/XeVM/xevm_dpas.mlir deleted file mode 100644 index 1d9ef480a..000000000 --- a/test/mlir/test/gc/gpu-runner/XeVM/xevm_dpas.mlir +++ /dev/null @@ -1,74 +0,0 @@ -// RUN: gc-gpu-runner --shared-libs=%mlir_runner_utils %s | FileCheck %s - -module @gemm attributes {gpu.container_module} { - func.func @test(%src : memref<8x16xf32>) -> memref<8x16xf32> attributes {llvm.emit_c_interface} { - %c1 = arith.constant 1 : index - %c16 = arith.constant 16 : index // Multiple of the *maximum sub-group size* (see `intel_reqd_sub_group_size`) - %memref_src = gpu.alloc host_shared () : memref<8x16xf32> - memref.copy %src, %memref_src : memref<8x16xf32> to memref<8x16xf32> - %src_ptr_as_idx = memref.extract_aligned_pointer_as_index %memref_src : memref<8x16xf32> -> index - %src_ptr_as_i64 = arith.index_cast %src_ptr_as_idx : index to i64 - %src_ptr = llvm.inttoptr %src_ptr_as_i64 : i64 to !llvm.ptr - %src_ptr_casted = llvm.addrspacecast %src_ptr : !llvm.ptr to !llvm.ptr<1> - - %memref_dst = gpu.alloc host_shared () : memref<8x16xf32> - %dst_ptr_as_idx = memref.extract_aligned_pointer_as_index %memref_dst : memref<8x16xf32> -> index - %dst_ptr_as_i64 = arith.index_cast %dst_ptr_as_idx : index to i64 - %dst_ptr = llvm.inttoptr %dst_ptr_as_i64 : i64 to !llvm.ptr - %dst_ptr_casted = llvm.addrspacecast %dst_ptr : !llvm.ptr to !llvm.ptr<1> - - gpu.launch blocks(%arg10, %arg11, %arg12) in (%arg16 = %c1, %arg17 = %c1, %arg18 = %c1) threads(%arg13, %arg14, %arg15) in (%arg19 = %c16, %arg20 = %c1, %arg21 = %c1) { - %base_width = arith.constant 64 : i32 // bytewidth of the block - %base_height = arith.constant 8 : i32 // number of rows - %base_pitch = arith.constant 64 : i32 // bytewidth of the base row - %x = arith.constant 0 : i32 - %y = arith.constant 0 : i32 - // If `intel_reqd_sub_group_size = 16` is not set, the default (32) is used and this `blockload2d` would only load 4 elements into vector<8xi32> - %loaded = xevm.blockload2d %src_ptr_casted, %base_width, %base_height, %base_pitch, %x, %y {elem_size_in_bits=32, tile_width=16, tile_height=8, v_blocks=1, transpose=false, vnni_transform=false, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<8xi32> - %loaded_f32 = vector.bitcast %loaded : vector<8xi32> to vector<8xf32> - %c0 = arith.constant 0 : i32 - %thread_x = gpu.thread_id x - %thread_x_i32 = arith.index_cast %thread_x : index to i32 - %thread_x_f32 = arith.sitofp %thread_x_i32 : i32 to f32 - %loaded_f32_modified = vector.insertelement %thread_x_f32, %loaded_f32[%c0 : i32] : vector<8xf32> - %loaded_modified = vector.bitcast %loaded_f32_modified : vector<8xf32> to vector<8xi32> - xevm.blockstore2d %dst_ptr_casted, %base_width, %base_height, %base_pitch, %x, %y, %loaded_modified {elem_size_in_bits=32, tile_width=16, tile_height=8, v_blocks=1, l1_cache_control=Default, l3_cache_control=Default} : (!llvm.ptr<1>, i32, i32, i32, i32, i32, vector<8xi32>) - gpu.terminator - } - - return %memref_dst : memref<8x16xf32> - } - - func.func @main() attributes {llvm.emit_c_interface} { - %A = memref.alloc() : memref<8x16xf32> - %c0 = arith.constant 0 : index - %c1 = arith.constant 1 : index - %c8 = arith.constant 8 : index - %c16 = arith.constant 16 : index - %c11_f32 = arith.constant 11.11 : f32 - scf.for %i = %c0 to %c8 step %c1 { - scf.for %j = %c0 to %c16 step %c1 { - memref.store %c11_f32, %A[%i, %j] : memref<8x16xf32> - } - } - %B = call @test(%A) : (memref<8x16xf32>) -> memref<8x16xf32> - %B_cast = memref.cast %B : memref<8x16xf32> to memref<*xf32> - %A_cast = memref.cast %A : memref<8x16xf32> to memref<*xf32> - call @printMemrefF32(%A_cast) : (memref<*xf32>) -> () - call @printMemrefF32(%B_cast) : (memref<*xf32>) -> () - - // CHECK: Unranked Memref base@ = 0x{{[0-9a-f]+}} - // CHECK-NEXT: [11.11{{.*}}] - // CHECK-COUNT-96: 11.11 - // CHECK-NEXT: [11.11{{.*}}] - - // CHECK-NEXT: Unranked Memref base@ = 0x{{[0-9a-f]+}} - // CHECK: 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 - // CHECK-COUNT-96: 11.11 - // CHECK-NEXT: [11.11{{.*}}] - - memref.dealloc %A : memref<8x16xf32> - return - } - func.func private @printMemrefF32(%ptr : memref<*xf32>) -} diff --git a/test/mlir/test/gc/python/dialects/cpuruntime.py b/test/mlir/test/gc/python/dialects/cpuruntime.py deleted file mode 100644 index 3b08badcf..000000000 --- a/test/mlir/test/gc/python/dialects/cpuruntime.py +++ /dev/null @@ -1,55 +0,0 @@ -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -# RUN: %python %s | FileCheck %s - -from gc_mlir.dialects import cpuruntime, func -from gc_mlir.ir import * -from gc_mlir.passmanager import PassManager - - -def run(f): - print("\nTEST:", f.__name__) - f() - return f - - -# CHECK-LABEL: TEST: testCPURuntimeOps -@run -def testCPURuntimeOps(): - with Context() as ctx, Location.unknown(): - module = Module.create() - with InsertionPoint(module.body): - - @func.FuncOp.from_py_func(F32Type.get(), IntegerType.get_signless(32)) - def do_print(arg1, arg2): - cpuruntime.printf("Hello world %f %d", [arg1, arg2]) - return - - # CHECK-LABEL: func @do_print( - # CHECK-SAME: %[[ARG_0:.*]]: f32, %[[ARG_1:.*]]: i32) { - # CHECK: cpuruntime.printf "Hello world %f %d" %[[ARG_0]], %[[ARG_1:.*]] : f32, i32 - # CHECK: return - # CHECK: } - print(module) - - -# CHECK-LABEL: TEST: testConvertToLLVM -@run -def testConvertToLLVM(): - with Context(): - module = Module.parse( - """ - module { - func.func @do_print(%arg0: f32, %arg1: i32) { - cpuruntime.printf "Hello world %f %d" %arg0, %arg1 : f32, i32 - return - } - } - """ - ) - pm = PassManager.parse("builtin.module(convert-cpuruntime-to-llvm)") - # CHECK-NOT: cpuruntime.printf - # CHECK: llvm.call @printf - pm.run(module.operation) - print(module) diff --git a/test/mlir/test/gc/python/dialects/lit.local.cfg b/test/mlir/test/gc/python/dialects/lit.local.cfg deleted file mode 100644 index 773d16371..000000000 --- a/test/mlir/test/gc/python/dialects/lit.local.cfg +++ /dev/null @@ -1,2 +0,0 @@ -if not config.gc_use_dnnl: - config.excludes.add("onednn_graph.py") \ No newline at end of file diff --git a/test/mlir/test/gc/python/dialects/onednn_graph.py b/test/mlir/test/gc/python/dialects/onednn_graph.py deleted file mode 100644 index 845626ce2..000000000 --- a/test/mlir/test/gc/python/dialects/onednn_graph.py +++ /dev/null @@ -1,58 +0,0 @@ -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -# RUN: %python %s | FileCheck %s - -from gc_mlir.ir import * -from gc_mlir.dialects import onednn_graph, func -from gc_mlir.passmanager import PassManager - - -def run(f): - print("\nTEST:", f.__name__) - f() - return f - - -# CHECK-LABEL: TEST: testOneDNNGraphOps -@run -def testOneDNNGraphOps(): - with Context() as ctx, Location.unknown(): - module = Module.create() - with InsertionPoint(module.body): - f32 = F32Type.get(ctx) - tensor_type = RankedTensorType.get([128, 128], f32) - - @func.FuncOp.from_py_func(tensor_type, tensor_type) - def entry(arg1, arg2): - res1 = onednn_graph.matmul( - arg1, arg2, bias=None, transpose_a=False, transpose_b=False - ) - res2 = onednn_graph.add(res1, arg2) - return onednn_graph.relu(res2) - - # CHECK: [[MM:%.+]] = onednn_graph.matmul - # CHECK: [[ADD:%.+]] = onednn_graph.add - # CHECK: [[RELU:%.+]] = onednn_graph.relu - print(module) - - -# CHECK-LABEL: TEST: testConvertToLinalg -@run -def testConvertToLinalg(): - with Context(): - module = Module.parse( - """ - func.func @matmul(%arg0: tensor<128x512xbf16>, %arg1: tensor<512x256xbf16>) -> tensor<128x256xbf16> { - %0 = onednn_graph.matmul %arg0, %arg1 : (tensor<128x512xbf16>, tensor<512x256xbf16>) -> tensor<128x256xbf16> - return %0 : tensor<128x256xbf16> - } - """ - ) - pm = PassManager.parse("builtin.module(convert-onednn-graph-to-linalg)") - # CHECK: [[C0:%.+]] = arith.constant 0 - # CHECK: [[INIT:%.+]] = tensor.empty() - # CHECK: [[FILLED:%.+]] = linalg.fill ins([[C0]] : bf16) outs([[INIT]] : tensor<128x256xbf16>) -> tensor<128x256xbf16> - # CHECK: linalg.matmul ins(%arg0, %arg1 : tensor<128x512xbf16>, tensor<512x256xbf16>) outs([[FILLED]] : tensor<128x256xbf16>) -> tensor<128x256xbf16> - pm.run(module.operation) - print(module) diff --git a/test/mlir/test/gc/python/lit.local.cfg b/test/mlir/test/gc/python/lit.local.cfg deleted file mode 100644 index 6deb1f4d5..000000000 --- a/test/mlir/test/gc/python/lit.local.cfg +++ /dev/null @@ -1,7 +0,0 @@ -config.suffixes.add(".py") - -if not config.gc_use_dnnl: - config.excludes.add("smoketest.py") - -if not config.enable_bindings_python: - config.unsupported = True \ No newline at end of file diff --git a/test/mlir/test/gc/python/smoketest.py b/test/mlir/test/gc/python/smoketest.py deleted file mode 100644 index 5045a1f8f..000000000 --- a/test/mlir/test/gc/python/smoketest.py +++ /dev/null @@ -1,57 +0,0 @@ -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -# RUN: %python %s | FileCheck %s - -from gc_mlir.dialects import func, onednn_graph -from gc_mlir.graph_compiler import GraphCompiler -from gc_mlir.ir import * - - -def run(f): - print("\nTEST:", f.__name__) - f() - return f - - -# CHECK-LABEL: TEST: testCreateOp -# CHECK onednn_graph.add -@run -def testCreateOp(): - with Context() as ctx, Location.unknown(): - module = Module.create() - f32 = F32Type.get(ctx) - tensor_type = RankedTensorType.get([128, 256], f32) - with InsertionPoint(module.body): - f = func.FuncOp( - name="add", - type=FunctionType.get( - inputs=[tensor_type, tensor_type], results=[tensor_type] - ), - ) - with InsertionPoint(f.add_entry_block()): - arg0, arg1 = f.entry_block.arguments - result = onednn_graph.AddOp(arg0, arg1).result - func.ReturnOp([result]) - print(module) - - -# CHECK-LABEL: TEST: testCompiler -@run -def testCompiler(): - with Context(): - module = Module.parse( - """ - func.func @matmul(%arg0: tensor<128x512xf32>, %arg1: tensor<512x256xf32>) -> tensor<128x256xf32> { - %0 = onednn_graph.matmul %arg0, %arg1 : (tensor<128x512xf32>, tensor<512x256xf32>) -> tensor<128x256xf32> - return %0 : tensor<128x256xf32> - } - """ - ) - - compiler = GraphCompiler( - pipeline="builtin.module(convert-onednn-graph-to-linalg)" - ) - compiler.compile(module) - # CHECK-NOT: onednn_graph.matmul - print(module) diff --git a/test/mlir/unittests/Analysis/CMakeLists.txt b/test/mlir/unittests/Analysis/CMakeLists.txt deleted file mode 100644 index ed253bfdf..000000000 --- a/test/mlir/unittests/Analysis/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -add_mlir_unittest(GCAnalysisTests - TargetDescriptionAnalysisTest.cpp -) -target_link_libraries(GCAnalysisTests - PRIVATE - GcPasses - GcAnalysis - GcJitWrapper) diff --git a/test/mlir/unittests/Analysis/TargetDescriptionAnalysisTest.cpp b/test/mlir/unittests/Analysis/TargetDescriptionAnalysisTest.cpp deleted file mode 100644 index a3ba8261b..000000000 --- a/test/mlir/unittests/Analysis/TargetDescriptionAnalysisTest.cpp +++ /dev/null @@ -1,81 +0,0 @@ -//===-- TargetDescriptionAnalysisTest.cpp -----------------------*- C++ -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -#include "gc/Analysis/TargetDescriptionAnalysis.h" -#include "gc/ExecutionEngine/Driver/Driver.h" -#include "mlir/AsmParser/AsmParser.h" -#include "mlir/ExecutionEngine/MemRefUtils.h" -#include "mlir/IR/AsmState.h" -#include "mlir/IR/BuiltinOps.h" -#include "mlir/IR/MLIRContext.h" -#include "mlir/Parser/Parser.h" -#include "mlir/Pass/PassManager.h" -#include "llvm/Support/ErrorOr.h" -#include "llvm/Support/MemoryBuffer.h" -#include "llvm/Support/SourceMgr.h" -#include "llvm/Support/raw_ostream.h" -#include "gtest/gtest.h" -#include - -using namespace mlir; - -static const char code1[] = R"mlir( -module attributes { -dlti.target_system_spec = #dlti.target_system_spec< -"CPU": #dlti.target_device_spec< - #dlti.dl_entry<"L1_cache_size_in_bytes", 49152 : ui32>, - #dlti.dl_entry<"L2_cache_size_in_bytes", 2097152 : ui64>, - #dlti.dl_entry<"L3_cache_size_in_bytes", "110100480">, - #dlti.dl_entry<"num_threads", 56 : i32>, - #dlti.dl_entry<"max_vector_width", 512 : i64>> ->} {} -)mlir"; - -TEST(TargetDescriptionAnalysis, CPUNormal) { - MLIRContext ctx{gc::initCompilerAndGetDialects()}; - std::unique_ptr ir_buffer = - llvm::MemoryBuffer::getMemBuffer(code1); - // Parse the input mlir. - llvm::SourceMgr sourceMgr; - sourceMgr.AddNewSourceBuffer(std::move(ir_buffer), llvm::SMLoc()); - mlir::OwningOpRef module = - mlir::parseSourceFile(sourceMgr, &ctx); - ASSERT_TRUE(module); - auto CPUTagetDesc = gc::CPUTargetDescriptionAnalysis(module.get()); - ASSERT_EQ(CPUTagetDesc.getNumThreads(), 56); - ASSERT_EQ(CPUTagetDesc.getCacheSize(1), 49152); - ASSERT_EQ(CPUTagetDesc.getCacheSize(2), 2097152); - ASSERT_EQ(CPUTagetDesc.getCacheSize(3), 110100480); - ASSERT_EQ(CPUTagetDesc.getMaxVectorWidth(), 512); -} - -static const char code2[] = R"mlir( -module attributes { -dlti.target_system_spec = #dlti.target_system_spec< -"CPU": #dlti.target_device_spec< - #dlti.dl_entry<"L1_cache_size_in_bytes", 49152 : ui32>, - #dlti.dl_entry<"L2_cache_size_in_bytes", 2097152 : ui32>> ->} {} -)mlir"; - -TEST(DISABLED_TargetDescriptionAnalysis, CPUMissingValue) { - MLIRContext ctx{gc::initCompilerAndGetDialects()}; - std::unique_ptr ir_buffer = - llvm::MemoryBuffer::getMemBuffer(code2); - // Parse the input mlir. - llvm::SourceMgr sourceMgr; - sourceMgr.AddNewSourceBuffer(std::move(ir_buffer), llvm::SMLoc()); - mlir::OwningOpRef module = - mlir::parseSourceFile(sourceMgr, &ctx); - ASSERT_TRUE(module); - auto CPUTagetDesc = gc::CPUTargetDescriptionAnalysis(module.get()); - ASSERT_EQ(CPUTagetDesc.getNumThreads(), 1); - ASSERT_EQ(CPUTagetDesc.getCacheSize(1), 49152); - ASSERT_EQ(CPUTagetDesc.getCacheSize(2), 2097152); - ASSERT_EQ(CPUTagetDesc.getCacheSize(3), 1048576); - ASSERT_EQ(CPUTagetDesc.getMaxVectorWidth(), 512); -} \ No newline at end of file diff --git a/test/mlir/unittests/CMakeLists.txt b/test/mlir/unittests/CMakeLists.txt deleted file mode 100644 index e07b93ae2..000000000 --- a/test/mlir/unittests/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -add_definitions(-DMLIR_INCLUDE_TESTS) -add_custom_target(GCUnitTests) -set_target_properties(GCUnitTests PROPERTIES FOLDER "MLIR GC Tests") - -# To silence warning caused by Wundef. -add_definitions(-DGTEST_NO_LLVM_SUPPORT=0) - -function(add_mlir_unittest test_dirname) - add_unittest(GCUnitTests ${test_dirname} ${ARGN}) -endfunction() - -add_subdirectory(Analysis) -add_subdirectory(Example) -add_subdirectory(ExecutionEngine) -add_subdirectory(Linalgx) -add_subdirectory(Transforms) - diff --git a/test/mlir/unittests/Example/CMakeLists.txt b/test/mlir/unittests/Example/CMakeLists.txt deleted file mode 100644 index 2439010b5..000000000 --- a/test/mlir/unittests/Example/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -add_mlir_unittest(GCExampleTests - Example.cpp -) -target_link_libraries(GCExampleTests - PRIVATE - MLIRLinalgx) diff --git a/test/mlir/unittests/Example/Example.cpp b/test/mlir/unittests/Example/Example.cpp deleted file mode 100644 index d788ba4ec..000000000 --- a/test/mlir/unittests/Example/Example.cpp +++ /dev/null @@ -1,14 +0,0 @@ -//===-- Example.cpp - Tests for Example -------------------------*- C++ -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "gc/Dialect/Linalgx/LinalgxDialect.h" -#include "gtest/gtest.h" - -TEST(example, HelloWorld) { - ASSERT_EQ(mlir::linalgx::LinalgxDialect::getDialectNamespace(), "linalgx"); -} diff --git a/test/mlir/unittests/ExecutionEngine/BrgemmRuntime.cpp b/test/mlir/unittests/ExecutionEngine/BrgemmRuntime.cpp deleted file mode 100644 index cdc35ff43..000000000 --- a/test/mlir/unittests/ExecutionEngine/BrgemmRuntime.cpp +++ /dev/null @@ -1,206 +0,0 @@ -//===-- BrgemmRuntime.cpp - Brgemm Runtime ----------------------*- C++ -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "gtest/gtest.h" -#include -#include - -// manually include xbyak header here to avoid no-exception compile issue -#define XBYAK_NO_EXCEPTION -#include // NOLINT -#undef XBYAK_NO_EXCEPTION - -#include "gc/ExecutionEngine/CPURuntime/Microkernel/BrgemmInterface.h" -#include - -extern "C" { -extern int gc_runtime_keep_alive; -} - -template inline T randomFP(int range = 1, float delta = 0.0f) { - float fraction = (float)(rand()) / (float)(RAND_MAX); - return T(fraction * range - delta); -} - -template -inline void randomInitWithFP(T *buffer, size_t size, int range = 1, - float delta = 0.0f) { - for (size_t index = 0; index < size; index++) - buffer[index] = randomFP(range, delta); -} - -template -inline bool compareDataFP(T *ref, T *dst, size_t size, float rtol = 1e-4f, - float atol = 1e-6f) { - for (size_t index = 0; index < size; index++) { - const float ref_f32 = static_cast(ref[index]); - const float dst_f32 = static_cast(dst[index]); - const double diff_f32 = dst_f32 - ref_f32; - const double gap = double(rtol) * (std::abs(ref_f32) > std::abs(dst_f32) - ? std::abs(ref_f32) - : std::abs(dst_f32)) + - atol; - bool good = std::abs(diff_f32) <= gap; - EXPECT_TRUE(good) << "Index: " << index << ", ref_f32=" << ref_f32 - << ", dst_f32=" << dst_f32; - if (!good) - return false; - } - return true; -} - -template -inline void testBrgemmRuntimeFP(int batch, int M, int N, int K, int LDA, - int LDB, int LDC, int strideA, int strideB, - float beta) { - using dnnl_f32_enum_val_t = std::integral_constant; - using dnnl_bf16_enum_val_t = std::integral_constant; - constexpr int dtypeA = - std::conditional::value, dnnl_f32_enum_val_t, - dnnl_bf16_enum_val_t>::type::value; - constexpr int dtypeB = dtypeA; - - T A[batch * M * K]; - T B[batch * K * N]; - float refC[M * N]; - - randomInitWithFP(A, batch * M * K, 10, 10.0f); - randomInitWithFP(B, batch * K * N, 10, 10.0f); - randomInitWithFP(refC, M * N, 10, 10.0f); - - float dstC[M * N]; - memcpy(dstC, refC, sizeof(float) * M * N); - - // Calculate reference - auto refHandle = dnnl_brgemm_dispatch_naive(M, N, K, LDA, LDB, LDC, strideA, - strideB, beta, dtypeA, dtypeB); - dnnl_brgemm_execute_naive(refHandle, A, 0, B, 0, refC, 0, batch); - - // Calculate destination - auto dstHandle = dnnl_brgemm_dispatch(M, N, K, LDA, LDB, LDC, strideA, - strideB, beta, dtypeA, dtypeB); - dnnl_brgemm_tileconfig(dstHandle); - dnnl_brgemm_execute(dstHandle, A, 0, B, 0, dstC, 0, batch); - dnnl_brgemm_tilerelease(); - - ASSERT_TRUE(compareDataFP(refC, dstC, M * N)); -} - -template -inline void randomInitWithInt(T *buffer, size_t size, int range, - int delta = 0) { - for (size_t index = 0; index < size; index++) - buffer[index] = rand() % range - delta; -} - -template inline bool compareDataInt(T *ref, T *dst, size_t size) { - for (size_t index = 0; index < size; index++) { - bool good = ref[index] == dst[index]; - EXPECT_TRUE(good) << "Index: " << index << ", ref=" << ref[index] - << ", dst=" << dst[index]; - if (!good) - return false; - } - return true; -} - -using namespace dnnl::impl::cpu::x64; - -inline void testBrgemmRuntimeInt(int batch, int M, int N, int K, int LDA, - int LDB, int LDC, int strideA, int strideB, - float beta) { - constexpr int dtypeA = 6; // dnnl_u8 enum val - constexpr int dtypeB = 5; // dnnl_s8 enum val - - uint8_t A[batch * M * K]; - int8_t B[batch * K * N]; - int32_t refC[M * N]; - - randomInitWithInt(A, batch * M * K, 100, 100); - randomInitWithInt(B, batch * K * N, 100, 100); - randomInitWithInt(refC, M * N, 500, 500); - - int32_t dstC[M * N]; - memcpy(dstC, refC, sizeof(int32_t) * M * N); - - // Calculate reference - auto refHandle = dnnl_brgemm_dispatch_naive(M, N, K, LDA, LDB, LDC, strideA, - strideB, beta, dtypeA, dtypeB); - dnnl_brgemm_execute_naive(refHandle, A, 0, B, 0, refC, 0, batch); - - // Calculate destination - auto dstHandle = dnnl_brgemm_dispatch(M, N, K, LDA, LDB, LDC, strideA, - strideB, beta, dtypeA, dtypeB); - dnnl_brgemm_tileconfig(dstHandle); - dnnl_brgemm_execute(dstHandle, A, 0, B, 0, dstC, 0, batch); - dnnl_brgemm_tilerelease(); - - ASSERT_TRUE(compareDataInt(refC, dstC, M * N)); -} - -TEST(ExecutionEngine, TestBrgemmRuntimeF32) { - gc_runtime_keep_alive = 0; - - bool supported_platform = - mayiuse(avx512_core_amx) || mayiuse(avx512_core) || mayiuse(avx2); - if (!supported_platform) - GTEST_SKIP(); - - srand(static_cast(time(nullptr))); - - constexpr int batch = 4; - constexpr int M = 32, N = 32, K = 32; - constexpr int LDA = 32, LDB = 32, LDC = 32; - constexpr int strideA = 1024, strideB = 1024; - - testBrgemmRuntimeFP(batch, M, N, K, LDA, LDB, LDC, strideA, strideB, - 0.0f); - testBrgemmRuntimeFP(batch, M, N, K, LDA, LDB, LDC, strideA, strideB, - 1.0f); -} - -TEST(ExecutionEngine, TestBrgemmRuntimeBF16) { - gc_runtime_keep_alive = 0; - - bool supported_platform = mayiuse(avx512_core_amx) || - mayiuse(avx512_core_bf16) || mayiuse(avx2_vnni_2); - if (!supported_platform) - GTEST_SKIP(); - - srand(static_cast(time(nullptr))); - - constexpr int batch = 4; - constexpr int M = 32, N = 32, K = 32; - constexpr int LDA = 32, LDB = 32, LDC = 32; - constexpr int strideA = 1024, strideB = 1024; - - testBrgemmRuntimeFP(batch, M, N, K, LDA, LDB, LDC, strideA, strideB, - 0.0f); - testBrgemmRuntimeFP(batch, M, N, K, LDA, LDB, LDC, strideA, strideB, - 1.0f); -} - -TEST(ExecutionEngine, TestBrgemmRuntimeU8S8) { - gc_runtime_keep_alive = 0; - - bool supported_platform = mayiuse(avx512_core_amx) || - mayiuse(avx512_core_vnni) || mayiuse(avx512_core) || - mayiuse(avx2_vnni_2) || mayiuse(avx2_vnni); - if (!supported_platform) - GTEST_SKIP(); - - srand(static_cast(time(nullptr))); - - constexpr int batch = 4; - constexpr int M = 32, N = 32, K = 32; - constexpr int LDA = 32, LDB = 32, LDC = 32; - constexpr int strideA = 1024, strideB = 1024; - - testBrgemmRuntimeInt(batch, M, N, K, LDA, LDB, LDC, strideA, strideB, 0.0f); - testBrgemmRuntimeInt(batch, M, N, K, LDA, LDB, LDC, strideA, strideB, 1.0f); -} diff --git a/test/mlir/unittests/ExecutionEngine/CMakeLists.txt b/test/mlir/unittests/ExecutionEngine/CMakeLists.txt deleted file mode 100644 index ec703d841..000000000 --- a/test/mlir/unittests/ExecutionEngine/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -add_mlir_unittest(GCExecutionEngineTests - JitWrapper.cpp - BrgemmRuntime.cpp -) - -get_property(GC_DNNL_INCLUDES GLOBAL PROPERTY GC_DNNL_INCLUDES) -target_include_directories(GCExecutionEngineTests PRIVATE ${GC_DNNL_INCLUDES}) - -target_link_libraries(GCExecutionEngineTests - PRIVATE - GcJitWrapper - GcCpuRuntime -) - -if(GC_ENABLE_IMEX) - add_subdirectory(IMEX) -endif() diff --git a/test/mlir/unittests/ExecutionEngine/IMEX/CMakeLists.txt b/test/mlir/unittests/ExecutionEngine/IMEX/CMakeLists.txt deleted file mode 100644 index 7110b5db0..000000000 --- a/test/mlir/unittests/ExecutionEngine/IMEX/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -add_mlir_unittest(GCExecutionEngineIMEXGpuTests - IMEXGpuOclRuntimeTest.cpp -) -target_link_libraries(GCExecutionEngineIMEXGpuTests - PRIVATE - GcJitWrapper - GcGpuOclRuntime - GcGpuPasses -) diff --git a/test/mlir/unittests/ExecutionEngine/IMEX/IMEXGpuOclRuntimeTest.cpp b/test/mlir/unittests/ExecutionEngine/IMEX/IMEXGpuOclRuntimeTest.cpp deleted file mode 100644 index d2d15d8a4..000000000 --- a/test/mlir/unittests/ExecutionEngine/IMEX/IMEXGpuOclRuntimeTest.cpp +++ /dev/null @@ -1,241 +0,0 @@ -//===-- GpuOclRuntime.cpp - -------------------------------------*- C++ -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include -#include - -#include "gc/ExecutionEngine/Driver/Driver.h" -#include "gc/ExecutionEngine/GPURuntime/GpuOclRuntime.h" -#include "gc/Transforms/Passes.h" -#include "gc/Utils/Error.h" - -#include "mlir/ExecutionEngine/MemRefUtils.h" -#include "mlir/IR/BuiltinOps.h" -#include "mlir/IR/MLIRContext.h" -#include "mlir/InitAllDialects.h" -#include "mlir/InitAllPasses.h" -#include "mlir/Parser/Parser.h" -#include "mlir/Target/LLVMIR/Export.h" -#include "mlir/Target/LLVMIR/ModuleTranslation.h" - -#include "llvm/Support/MemoryBuffer.h" -#include "llvm/Support/SourceMgr.h" - -#include "gtest/gtest.h" - -using namespace mlir; -using namespace gc::gpu; - -constexpr char addStatic[] = R"mlir( -module @test { - func.func @entry(%arg0: memref<64x64xf32>, %arg1: memref<64x64xf32>, %arg2: memref<64x64xf32>) { - %0 = bufferization.to_tensor %arg0 restrict : memref<64x64xf32> - %1 = bufferization.to_tensor %arg1 restrict : memref<64x64xf32> - %2 = tensor.empty() : tensor<64x64xf32> - %3 = linalg.add ins(%1, %0 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%2 : tensor<64x64xf32>) -> tensor<64x64xf32> - bufferization.materialize_in_destination %3 in restrict writable %arg2 : (tensor<64x64xf32>, memref<64x64xf32>) -> () - return - } -} -)mlir"; - -constexpr char addDynamic[] = R"mlir( -module @test { - func.func @entry(%arg0: memref, %arg1: memref, %arg2: memref) { - %c0 = arith.constant 0 : index - %c1 = arith.constant 1 : index - %0 = bufferization.to_tensor %arg0 restrict : memref - %1 = bufferization.to_tensor %arg1 restrict : memref - %d0 = memref.dim %arg0, %c0 : memref - %d1 = memref.dim %arg0, %c1 : memref - %2 = tensor.empty(%d0, %d1) : tensor - %3 = linalg.add ins(%1, %0 : tensor, tensor) outs(%2 : tensor) -> tensor - bufferization.materialize_in_destination %3 in restrict writable %arg2 : (tensor, memref) -> () - return - } -} -)mlir"; - -constexpr char matmulAddStatic[] = R"mlir( -module @fragment_name attributes {"#dlti.sys_spec" = #dlti.target_system_spec<"GPU" : #dlti.target_device_spec<#dlti.dl_entry<"max_work_group_size", 16 : i64>>>} { - func.func @entry(%arg0: memref<128x256xf16>, %arg1: memref<256x256xf16>, %arg2: memref<128x256xf16>) { - %0 = bufferization.to_tensor %arg0 restrict : memref<128x256xf16> - %1 = bufferization.to_tensor %arg1 restrict : memref<256x256xf16> - %2 = tensor.empty() : tensor<128x256xf16> - %cst = arith.constant 0.000000e+00 : f16 - %3 = linalg.fill ins(%cst : f16) outs(%2 : tensor<128x256xf16>) -> tensor<128x256xf16> - %4 = linalg.matmul ins(%0, %1 : tensor<128x256xf16>, tensor<256x256xf16>) outs(%3 : tensor<128x256xf16>) -> tensor<128x256xf16> - %5 = tensor.empty() : tensor<128x256xf16> - %6 = linalg.add ins(%4, %0 : tensor<128x256xf16>, tensor<128x256xf16>) outs(%5 : tensor<128x256xf16>) -> tensor<128x256xf16> - bufferization.materialize_in_destination %6 in restrict writable %arg2 : (tensor<128x256xf16>, memref<128x256xf16>) -> () - return - } -} -)mlir"; - -struct TestBase { - OclRuntime runtime = gcGetOrReport(OclRuntime::get()); - cl_command_queue queue = gcGetOrReport(runtime.createQueue()); - OclContext ctx{runtime, queue}; - MLIRContext mlirCtx{gc::initCompilerAndGetDialects()}; - - virtual void exec(std::shared_ptr &mod) = 0; - - virtual ~TestBase() { gcGetOrReport(runtime.releaseQueue(queue)); } - - OwningOpRef parse(const char *code) { - std::unique_ptr memBuf = - llvm::MemoryBuffer::getMemBuffer(code); - llvm::SourceMgr srcMgr; - srcMgr.AddNewSourceBuffer(std::move(memBuf), SMLoc()); - return parseSourceFile(srcMgr, &mlirCtx); - } -}; - -template struct TestAdd : TestBase { - static constexpr unsigned size = N * M; - float *buf0 = gcGetOrReport(runtime.usmNewDev(size)); - float *buf1 = gcGetOrReport(runtime.usmNewDev(size)); - float *buf2 = gcGetOrReport(runtime.usmNewShared(size)); - - explicit TestAdd() { - float cpuBuf[size]; - std::fill(cpuBuf, cpuBuf + size, 2.0f); - assert(runtime.usmCpy(ctx, cpuBuf, buf0, size)); - assert(runtime.usmCpy(ctx, cpuBuf, buf1, size)); - gcGetOrReport(ctx.finish()); - } - - ~TestAdd() override { - assert(runtime.usmFree(buf0)); - assert(runtime.usmFree(buf1)); - assert(runtime.usmFree(buf2)); - } - - void test(const char *code) { - OclModuleBuilderOpts opts; - opts.pipeline = [](OpPassManager &pm) { - mlir::gc::populateIMEXPipeline(pm, {}); - }; - OclModuleBuilder builder(parse(code), opts); - auto mod = gcGetOrReport(builder.build(runtime)); - exec(mod); - - float cpuBuf[size]; - assert(runtime.usmCpy(ctx, buf2, cpuBuf, size)); - gcGetOrReport(ctx.finish()); - - for (unsigned i = 0; i < size; i++) { - // std::cout << buf2[i] << " "; - assert(buf2[i] == 4.0f); - } - // std::cout << "\n"; - - for (float i : cpuBuf) { - // std::cout << i << " "; - assert(i == 4.0f); - } - } -}; - -template struct TestMatmulAdd : TestBase { - static constexpr unsigned size1 = N * M; - static constexpr unsigned size2 = M * M; - cl_half *buf0 = gcGetOrReport(runtime.usmNewDev(size1)); - cl_half *buf1 = gcGetOrReport(runtime.usmNewDev(size2)); - cl_half *buf2 = gcGetOrReport(runtime.usmNewShared(size1)); - - explicit TestMatmulAdd() { - cl_half cpuBuf[size2]; - std::fill(cpuBuf, cpuBuf + size2, 14336); - assert(runtime.usmCpy(ctx, cpuBuf, buf0, size1)); - assert(runtime.usmCpy(ctx, cpuBuf, buf1, size2)); - gcGetOrReport(ctx.finish()); - } - - ~TestMatmulAdd() override { - assert(runtime.usmFree(buf0)); - assert(runtime.usmFree(buf1)); - assert(runtime.usmFree(buf2)); - } - - void test(const char *code) { - OclModuleBuilderOpts opts; - opts.pipeline = [](OpPassManager &pm) { - mlir::gc::populateIMEXPipeline(pm, {}); - }; - OclModuleBuilder builder(parse(code), opts); - auto mod = gcGetOrReport(builder.build(runtime)); - exec(mod); - - gcGetOrReport(ctx.finish()); - for (unsigned i = 0; i < size1; i++) { - // std::cout << buf2[i] << " "; - assert(buf2[i] == 21512); - } - // std::cout << "\n"; - } -}; - -TEST(GpuOclRuntime, TestAddStatic) { - struct TestAddStatic1 : TestAdd<64> { - void exec(std::shared_ptr &mod) override { - assert(mod->isStatic); - StaticExecutor<3> exec(mod); - exec(ctx, buf0, buf1, buf2); - // Check if the executor is allocated on the stack - assert(exec.isSmall()); - } - } test1; - test1.test(addStatic); - - struct TestAddStatic2 : TestAdd<64> { - void exec(std::shared_ptr &mod) override { - assert(mod->isStatic); - StaticExecutor<3> exec(mod); - exec.arg(buf0); - exec.arg(buf1); - exec.arg(buf2); - // Check if the executor is allocated on the stack - assert(exec.isSmall()); - exec(ctx); - } - } test2; - test2.test(addStatic); -} - -TEST(GpuOclRuntime, TestAddDynamic) { - GTEST_SKIP() << "Dynamic shapes are not yet supported"; - struct TestAddDynamic : TestAdd<32, 64> { - void exec(std::shared_ptr &mod) override { - assert(!mod->isStatic); - int64_t shape[] = {32, 64}; - int64_t strides[] = {64, 1}; - DynamicExecutor<24> exec(mod); - exec.arg(buf0, 2, shape, strides); - exec.arg(buf1, 2, shape, strides); - exec.arg(buf2, 2, shape, strides); - exec(ctx); - // Check if the executor is allocated on the stack - assert(exec.isSmall()); - } - } test; - test.test(addDynamic); -} - -TEST(GpuOclRuntime, TestMatmulAddStatic) { - struct Test : TestMatmulAdd<128, 256> { - void exec(std::shared_ptr &mod) override { - assert(mod->isStatic); - StaticExecutor<3> exec(mod); - exec(ctx, buf0, buf1, buf2); - assert(exec.isSmall()); - } - } test; - test.test(matmulAddStatic); -} diff --git a/test/mlir/unittests/ExecutionEngine/JitWrapper.cpp b/test/mlir/unittests/ExecutionEngine/JitWrapper.cpp deleted file mode 100644 index f7b93eaa6..000000000 --- a/test/mlir/unittests/ExecutionEngine/JitWrapper.cpp +++ /dev/null @@ -1,70 +0,0 @@ -//===-- JitWrapper.cpp - Wrapper for JIT ------------------------*- C++ -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "gc/ExecutionEngine/Driver/Driver.h" -#include "mlir/AsmParser/AsmParser.h" -#include "mlir/ExecutionEngine/MemRefUtils.h" -#include "mlir/IR/AsmState.h" -#include "mlir/IR/BuiltinOps.h" -#include "mlir/IR/MLIRContext.h" -#include "mlir/Parser/Parser.h" -#include "mlir/Pass/PassManager.h" -#include "llvm/Support/ErrorOr.h" -#include "llvm/Support/MemoryBuffer.h" -#include "llvm/Support/SourceMgr.h" -#include "llvm/Support/raw_ostream.h" -#include "gtest/gtest.h" -#include - -using namespace mlir; - -static const char code1[] = R"mlir( -module { -llvm.mlir.global constant @__num_orig_num_args(3 : i32) : i32 -func.func @compute(%a: tensor<128xf32>, %b: tensor<128xf32>) -> tensor<128xf32> attributes { llvm.emit_c_interface } { - %out = tensor.empty() : tensor<128xf32> - %2 = linalg.add ins(%a, %b : tensor<128xf32>,tensor<128xf32>) outs(%out : tensor<128xf32>) -> tensor<128xf32> - return %2 : tensor<128xf32> -} -} -)mlir"; - -extern "C" { -extern int gc_runtime_keep_alive; -} - -TEST(ExecutionEngine, JitWrapper) { - gc_runtime_keep_alive = 0; - MLIRContext ctx{gc::initCompilerAndGetDialects()}; - std::unique_ptr ir_buffer = - llvm::MemoryBuffer::getMemBuffer(code1); - // Parse the input mlir. - llvm::SourceMgr sourceMgr; - sourceMgr.AddNewSourceBuffer(std::move(ir_buffer), llvm::SMLoc()); - mlir::OwningOpRef module = - mlir::parseSourceFile(sourceMgr, &ctx); - ASSERT_TRUE(module); - auto jited = gc::JitModule::create(module.get()); - bool jit_success = static_cast(jited); - if (!jit_success) { - auto err = jited.takeError(); - llvm::errs() << err; - llvm::consumeError(std::move(err)); - } - ASSERT_TRUE(jit_success); - OwningMemRef bufA{ - {128}, {128}, [](float &ptr, ArrayRef) { ptr = 1.0f; }}; - OwningMemRef bufB{ - {128}, {128}, [](float &ptr, ArrayRef idx) { ptr = idx[0]; }}; - OwningMemRef bufC{{128}, {128}}; - void *args[] = {&*bufA, &*bufB, &*bufC}; - jited.get()->call(args, 3); - for (int i = 0; i < 128; i++) { - ASSERT_EQ(bufC[{i}], 1.0f + i); - } -} diff --git a/test/mlir/unittests/Linalgx/CMakeLists.txt b/test/mlir/unittests/Linalgx/CMakeLists.txt deleted file mode 100644 index 7e5dc1db7..000000000 --- a/test/mlir/unittests/Linalgx/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -add_mlir_unittest(GCLinalgxTests - TestUtils.cpp -) -get_property(dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS) -target_link_libraries(GCLinalgxTests - PRIVATE - MLIRLinalgx - ${dialect_libs}) diff --git a/test/mlir/unittests/Linalgx/TestUtils.cpp b/test/mlir/unittests/Linalgx/TestUtils.cpp deleted file mode 100644 index a292dc1a9..000000000 --- a/test/mlir/unittests/Linalgx/TestUtils.cpp +++ /dev/null @@ -1,334 +0,0 @@ -//===-- TestUtils.cpp - Tests for Linalgx Utils -----------------*- C++ -*-===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "gtest/gtest.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/Math/IR/Math.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/Tensor/IR/Tensor.h" -#include "mlir/Dialect/Traits.h" -#include "mlir/IR/Attributes.h" -#include "mlir/IR/Builders.h" -#include "mlir/IR/BuiltinOps.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/Diagnostics.h" -#include "mlir/IR/DialectRegistry.h" -#include "mlir/IR/MLIRContext.h" -#include "mlir/IR/OperationSupport.h" -#include "mlir/IR/Value.h" -#include "mlir/Parser/Parser.h" -#include "mlir/Support/LogicalResult.h" - -#include "gc/Dialect/Linalgx/LinalgxDialect.h" -#include "gc/Dialect/Linalgx/Utils.h" - -using namespace mlir; -using namespace mlir::linalgx; - -// ----------------------------------------------------------------------------- -// Test Helpers -// ----------------------------------------------------------------------------- - -OpBuilder getMLIRBuilder(MLIRContext *context) { - DialectRegistry registry; - registry.insert(); - registry.insert(); - registry.insert(); - registry.insert(); - registry.insert(); - - context->appendDialectRegistry(registry); - context->loadAllAvailableDialects(); - - return OpBuilder(context); -} - -ModuleOp getTestModule(OpBuilder &builder, - function_ref createBody, - TypeRange types) { - auto loc = builder.getUnknownLoc(); - auto module = ModuleOp::create(builder.getUnknownLoc()); - // Create new function at the end of module - builder.setInsertionPointToEnd(module.getBody()); - - auto function = builder.create( - loc, "test", builder.getFunctionType(types, {})); - - // Create entry block for function - Block *body = function.addEntryBlock(); - builder.setInsertionPointToStart(body); - - createBody(builder, body->getArguments()); - builder.create(loc); - - return module; -} - -FailureOr createMatmulTestBody(OpBuilder &builder, - linalgx::PackingType opType, - Value tensorA, Value tensorB, - Value tensorC) { - FailureOr op = linalgx::makeGenericPackedMatmulOp( - builder, builder.getUnknownLoc(), opType, {tensorA, tensorB}, {tensorC}); - return op; -} - -bool compareModule(MLIRContext *context, ModuleOp *module, - const std::string &moduleStr) { - OwningOpRef moduleCmp = - parseSourceString(moduleStr, context); - return OperationEquivalence::isEquivalentTo( - *moduleCmp, *module, OperationEquivalence::ignoreValueEquivalence, - /*markEquivalent=*/nullptr, OperationEquivalence::Flags::IgnoreLocations); -} - -// ----------------------------------------------------------------------------- -// Tests -// ----------------------------------------------------------------------------- - -TEST(TestUtils, Matmul4D) { - MLIRContext context; - OpBuilder builder = getMLIRBuilder(&context); - // Test params - auto opType = linalgx::PackingType::MM4D; - Type shapeA = RankedTensorType::get({2, 8, 32, 16}, // - builder.getF32Type()); - Type shapeB = RankedTensorType::get({4, 8, 16, 32}, // - builder.getF32Type()); - Type shapeC = RankedTensorType::get({2, 4, 32, 32}, // - builder.getF32Type()); - // Expected IR - std::string moduleStr = R"mlir( - func.func @test(%arg0: tensor<2x8x32x16xf32>, %arg1: tensor<4x8x16x32xf32>, %arg2: tensor<2x4x32x32xf32>) { - %0 = linalg.generic { - indexing_maps = [affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d4, d1, d5)>, - affine_map<(d0, d1, d2, d3, d4, d5) -> (d2, d4, d5, d3)>, - affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d2, d1, d3)>], - iterator_types = ["parallel", "parallel", "parallel", "parallel", "reduction", "reduction"] - } - ins(%arg0, %arg1 : tensor<2x8x32x16xf32>, tensor<4x8x16x32xf32>) - outs(%arg2 : tensor<2x4x32x32xf32>) { - ^bb0(%in: f32, %in_0: f32, %out: f32): - %1 = arith.mulf %in, %in_0 : f32 - %2 = arith.addf %out, %1 : f32 - linalg.yield %2 : f32 - } -> tensor<2x4x32x32xf32> - return - } - )mlir"; - // Make test module - FailureOr op; - auto makeVnniMatmul = [&](OpBuilder &b, ValueRange vals) { - op = createMatmulTestBody(b, opType, vals[0], vals[1], vals[2]); - }; - ModuleOp module = - getTestModule(builder, makeVnniMatmul, {shapeA, shapeB, shapeC}); - // Get result - ASSERT_TRUE(succeeded(op)); - ASSERT_TRUE(isMatmulOp(*op)); - ASSERT_TRUE(isGenericPackedMatmulOp(*op, opType)); - ASSERT_FALSE(isGenericPackedMatmulOp(*op, linalgx::PackingType::VNNI_MM2D)); - ASSERT_FALSE(isGenericPackedMatmulOp(*op, linalgx::PackingType::VNNI_MM4D)); - ASSERT_FALSE(isGenericPackedMatmulOp(*op, linalgx::PackingType::VNNI_BRMM3D)); - ASSERT_TRUE(compareModule(&context, &module, moduleStr)); -} - -TEST(TestUtils, VnniMatmul2D) { - MLIRContext context; - OpBuilder builder = getMLIRBuilder(&context); - // Test params - auto opType = linalgx::PackingType::VNNI_MM2D; - Type shapeA = RankedTensorType::get({256, 64}, // - builder.getIntegerType(8)); - Type shapeB = RankedTensorType::get({16, 2, 8, 32, 4}, // - builder.getIntegerType(8)); - Type shapeC = RankedTensorType::get({256, 512}, // - builder.getIntegerType(32)); - // Expected IR - std::string moduleStr = R"mlir( - func.func @test(%arg0: tensor<256x64xi8>, %arg1: tensor<16x2x8x32x4xi8>, %arg2: tensor<256x512xi32>) { - %0 = linalg.generic { - indexing_maps = [affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d3 * 32 + d4 * 4 + d5)>, - affine_map<(d0, d1, d2, d3, d4, d5) -> (d1, d3, d4, d2, d5)>, - affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d1 * 32 + d2)>], - iterator_types = ["parallel", "parallel", "parallel", "reduction", "reduction", "reduction"] - } - ins(%arg0, %arg1 : tensor<256x64xi8>, tensor<16x2x8x32x4xi8>) - outs(%arg2 : tensor<256x512xi32>) { - ^bb0(%in: i8, %in_0: i8, %out: i32): - %4 = arith.extsi %in : i8 to i32 - %5 = arith.extsi %in_0 : i8 to i32 - %6 = arith.muli %4, %5 : i32 - %7 = arith.addi %out, %6 : i32 - linalg.yield %7 : i32 - } -> tensor<256x512xi32> - return - } - )mlir"; - // Make test module - FailureOr op; - auto makeVnniMatmul = [&](OpBuilder &b, ValueRange vals) { - op = createMatmulTestBody(b, opType, vals[0], vals[1], vals[2]); - }; - ModuleOp module = - getTestModule(builder, makeVnniMatmul, {shapeA, shapeB, shapeC}); - // Get result - ASSERT_TRUE(succeeded(op)); - ASSERT_TRUE(isMatmulOp(*op)); - ASSERT_TRUE(isGenericPackedMatmulOp(*op, opType)); - ASSERT_FALSE(isGenericPackedMatmulOp(*op, linalgx::PackingType::MM4D)); - ASSERT_FALSE(isGenericPackedMatmulOp(*op, linalgx::PackingType::VNNI_MM4D)); - ASSERT_FALSE(isGenericPackedMatmulOp(*op, linalgx::PackingType::VNNI_BRMM3D)); - ASSERT_TRUE(compareModule(&context, &module, moduleStr)); -} - -TEST(TestUtils, VnniMatmul4D) { - MLIRContext context; - OpBuilder builder = getMLIRBuilder(&context); - // Test params - auto opType = linalgx::PackingType::VNNI_MM4D; - Type shapeA = RankedTensorType::get({2, 8, 32, 32}, // - builder.getBF16Type()); - Type shapeB = RankedTensorType::get({4, 8, 16, 32, 2}, // - builder.getBF16Type()); - Type shapeC = RankedTensorType::get({2, 4, 32, 32}, // - builder.getBF16Type()); - // Expected IR - std::string moduleStr = R"mlir( - func.func @test(%arg0: tensor<2x8x32x32xbf16>, %arg1: tensor<4x8x16x32x2xbf16>, %arg2: tensor<2x4x32x32xbf16>) { - %0 = linalg.generic { - indexing_maps = [affine_map<(d0, d1, d2, d3, d4, d5, d6) -> (d0, d4, d1, d5 * 2 + d6)>, - affine_map<(d0, d1, d2, d3, d4, d5, d6) -> (d2, d4, d5, d3, d6)>, - affine_map<(d0, d1, d2, d3, d4, d5, d6) -> (d0, d2, d1, d3)>], - iterator_types = ["parallel", "parallel", "parallel", "parallel", "reduction", "reduction", "reduction"] - } - ins(%arg0, %arg1 : tensor<2x8x32x32xbf16>, tensor<4x8x16x32x2xbf16>) - outs(%arg2 : tensor<2x4x32x32xbf16>) { - ^bb0(%in: bf16, %in_0: bf16, %out: bf16): - %1 = arith.mulf %in, %in_0 : bf16 - %2 = arith.addf %out, %1 : bf16 - linalg.yield %2 : bf16 - } -> tensor<2x4x32x32xbf16> - return - } - )mlir"; - // Make test module - FailureOr op; - auto makeVnniMatmul = [&](OpBuilder &b, ValueRange vals) { - op = createMatmulTestBody(b, opType, vals[0], vals[1], vals[2]); - }; - ModuleOp module = - getTestModule(builder, makeVnniMatmul, {shapeA, shapeB, shapeC}); - // Get result - ASSERT_TRUE(succeeded(op)); - ASSERT_TRUE(isMatmulOp(*op)); - ASSERT_TRUE(isGenericPackedMatmulOp(*op, opType)); - ASSERT_FALSE(isGenericPackedMatmulOp(*op, linalgx::PackingType::MM4D)); - ASSERT_FALSE(isGenericPackedMatmulOp(*op, linalgx::PackingType::VNNI_MM2D)); - ASSERT_FALSE(isGenericPackedMatmulOp(*op, linalgx::PackingType::VNNI_BRMM3D)); - ASSERT_TRUE(compareModule(&context, &module, moduleStr)); -} - -TEST(TestUtils, VnniBatchReduceMatmul3D) { - MLIRContext context; - OpBuilder builder = getMLIRBuilder(&context); - // Test params - auto opType = linalgx::PackingType::VNNI_BRMM3D; - Type shapeA = RankedTensorType::get({512, 32, 64}, // - builder.getBF16Type()); - Type shapeB = RankedTensorType::get({512, 32, 128, 2}, // - builder.getBF16Type()); - Type shapeC = RankedTensorType::get({32, 128}, // - builder.getF32Type()); - // Expected IR - std::string moduleStr = R"mlir( - func.func @test(%arg0: tensor<512x32x64xbf16>, %arg1: tensor<512x32x128x2xbf16>, %arg2: tensor<32x128xf32>) { - %0 = linalg.generic { - indexing_maps = [affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d3 * 2 + d4)>, - affine_map<(d0, d1, d2, d3, d4) -> (d0, d3, d2, d4)>, - affine_map<(d0, d1, d2, d3, d4) -> (d1, d2)>], - iterator_types = ["reduction", "parallel", "parallel", "reduction", "reduction"]} - ins(%arg0, %arg1 : tensor<512x32x64xbf16>, tensor<512x32x128x2xbf16>) - outs(%arg2 : tensor<32x128xf32>) { - ^bb0(%in: bf16, %in_0: bf16, %out: f32): - %1 = arith.extf %in : bf16 to f32 - %2 = arith.extf %in_0 : bf16 to f32 - %3 = arith.mulf %1, %2 : f32 - %4 = arith.addf %out, %3 : f32 - linalg.yield %4 : f32 - } -> tensor<32x128xf32> - return - } - )mlir"; - // Make test module - FailureOr op; - auto makeVnniMatmul = [&](OpBuilder &b, ValueRange vals) { - op = createMatmulTestBody(b, opType, vals[0], vals[1], vals[2]); - }; - ModuleOp module = - getTestModule(builder, makeVnniMatmul, {shapeA, shapeB, shapeC}); - // Get result - ASSERT_TRUE(succeeded(op)); - ASSERT_TRUE(isMatmulOp(*op)); - ASSERT_TRUE(isGenericPackedMatmulOp(*op, opType)); - ASSERT_FALSE(isGenericPackedMatmulOp(*op, linalgx::PackingType::MM4D)); - ASSERT_FALSE(isGenericPackedMatmulOp(*op, linalgx::PackingType::VNNI_MM2D)); - ASSERT_FALSE(isGenericPackedMatmulOp(*op, linalgx::PackingType::VNNI_MM4D)); - ASSERT_TRUE(compareModule(&context, &module, moduleStr)); -} - -TEST(TestUtils, IsMatmulOp) { - MLIRContext context; - OpBuilder builder = getMLIRBuilder(&context); - // Test params - Type shapeA = RankedTensorType::get({512, 32}, // - builder.getBF16Type()); - Type shapeB = RankedTensorType::get({32, 128}, // - builder.getBF16Type()); - Type shapeC = RankedTensorType::get({512, 128}, // - builder.getF32Type()); - // Make test module - Operation *op; - auto makeLinalgOp = [&](OpBuilder &b, ValueRange vals) { - op = b.create( - b.getUnknownLoc(), ValueRange{vals[0], vals[1]}, ValueRange{vals[2]}); - }; - ModuleOp module = - getTestModule(builder, makeLinalgOp, {shapeA, shapeB, shapeC}); - // Get result - ASSERT_TRUE(op); - ASSERT_TRUE(module); - ASSERT_TRUE(isMatmulOp(op)); -} - -TEST(TestUtils, NotMatmulOp) { - MLIRContext context; - OpBuilder builder = getMLIRBuilder(&context); - // Test params - Type shapeA = RankedTensorType::get({512, 512}, // - builder.getBF16Type()); - Type shapeB = RankedTensorType::get({512, 512}, // - builder.getBF16Type()); - Type shapeC = RankedTensorType::get({512, 512}, // - builder.getF32Type()); - // Make test module - Operation *op; - auto makeLinalgOp = [&](OpBuilder &b, ValueRange vals) { - op = b.create( - b.getUnknownLoc(), ValueRange{vals[0], vals[1]}, ValueRange{vals[2]}); - }; - ModuleOp module = - getTestModule(builder, makeLinalgOp, {shapeA, shapeB, shapeC}); - // Get result - ASSERT_TRUE(op); - ASSERT_TRUE(module); - ASSERT_FALSE(isMatmulOp(op)); -} diff --git a/test/mlir/unittests/Transforms/CMakeLists.txt b/test/mlir/unittests/Transforms/CMakeLists.txt deleted file mode 100644 index 26445a970..000000000 --- a/test/mlir/unittests/Transforms/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -add_mlir_unittest(GCTransformsTests - StaticMemoryPlanning.cpp -) -target_link_libraries(GCTransformsTests - PRIVATE - GcPasses) - -if(GC_ENABLE_GPU) - add_subdirectory(GPU) -endif() \ No newline at end of file diff --git a/test/mlir/unittests/Transforms/GPU/CMakeLists.txt b/test/mlir/unittests/Transforms/GPU/CMakeLists.txt deleted file mode 100644 index 326948bb3..000000000 --- a/test/mlir/unittests/Transforms/GPU/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -add_mlir_unittest(GpuTransformsTests - GpuUtilsTest.cpp -) -target_link_libraries(GpuTransformsTests PRIVATE GcGpuPasses) -target_include_directories(GpuTransformsTests PRIVATE ${PROJECT_SOURCE_DIR}/lib) diff --git a/test/mlir/unittests/Transforms/GPU/GpuUtilsTest.cpp b/test/mlir/unittests/Transforms/GPU/GpuUtilsTest.cpp deleted file mode 100644 index 09b5e8aaa..000000000 --- a/test/mlir/unittests/Transforms/GPU/GpuUtilsTest.cpp +++ /dev/null @@ -1,99 +0,0 @@ -//===- GpuUtilsTest.cpp - Tests for GpuUtils-------------------------------===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include - -#include "gc/Transforms/GPU/GpuUtils.h" - -#include "gtest/gtest.h" - -TEST(testAdjustTiles, GputUtilsTest) { - bool print = false; - auto testAdjust = [print](int64_t totalSize, SmallVector &tiles, - const SmallVector &expected, - bool xeGpu = false) { - if (print) { - std::cout << totalSize << ": ["; - for (unsigned i = 0; i < tiles.size(); i++) { - std::cout << tiles[i] << (i + 1 < tiles.size() ? ", " : ""); - } - std::cout << "] -> ["; - } - - gc::adjustTiles(totalSize, tiles, xeGpu); - - if (print) { - for (unsigned i = 0; i < tiles.size(); i++) { - std::cout << tiles[i] << (i + 1 < tiles.size() ? ", " : ""); - } - std::cout << "]" << std::endl; - } - - EXPECT_EQ(tiles, expected); - }; - auto test = [testAdjust](int64_t totalSize, SmallVector tiles, - SmallVector expected, bool xeGpu = false) { - if (tiles.size() != 2 || tiles[0] == tiles[1]) { - testAdjust(totalSize, tiles, expected, xeGpu); - return; - } - SmallVector reversed(tiles.rbegin(), tiles.rend()); - testAdjust(totalSize, tiles, expected, xeGpu); - std::reverse(expected.begin(), expected.end()); - testAdjust(totalSize, reversed, expected, xeGpu); - }; - - test(8, {1, 1}, {1, 1}); - test(8, {1, 2}, {1, 2}); - test(8, {2, 2}, {2, 2}); - test(8, {1, 4}, {1, 4}); - test(8, {1, 8}, {1, 8}); - test(8, {2, 8}, {2, 4}); - test(8, {1, 32}, {1, 8}); - test(8, {2, 32}, {1, 8}); - test(8, {4, 32}, {1, 8}); - test(8, {8, 32}, {2, 4}); - test(8, {16, 32}, {2, 4}); - test(8, {32, 32}, {2, 4}); - test(8, {64, 32}, {4, 2}); - test(8, {128, 32}, {4, 2}); - - test(8192, {1024, 1024}, {64, 128}); - test(8192, {32, 32}, {32, 32}); - test(8192, {16, 64}, {16, 64}); - test(8192, {8, 128}, {8, 128}); - test(8192, {4, 256}, {4, 256}); - test(8192, {2, 512}, {2, 512}); - test(8192, {1, 1024}, {1, 1024}); - test(8192, {512, 2}, {512, 2}); - test(8192, {256, 4}, {256, 4}); - test(8192, {128, 8}, {128, 8}); - test(8192, {64, 16}, {64, 16}); - test(8192, {32, 32}, {32, 32}); - - test(16384, {1, 1, 1}, {1, 1, 1}); - test(16384, {1, 2, 4}, {1, 2, 4}); - test(16384, {2, 4, 8}, {2, 4, 8}); - test(16384, {4, 8, 16}, {4, 8, 16}); - test(16384, {8, 16, 32}, {8, 16, 32}); - test(16384, {16, 32, 64}, {16, 32, 32}); - test(16384, {32, 64, 128}, {8, 32, 64}); - test(16384, {64, 128, 256}, {8, 32, 64}); - test(16384, {128, 256, 512}, {4, 64, 64}); - - test(16384, {7, 17, 111}, {7, 17, 111}); - test(16384, {7, 117, 111}, {7, 39, 37}); - test(16384, {6, 256, 512}, {1, 128, 128}); - test(16384, {60, 128, 512}, {4, 32, 128}); - test(16384, {119, 256, 512}, {7, 32, 64}); - test(16384, {109, 256, 512}, {109, 8, 16}); - - test(16384, {8, 16, 32, 256, 512}, {1, 1, 1, 128, 128}, true); - test(16384, {8, 16, 32, 1024, 256}, {1, 1, 1, 256, 64}, true); - test(16384, {8, 16, 32, 16, 4096}, {1, 1, 1, 8, 2048}, true); -} diff --git a/test/mlir/unittests/Transforms/StaticMemoryPlanning.cpp b/test/mlir/unittests/Transforms/StaticMemoryPlanning.cpp deleted file mode 100644 index b07bd8932..000000000 --- a/test/mlir/unittests/Transforms/StaticMemoryPlanning.cpp +++ /dev/null @@ -1,205 +0,0 @@ -//===- StaticMemoryPlanning.cpp - Tests for StaticMemoryPlanning-----------===// -// -// This file is licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -#include "gc/Transforms/StaticMemoryPlanning.h" - -#include "gtest/gtest.h" -#include - -using namespace mlir::gc; -TEST(static_memory_planner, TestStaticMemoryPlanning) { - /* - {0} {160} {280} {450} - | 0 | 1 | 2 | - | 3 | 4/5 | | - | 7 | 6 | - {100} - */ - memoryplan::Traces traces = {{0, 100}, {1, 120}, {2, 100}, {0, 0}, - {3, 50}, {4, 60}, {2, 0}, {4, 0}, - {8, 100}, {8, 0}, {5, 60}, {5, 0}, - {1, 0}, {6, 350}, {3, 0}, {7, 100}}; - std::unordered_map out; - std::unordered_map> inplace_selection; - size_t total = memoryplan::scheduleMemoryAllocations( - traces, 1, false, memoryplan::InplaceInfoMap(), out, inplace_selection); - std::unordered_map expected_out = { - {0, 0}, {1, 160}, {2, 280}, {3, 0}, {4, 100}, - {5, 100}, {6, 100}, {7, 0}, {8, 280}}; - EXPECT_EQ(total, 450UL); - EXPECT_EQ(out, expected_out); - - total = memoryplan::scheduleMemoryAllocations( - traces, 1, true, memoryplan::InplaceInfoMap(), out, inplace_selection); - expected_out = {{0, 0}, {1, 160}, {2, 280}, {3, 0}, {4, 100}, - {5, 100}, {6, 100}, {7, 0}, {8, 280}}; - EXPECT_EQ(total, 450UL); - EXPECT_EQ(out, expected_out); -} - -TEST(static_memory_planner, TestStaticMemoryPlanningInplace) { - using namespace memoryplan; - using inplace_outdata = std::unordered_map>; - using inplace_data = InplaceInfoMap; - // simple inplace (need merge + split) - { - memoryplan::Traces traces = {{1, 120}, {2, 100}, {3, 200}, {1, 0}, - {2, 0}, {3, 0}, {4, 220}, {4, 0}}; - std::unordered_map out; - inplace_outdata inplace_selection; - inplace_data inplace_hint = { - {3, {{1, InplaceKind::FREE}, {2, InplaceKind::FREE}}}}; - size_t total = memoryplan::scheduleMemoryAllocations( - traces, 1, false, inplace_hint, out, inplace_selection); - std::unordered_map expected_out = { - {1, 0}, {2, 120}, {3, 0}, {4, 0}}; - EXPECT_EQ(total, 220UL); - EXPECT_EQ(out, expected_out); - - inplace_outdata expected_inplace = {{3, {1, 2}}}; - EXPECT_EQ(inplace_selection, expected_inplace); - } - - // inplace extend - { - memoryplan::Traces traces = {{1, 120}, {2, 100}, {4, 250}, {3, 250}, - {1, 0}, {2, 0}, {3, 0}, {4, 0}}; - std::unordered_map out; - inplace_outdata inplace_selection; - inplace_data inplace_hint = { - {3, {{1, InplaceKind::FREE}, {2, InplaceKind::FREE}}}}; - size_t total = memoryplan::scheduleMemoryAllocations( - traces, 1, false, inplace_hint, out, inplace_selection); - std::unordered_map expected_out = { - {1, 0}, {2, 120}, {3, 0}, {4, 250}}; - EXPECT_EQ(total, 500UL); - EXPECT_EQ(out, expected_out); - - inplace_outdata expected_inplace = {{3, {1, 2}}}; - EXPECT_EQ(inplace_selection, expected_inplace); - } - - // inplace 2 buffers into one - { - memoryplan::Traces traces = {{1, 120}, {2, 100}, {3, 150}, {4, 50}, {5, 10}, - {1, 0}, {2, 0}, {3, 0}, {4, 0}, {5, 0}}; - std::unordered_map out; - inplace_outdata inplace_selection; - inplace_data inplace_hint = { - {3, {{1, InplaceKind::FREE}, {2, InplaceKind::FREE}}}, - {4, {{1, InplaceKind::FREE}, {2, InplaceKind::FREE}}}}; - size_t total = memoryplan::scheduleMemoryAllocations( - traces, 1, false, inplace_hint, out, inplace_selection); - std::unordered_map expected_out = { - {1, 0}, {2, 120}, {3, 0}, {4, 150}, {5, 220}}; - EXPECT_EQ(total, 230UL); - EXPECT_EQ(out, expected_out); - - inplace_outdata expected_inplace = {{3, {1, 2}}, {4, {2}}}; - EXPECT_EQ(inplace_selection, expected_inplace); - } - - // inplace 2 buffers into one, but require zero offset - { - memoryplan::Traces traces = {{1, 120}, {2, 100}, {3, 150}, {4, 50}, {5, 10}, - {1, 0}, {2, 0}, {3, 0}, {4, 0}, {5, 0}}; - std::unordered_map out; - inplace_outdata inplace_selection; - inplace_data inplace_hint = { - {3, {{1, InplaceKind::FREE}, {2, InplaceKind::ZERO_OFFSET}}}, - {4, {{1, InplaceKind::FREE}, {2, InplaceKind::FREE}}}}; - size_t total = memoryplan::scheduleMemoryAllocations( - traces, 1, false, inplace_hint, out, inplace_selection); - std::unordered_map expected_out = { - {1, 0}, {2, 150}, {3, 0}, {4, 150}, {5, 250}}; - EXPECT_EQ(total, 260UL); - EXPECT_EQ(out, expected_out); - - inplace_outdata expected_inplace = {{3, {1}}, {4, {2}}}; - EXPECT_EQ(inplace_selection, expected_inplace); - } - - // inplace 2 buffers into one, but require zero offset for split buffer - // buffer4 cannot reuse buffer 2 because it requires zero offset - { - memoryplan::Traces traces = {{1, 120}, {2, 100}, {3, 150}, {4, 50}, {5, 10}, - {1, 0}, {2, 0}, {3, 0}, {4, 0}, {5, 0}}; - std::unordered_map out; - inplace_outdata inplace_selection; - inplace_data inplace_hint = { - {3, {{1, InplaceKind::FREE}, {2, InplaceKind::FREE}}}, - {4, {{1, InplaceKind::FREE}, {2, InplaceKind::ZERO_OFFSET}}}}; - size_t total = memoryplan::scheduleMemoryAllocations( - traces, 1, false, inplace_hint, out, inplace_selection); - std::unordered_map expected_out = { - {1, 0}, {2, 120}, {3, 0}, {4, 220}, {5, 270}}; - EXPECT_EQ(total, 280UL); - EXPECT_EQ(out, expected_out); - - inplace_outdata expected_inplace = {{3, {1, 2}}}; - EXPECT_EQ(inplace_selection, expected_inplace); - } - - // merge free to the right - { - memoryplan::Traces traces = {{1, 120}, {2, 100}, {3, 150}, {2, 0}, {4, 150}, - {5, 10}, {1, 0}, {3, 0}, {4, 0}, {5, 0}}; - std::unordered_map out; - inplace_outdata inplace_selection; - inplace_data inplace_hint = {{4, {{1, InplaceKind::FREE}}}}; - size_t total = memoryplan::scheduleMemoryAllocations( - traces, 1, false, inplace_hint, out, inplace_selection); - std::unordered_map expected_out = { - {1, 0}, {2, 120}, {3, 220}, {4, 0}, {5, 150}}; - EXPECT_EQ(total, 370UL); - EXPECT_EQ(out, expected_out); - - inplace_outdata expected_inplace = {{4, {1}}}; - EXPECT_EQ(inplace_selection, expected_inplace); - } - - // perfect matches - { - memoryplan::Traces traces = {{1, 120}, {2, 100}, {3, 100}, {4, 120}, - {1, 0}, {2, 0}, {3, 0}, {4, 0}, - {5, 200}, {5, 0}}; - std::unordered_map out; - inplace_outdata inplace_selection; - inplace_data inplace_hint = { - {3, {{1, InplaceKind::FREE}, {2, InplaceKind::FREE}}}, - {4, {{1, InplaceKind::FREE}, {2, InplaceKind::FREE}}}}; - size_t total = memoryplan::scheduleMemoryAllocations( - traces, 1, false, inplace_hint, out, inplace_selection); - std::unordered_map expected_out = { - {1, 0}, {2, 120}, {3, 120}, {4, 0}, {5, 0}}; - EXPECT_EQ(total, 220UL); - EXPECT_EQ(out, expected_out); - - inplace_outdata expected_inplace = {{3, {2}}, {4, {1}}}; - EXPECT_EQ(inplace_selection, expected_inplace); - } - - // selected inputs - { - memoryplan::Traces traces = {{1, 120}, {2, 100}, {3, 100}, {4, 120}, - {1, 0}, {2, 0}, {3, 0}, {4, 0}, - {5, 200}, {5, 0}}; - std::unordered_map out; - inplace_outdata inplace_selection; - inplace_data inplace_hint = {{3, {{1, InplaceKind::FREE}}}, - {4, {{2, InplaceKind::FREE}}}}; - size_t total = memoryplan::scheduleMemoryAllocations( - traces, 1, false, inplace_hint, out, inplace_selection); - std::unordered_map expected_out = { - {1, 0}, {2, 120}, {3, 0}, {4, 120}, {5, 0}}; - EXPECT_EQ(total, 240UL); - EXPECT_EQ(out, expected_out); - - inplace_outdata expected_inplace = {{3, {1}}, {4, {2}}}; - EXPECT_EQ(inplace_selection, expected_inplace); - } -}