dev-long-tests.yml

name: dev-long-tests
# Tests generally longer than 10mn

concurrency:
  group: long-${{ github.ref }}
  cancel-in-progress: true

on:
  pull_request:
    branches: [ dev, release, actionsTest ]

permissions: read-all

jobs:
  # lasts ~7mn
  make-all:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
    - name: make all
      run: make all

  # lasts ~19mn
  make-test:
    runs-on: ubuntu-latest
    env:
      DEVNULLRIGHTS: 1
      READFROMBLOCKDEVICE: 1
    steps:
    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
    - name: make test
      run: |
        make -j test
        make -j zstd
        ./tests/test_process_substitution.bash ./zstd

  # lasts ~16mn
  make-test-macos:
    runs-on: macos-latest
    steps:
    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
    - name: make test on macos
      run: make -j test

  # lasts ~10mn
  make-test-32bit:
    runs-on: ubuntu-latest
    env:
      DEVNULLRIGHTS: 1
      READFROMBLOCKDEVICE: 1
    steps:
    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
    - name: make test # note: `make -j test success` seems to require a clean state
      run: |
        sudo apt-get -qqq update
        make libc6install
        make clean
        CFLAGS="-m32 -O2" make -j test V=1

  # lasts ~7mn
  test-largeDictionary:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
    - name: largeDictionary
      run: |
        CFLAGS="-Werror -O3" make -j -C tests test-largeDictionary

  # lasts ~9mn
  no-intrinsics-fuzztest:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
    - name: no intrinsics fuzztest
      run: MOREFLAGS="-DZSTD_NO_INTRINSICS" make -j -C tests fuzztest

  # lasts ~8mn
  tsan-zstreamtest:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
    - name: thread sanitizer zstreamtest
      run: CC=clang ZSTREAM_TESTTIME=-T3mn make tsan-test-zstream

  uasan-zstreamtest:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
    - name: ub + address sanitizer on zstreamtest
      run: CC=clang make uasan-test-zstream

  # lasts ~11mn
  tsan-fuzztest:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
    - name: thread sanitizer fuzztest
      run: CC=clang make -j tsan-fuzztest

  big-tests-zstreamtest32:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
    - name: zstream tests in 32bit mode, with big tests
      run: |
        sudo apt-get -qqq update
        make libc6install
        CC=clang make -C tests test-zstream32 FUZZER_FLAGS="--big-tests"

  # lasts ~13mn
  gcc-8-asan-ubsan-testzstd:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
    - name: gcc-8 + ASan + UBSan + Test Zstd
      # See https://askubuntu.com/a/1428822
      run: |
        echo "deb [arch=amd64] http://archive.ubuntu.com/ubuntu focal main universe" | sudo tee -a /etc/apt/sources.list
        sudo apt-get -qqq update
        make gcc8install
        CC=gcc-8 make -j uasan-test-zstd </dev/null V=1

  clang-asan-ubsan-testzstd:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
    - name: clang + ASan + UBSan + Test Zstd
      run: CC=clang make -j uasan-test-zstd </dev/null V=1

  gcc-asan-ubsan-testzstd-32bit:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
    - name: ASan + UBSan + Test Zstd, 32bit mode
      run: |
        sudo apt-get -qqq update
        make libc6install
        make -j uasan-test-zstd32 V=1

    # Note : external libraries must be turned off when using MSAN tests,
    # because they are not msan-instrumented,
    # so any data coming from these libraries is always considered "uninitialized"

  gcc-8-asan-ubsan-fuzz:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
    - name: gcc-8 + ASan + UBSan + Fuzz Test
      # See https://askubuntu.com/a/1428822
      run: |
        echo "deb [arch=amd64] http://archive.ubuntu.com/ubuntu focal main universe" | sudo tee -a /etc/apt/sources.list
        sudo apt-get -qqq update
        make gcc8install
        CC=gcc-8 FUZZER_FLAGS="--long-tests" make -j clean uasan-fuzztest

  clang-asan-ubsan-fuzz:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
    - name: clang + ASan + UBSan + Fuzz Test
      run: CC=clang FUZZER_FLAGS="--long-tests" make -j clean uasan-fuzztest

  # the only job decoding real legacy frames under sanitizers.
  # fuzzers do build these decoders, but only reach them if an input
  # happens to carry a legacy magic number.
  clang-asan-ubsan-legacy:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
    - name: clang + ASan + UBSan + legacy decoders
      run: CC=clang make uasan-test-legacy

  gcc-asan-ubsan-fuzz32:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
    - name: ASan + UBSan + Fuzz Test 32bit
      run: |
        sudo apt-get -qqq update
        make libc6install
        CFLAGS="-O3 -m32" FUZZER_FLAGS="--long-tests" make -j uasan-fuzztest

  clang-asan-fuzz32:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
    - name: clang + ASan + Fuzz Test 32bit
      run: |
        sudo apt-get -qqq update
        make libc6install
        CC=clang CFLAGS="-O3 -m32" FUZZER_FLAGS="--long-tests" make -j asan-fuzztest

# The following test seems to have issues on github CI specifically,
# it does not provide the `__mulodi4` instruction emulation
# required for signed 64-bit multiplication.
# Replaced by asan-only test (above)
#
#  clang-asan-ubsan-fuzz32:
#    runs-on: ubuntu-20.04
#    steps:
#    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
#    - name: clang + ASan + UBSan + Fuzz Test 32bit
#      run: |
#        sudo apt-get -qqq update
#        make libc6install
#        CC=clang CFLAGS="-O3 -m32" FUZZER_FLAGS="--long-tests" make uasan-fuzztest

  asan-ubsan-regression:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
    - name: ASan + UBSan + Regression Test
      run: make -j uasanregressiontest

  clang-asan-ubsan-regression:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
    - name: clang + ASan + UBSan + Regression Test
      run: CC=clang make -j uasanregressiontest

  msan-regression:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
    - name: MSan + Regression Test
      run: make -j msanregressiontest

  clang-msan-fuzz-unoptimized:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
      - name: clang + MSan + Fuzz Test
        # -O0 targets the unit tests on unoptimized code: random fuzzing is kept short.
        # clang-msan-fuzz runs the long random fuzzing.
        run: |
          sudo apt-get -qqq update
          sudo apt-get install clang
          CC=clang MOREFLAGS="-O0" FUZZERTEST=-T60s ZSTREAM_TESTTIME=-T30s make clean msan-fuzztest

  clang-msan-fuzz:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
    - name: clang + MSan + Fuzz Test
      run: |
        sudo apt-get -qqq update
        sudo apt-get install clang
        CC=clang FUZZER_FLAGS="--long-tests" make clean msan-fuzztest

  clang-msan-testzstd:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
    - name: clang + MSan + Test Zstd
      # ZSTDRTTEST= : skip the large data tests of playTests.sh, too slow under MSan.
      # They still run in the make-test* and ASan + UBSan testzstd jobs.
      run: |
        CC=clang make -j msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0 ZSTDRTTEST= V=1

  armfuzz:
      runs-on: ubuntu-latest
      steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
      - name: Qemu ARM emulation + Fuzz Test
        run: |
          sudo apt-get -qqq update
          make arminstall
          make armfuzz

  valgrind-fuzz-test:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
    - name: valgrind + fuzz test stack mode    # ~ 7mn
      shell: 'script -q -e -c "bash {0}"'
      run: |
        sudo apt-get -qqq update
        make valgrindinstall
        make -C tests test-valgrind
        make clean
        make -C tests test-fuzzer-stackmode

  mingw-long-test:
    runs-on: windows-latest
    defaults:
      run:
        shell: msys2 {0}
    steps:
    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
    - uses: msys2/setup-msys2@66cd2cce69caa17b53920067426061ca1de3a884 # tag=v2.32.0
      with:
        msystem: MINGW64
        install: make
        update: true
    # Based on https://ariya.io/2020/07/on-github-actions-with-msys2
    - name: install mingw gcc
      run: pacman --noconfirm -S gcc
    - name: MINGW64 gcc fuzztest
      run: |
        export CC="gcc"
        export CXX="g++"
        export FUZZERTEST="-T2mn"
        export ZSTREAM_TESTTIME="-T2mn"
        echo "Testing $CC $CXX MINGW64"
        make -v
        $CC --version
        $CXX --version
        make -C tests fuzztest

  # lasts ~20mn
  oss-fuzz:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        sanitizer: [address, undefined, memory]
    steps:
    - name: Build Fuzzers (${{ matrix.sanitizer }})
      id: build
      uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master
      with:
        oss-fuzz-project-name: 'zstd'
        dry-run: false
        sanitizer: ${{ matrix.sanitizer }}
    - name: Run Fuzzers (${{ matrix.sanitizer }})
      uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
      with:
        oss-fuzz-project-name: 'zstd'
        fuzz-seconds: 600
        dry-run: false
        sanitizer: ${{ matrix.sanitizer }}
    - name: Upload Crash
      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # tag=v7.0.1
      if: failure() && steps.build.outcome == 'success'
      with:
        name: ${{ matrix.sanitizer }}-artifacts
        path: ./out/artifacts