LLVM test suite #30
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # | |
| # Copyright (c) 2021-present, Trail of Bits, Inc. | |
| # All rights reserved. | |
| # | |
| # This source code is licensed in accordance with the terms specified in | |
| # the LICENSE file found in the root directory of this source tree. | |
| # | |
| name: "LLVM test suite" | |
| on: | |
| workflow_run: | |
| workflows: [Build] | |
| types: | |
| - completed | |
| branches: | |
| - "master" | |
| jobs: | |
| build: | |
| name: "Build VAST" | |
| strategy: | |
| matrix: | |
| llvm-version: [17] | |
| image-version: [22.04] | |
| runs-on: ubuntu-${{ matrix.image-version }} | |
| timeout-minutes: 60 | |
| container: | |
| image: | |
| ghcr.io/trailofbits/vast-ubuntu-${{ matrix.image-version }}-llvm-${{ matrix.llvm-version }}-dev:latest | |
| env: | |
| CMAKE_PREFIX_PATH: "/usr/lib/llvm-${{ matrix.llvm-version }}/lib/cmake/mlir/;/usr/lib/llvm-${{ matrix.llvm-version }}/lib/cmake/clang/" | |
| TOOLCHAIN: ${{ github.workspace }}/cmake/lld.toolchain.cmake | |
| LLVM_EXTERNAL_LIT: "/usr/local/bin/lit" | |
| steps: | |
| - name: Clone the VAST repository | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| fetch-depth: 1 | |
| - name: Configure build | |
| run: | | |
| cmake --preset ninja-multi-default --toolchain ${TOOLCHAIN} \ | |
| -DCMAKE_VERBOSE_MAKEFILE=True \ | |
| -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH} \ | |
| -DLLVM_EXTERNAL_LIT=${LLVM_EXTERNAL_LIT} | |
| - name: Build release | |
| run: | | |
| cmake --build --preset ninja-rel -j $(nproc) | |
| cpack -G TXZ --config ./builds/ninja-multi-default/CPackConfig.cmake | |
| - name: Upload VAST build artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: VAST | |
| path: ./builds/ninja-multi-default/package/* | |
| retention-days: 1 | |
| test: | |
| name: "Run LLVM test suite" | |
| needs: build | |
| strategy: | |
| matrix: | |
| llvm-version: [17] | |
| image-version: [22.04] | |
| vast-target: ['hl', 'llvm', 'bin'] | |
| runs-on: ubuntu-${{ matrix.image-version }} | |
| timeout-minutes: 60 | |
| container: | |
| image: | |
| ghcr.io/trailofbits/vast-ubuntu-${{ matrix.image-version }}-llvm-${{ matrix.llvm-version }}-dev:latest | |
| steps: | |
| - name: Fetch VAST artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: VAST | |
| - name: Unpack VAST | |
| run: | | |
| mkdir vast && tar -xf VAST-* -C vast --strip-components=1 | |
| - name: Export vast binaries | |
| run: | | |
| echo "${PWD}/vast/bin/" >> $GITHUB_PATH | |
| - name: Install test suite dependencies | |
| run: | | |
| apt-get update | |
| apt-get -y install clang | |
| pip3 install lit psutil | |
| - name: Clone test suite repository | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: trailofbits/vast-llvm-test-suite | |
| ref: main | |
| path: llvm-test-suite | |
| fetch-depth: 1 | |
| - name: Run test-suite | |
| continue-on-error: true | |
| run: | | |
| mkdir llvm-test-suite/build && cd llvm-test-suite/build/ | |
| cmake ../ -DCMAKE_BUILD_TYPE=Release \ | |
| -DCMAKE_C_COMPILER=$(readlink -f ../tools/compiler) \ | |
| -DCMAKE_CXX_COMPILER=$(readlink -f ../tools/compiler++) \ | |
| -C=../cmake/caches/O3.cmake \ | |
| -DTEST_SUITE_COLLECT_CODE_SIZE=OFF \ | |
| -DTEST_SUITE_SUBDIRS=SingleSource \ | |
| -DVAST_TEST_SUITE_ENABLE=ON \ | |
| -DVAST_TEST_SUITE_TARGET=${{ matrix.vast-target }} | |
| make -j $(nproc) -k | |
| - name: Collect run data (mlir) | |
| if: matrix.vast-target != 'bin' | |
| working-directory: ./llvm-test-suite/build | |
| run: python3 ../utils/mlir_compile_json_gen.py >> ../../results-${{ matrix.vast-target }}.json | |
| - name: Collect run data (bin) | |
| continue-on-error: true | |
| if: matrix.vast-target == 'bin' | |
| working-directory: ./llvm-test-suite/build | |
| run: lit --timeout=60 -v -o ../../results-${{ matrix.vast-target }}.json . | |
| - name: Upload results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| path: ./results-${{ matrix.vast-target }}.json | |
| name: results-${{ matrix.vast-target }} | |
| eval: | |
| name: "Eval test results" | |
| needs: test | |
| strategy: | |
| matrix: | |
| image-version: [22.04] | |
| runs-on: ubuntu-${{ matrix.image-version }} | |
| steps: | |
| - name: Fetch result artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: results-* | |
| merge-multiple: true | |
| - name: Install evaluator dependencies | |
| run: | | |
| pip3 install pandas scipy tabulate | |
| - name: Clone test suite repository | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: trailofbits/vast-llvm-test-suite | |
| sparse-checkout: utils/vast_compare.py | |
| ref: main | |
| path: llvm-test-suite | |
| fetch-depth: 1 | |
| - name: Generate the results | |
| run: | | |
| python3 llvm-test-suite/utils/vast_compare.py \ | |
| --columns vast-hl,vast-llvm,vast-bin \ | |
| --files results-hl.json,results-llvm.json,results-bin.json \ | |
| --output single-source-results | |
| - name: Post results as artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: llvm-test-suite-results | |
| path: | | |
| ./single-source-results.csv | |
| ./single-source-results.md |