tests: add widening and other tests #7542
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "QA" | |
| # Runs on each push and tests flowR for the default configuration. | |
| # Also publishes a pre-release build to npm and updates the documentation with new performance benchmarks. | |
| # Depending on the targets, etc. this may perform a different subset of steps! | |
| 'on': | |
| push: | |
| branches-ignore: | |
| - 'gh-pages' | |
| paths-ignore: | |
| - '**/*.md' | |
| - 'LICENSE' | |
| - '.gitignore' | |
| pull_request: | |
| types: [ opened, synchronize ] | |
| branches: [ main ] | |
| paths-ignore: | |
| - '**/*.md' | |
| - 'LICENSE' | |
| - '.gitignore' | |
| workflow_dispatch: | |
| inputs: | |
| force-full: | |
| description: "Force all steps" | |
| required: true | |
| default: false | |
| type: boolean | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: write | |
| jobs: | |
| debug: | |
| name: "🔍 Debug" | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 10 | |
| steps: | |
| - name: "🛒 Checkout Repository" | |
| uses: actions/checkout@v6 | |
| - name: "🌍 Load Versions to Use" | |
| run: bash .github/workflows/scripts/global-configuration.sh | |
| - name: "🔍 Debug state" | |
| run: | | |
| echo "::group::Versions" | |
| echo "Node Version: $ACTION_NODE_VERSION" | |
| echo "R Version: $ACTION_R_VERSION" | |
| echo "::endgroup::" | |
| echo "::group::Git Info" | |
| echo "Event: ${{ github.event_name }}" | |
| echo "Ref: ${{ github.ref }}" | |
| echo "Base Ref: ${{ github.base_ref }}" | |
| echo "Head Commit: ${{ github.sha }}" | |
| echo "::endgroup::" | |
| lint: | |
| needs: [ debug ] | |
| name: "👩🏫 Linting (local)" | |
| if: ${{ github.event_name != 'pull_request' || github.event.pull_request.base.ref != 'main' }} | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 20 | |
| steps: | |
| - name: "🛒 Checkout Repository" | |
| uses: actions/checkout@v6 | |
| with: | |
| submodules: true | |
| - name: "🌍 Load Versions to Use" | |
| run: bash .github/workflows/scripts/global-configuration.sh | |
| - name: "⬇️ Use Node.js" | |
| uses: actions/setup-node@v6 | |
| with: | |
| node-version: ${{ env.ACTION_NODE_VERSION }} | |
| registry-url: "https://registry.npmjs.org/" | |
| cache: npm | |
| - name: "👩🏫 Run linter" | |
| run: bash .github/workflows/scripts/run-flowr-command.sh lint-local | |
| lint-main: | |
| needs: [ debug ] | |
| name: "👩🏫 Linting on Main" | |
| if: ${{ (github.event_name == 'pull_request' && github.event.pull_request.base.ref == 'main') || github.base_ref == 'main' }} | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 20 | |
| steps: | |
| - name: "🛒 Checkout Repository" | |
| uses: actions/checkout@v6 | |
| with: | |
| submodules: true | |
| - name: "🌍 Load Versions to Use" | |
| run: bash .github/workflows/scripts/global-configuration.sh | |
| - name: "⬇️ Use Node.js" | |
| uses: actions/setup-node@v6 | |
| with: | |
| node-version: ${{ env.ACTION_NODE_VERSION }} | |
| registry-url: "https://registry.npmjs.org/" | |
| cache: npm | |
| - name: "👩🏫 Run linter" | |
| run: bash .github/workflows/scripts/run-flowr-command.sh lint | |
| test: | |
| needs: [ debug ] | |
| name: "⚗️ Test Suite (coverage)" | |
| if: ${{ github.event_name != 'pull_request' || github.event.pull_request.base.ref != 'main' }} | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 65 | |
| steps: | |
| - name: "🛒 Checkout Repository" | |
| uses: actions/checkout@v6 | |
| with: | |
| submodules: true | |
| - name: "🌍 Load Versions to Use" | |
| run: bash .github/workflows/scripts/global-configuration.sh | |
| - name: "⬇️ Use Node.js" | |
| uses: actions/setup-node@v6 | |
| with: | |
| node-version: ${{ env.ACTION_NODE_VERSION }} | |
| registry-url: "https://registry.npmjs.org/" | |
| cache: npm | |
| - name: "⬇️ Setup R" | |
| uses: ./.github/actions/setup-r | |
| timeout-minutes: 45 | |
| with: | |
| r-version: ${{ env.ACTION_R_VERSION }} | |
| - name: "📦 Install R Packages" | |
| uses: r-lib/actions/setup-r-dependencies@v2 | |
| timeout-minutes: 20 | |
| with: | |
| cache: 'true' | |
| cache-version: '1' | |
| extra-packages: | | |
| any::dplyr | |
| any::readr | |
| - name: "📦 Install Node Dependencies" | |
| # the sync below runs through ts-node, so the dependencies have to be there before it | |
| run: npm ci | |
| - name: "💾 Cache the Signature Database Bundles" | |
| uses: actions/cache@v4 | |
| with: | |
| # only the bundles, the shards unpacked next to them are re-derived on demand | |
| path: ~/.cache/flowr/sigdb/bundles | |
| key: sigdb-bundles-${{ hashFiles('src/data/sigdb/sigdb.remote.json') }} | |
| - name: "⬇️ Sync the Signature Database" | |
| # without a database every sigdb-backed test skips instead of checking anything; run through the same | |
| # script as the tests so the node dependencies are installed once and reused by both | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: bash .github/workflows/scripts/run-flowr-command.sh sync:sigdb | |
| - name: "🧪 Run the Tests" | |
| run: bash .github/workflows/scripts/run-flowr-command.sh "test:full -- --allowOnly=false" | |
| - name: "⚙️ Run System Tests" | |
| run: bash .github/workflows/scripts/run-flowr-command.sh "test:system -- --no-watch" | |
| - name: "⬆️ Upload Test Label Details" | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test-label-details | |
| path: | | |
| coverage/flowr-test-details.json | |
| coverage/flowr-test-results.json | |
| if-no-files-found: warn | |
| - name: "⬆️ Upload Coverage Reports to Codecov" | |
| if: github.ref == 'refs/heads/main' | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| fail_ci_if_error: true | |
| performance-test: | |
| needs: [ test ] | |
| name: "⏱️ Performance Test" | |
| # we do not run if the release workflow runs it with pushing | |
| if: ${{ | |
| !cancelled() | |
| && !failure() | |
| && (inputs.force-full || (github.event_name == 'push' && github.ref == 'refs/heads/main')) | |
| && !( | |
| startsWith(github.event.head_commit.message, '[release:minor]') || | |
| startsWith(github.event.head_commit.message, '[release:major]') || | |
| startsWith(github.event.head_commit.message, '[release:patch]') | |
| ) | |
| }} | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 75 | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| # currently we have a duplication, add your benchmark names here and in `performance-test-upload` | |
| name: ['artificial', 'real-world'] | |
| parser: ["tree-sitter", "r-shell"] | |
| steps: | |
| - name: "🛒 Checkout Repository" | |
| uses: actions/checkout@v6 | |
| with: | |
| submodules: true | |
| - name: "🌍 Load Versions to Use" | |
| run: bash .github/workflows/scripts/global-configuration.sh | |
| - name: "⬇️ Use Node.js" | |
| uses: actions/setup-node@v6 | |
| with: | |
| node-version: ${{ env.ACTION_NODE_VERSION }} | |
| registry-url: "https://registry.npmjs.org/" | |
| cache: npm | |
| - name: "⬇️ Setup R" | |
| uses: ./.github/actions/setup-r | |
| timeout-minutes: 45 | |
| with: | |
| r-version: ${{ env.ACTION_R_VERSION }} | |
| - name: "⏱️ Run the performance benchmarks" | |
| run: bash .github/workflows/scripts/run-flowr-command.sh test:performance -- 1 1 "${{ matrix.name }}" "${{ matrix.parser }}" | |
| - name: "⬆️ Upload Benchmark Results" | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark-results-${{ matrix.name }}-${{ matrix.parser }} | |
| path: test/performance/results/ | |
| if-no-files-found: error | |
| performance-test-upload: | |
| name: "⬆️ Upload Performance Test Results" | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 45 | |
| needs: [ performance-test ] | |
| steps: | |
| - name: "Checkout Repository" | |
| uses: actions/checkout@v6 | |
| with: | |
| submodules: true | |
| token: ${{ secrets.RELEASE_TOKEN }} | |
| - name: "⚙️ Name and Email for Git (config)" | |
| run: | | |
| git config --local user.email "action@github.com" | |
| git config --local user.name "GitHub Action" | |
| - name: "🌍 Load Versions to Use" | |
| run: bash .github/workflows/scripts/global-configuration.sh | |
| - name: "⬇️ Use Node.js" | |
| uses: actions/setup-node@v6 | |
| with: | |
| node-version: ${{ env.ACTION_NODE_VERSION }} | |
| cache: npm | |
| - name: "⬇️ Get the Test Label Details" | |
| uses: actions/download-artifact@v4 | |
| continue-on-error: true | |
| with: | |
| name: test-label-details | |
| path: test-labels/ | |
| # currently we have a duplication, add your benchmark names here and in `performance-test` and check the release.yaml | |
| - name: "⬇️ Get benchmark Artifacts (artificial, r-shell)" | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: benchmark-results-artificial-r-shell | |
| path: benchmark-ai-r-shell/ | |
| - name: "🧪 Add the Test Counters (artificial-r-shell)" | |
| run: bash .github/workflows/scripts/run-flowr-command.sh "benchmark:test-counts -- --results test-labels/flowr-test-results.json test-labels/flowr-test-details.json benchmark-ai-r-shell/artificial/artificial-summarized-graph-info.json" | |
| - name: "🥄 Merge and Produce Performance Results (artificial, r-shell)" | |
| uses: benchmark-action/github-action-benchmark@v1 | |
| with: | |
| name: '"artificial" Benchmark Suite' | |
| tool: 'customSmallerIsBetter' | |
| output-file-path: benchmark-ai-r-shell/artificial/artificial-summarized-graph.json | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| fail-on-alert: false | |
| summary-always: false | |
| comment-on-alert: true | |
| comment-always: true | |
| skip-fetch-gh-pages: true | |
| max-items-in-chart: 100 | |
| ref: "refs/heads/main" | |
| gh-repository: ${{ github.repository }} | |
| benchmark-data-dir-path: wiki/stats/benchmark/ | |
| auto-push: false | |
| # the counters are no measurement of speed, so they are uploaded on their own, where nothing alerts on | |
| # them: a release with more linting rules must never read as a performance regression. The page merges | |
| # the two suites back into one, see `mergeInfoSuites` in `wiki/stats/benchmark/stats.js`. | |
| - name: "🧮 Merge and Produce the Counters (artificial, r-shell)" | |
| uses: benchmark-action/github-action-benchmark@v1 | |
| with: | |
| name: '"artificial" Benchmark Suite [info]' | |
| tool: 'customSmallerIsBetter' | |
| output-file-path: benchmark-ai-r-shell/artificial/artificial-summarized-graph-info.json | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| fail-on-alert: false | |
| summary-always: false | |
| comment-on-alert: false | |
| comment-always: false | |
| alert-threshold: '100000%' | |
| skip-fetch-gh-pages: true | |
| max-items-in-chart: 100 | |
| ref: "refs/heads/main" | |
| gh-repository: ${{ github.repository }} | |
| benchmark-data-dir-path: wiki/stats/benchmark/ | |
| auto-push: false | |
| - name: "⬇️ Get benchmark Artifacts (real-world, r-shell)" | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: benchmark-results-real-world-r-shell | |
| path: benchmark-rw-r-shell | |
| - name: "🧪 Add the Test Counters (real-world-r-shell)" | |
| run: bash .github/workflows/scripts/run-flowr-command.sh "benchmark:test-counts -- --results test-labels/flowr-test-results.json test-labels/flowr-test-details.json benchmark-rw-r-shell/real-world/real-world-summarized-graph-info.json" | |
| - name: "🥄 Merge and Produce Performance Results (real-world, r-shell)" | |
| uses: benchmark-action/github-action-benchmark@v1 | |
| with: | |
| name: '"real-world" Benchmark Suite' | |
| tool: 'customSmallerIsBetter' | |
| output-file-path: benchmark-rw-r-shell/real-world/real-world-summarized-graph.json | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| fail-on-alert: false | |
| summary-always: false | |
| comment-on-alert: true | |
| comment-always: true | |
| skip-fetch-gh-pages: true | |
| max-items-in-chart: 100 | |
| ref: "refs/heads/main" | |
| gh-repository: ${{ github.repository }} | |
| benchmark-data-dir-path: wiki/stats/benchmark/ | |
| auto-push: false | |
| # the counters are no measurement of speed, so they are uploaded on their own, where nothing alerts on | |
| # them: a release with more linting rules must never read as a performance regression. The page merges | |
| # the two suites back into one, see `mergeInfoSuites` in `wiki/stats/benchmark/stats.js`. | |
| - name: "🧮 Merge and Produce the Counters (real-world, r-shell)" | |
| uses: benchmark-action/github-action-benchmark@v1 | |
| with: | |
| name: '"real-world" Benchmark Suite [info]' | |
| tool: 'customSmallerIsBetter' | |
| output-file-path: benchmark-rw-r-shell/real-world/real-world-summarized-graph-info.json | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| fail-on-alert: false | |
| summary-always: false | |
| comment-on-alert: false | |
| comment-always: false | |
| alert-threshold: '100000%' | |
| skip-fetch-gh-pages: true | |
| max-items-in-chart: 100 | |
| ref: "refs/heads/main" | |
| gh-repository: ${{ github.repository }} | |
| benchmark-data-dir-path: wiki/stats/benchmark/ | |
| auto-push: false | |
| - name: "⬇️ Get benchmark Artifacts (artificial, tree-sitter)" | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: benchmark-results-artificial-tree-sitter | |
| path: benchmark-ai-tree-sitter/ | |
| - name: "🧪 Add the Test Counters (artificial-tree-sitter)" | |
| run: bash .github/workflows/scripts/run-flowr-command.sh "benchmark:test-counts -- --results test-labels/flowr-test-results.json test-labels/flowr-test-details.json benchmark-ai-tree-sitter/artificial/artificial-summarized-graph-info.json" | |
| - name: "🥄 Merge and Produce Performance Results (artificial, tree-sitter)" | |
| uses: benchmark-action/github-action-benchmark@v1 | |
| with: | |
| name: '"artificial" Benchmark Suite (tree-sitter)' | |
| tool: 'customSmallerIsBetter' | |
| output-file-path: benchmark-ai-tree-sitter/artificial/artificial-summarized-graph.json | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| fail-on-alert: false | |
| summary-always: false | |
| comment-on-alert: true | |
| comment-always: true | |
| skip-fetch-gh-pages: true | |
| max-items-in-chart: 100 | |
| ref: "refs/heads/main" | |
| gh-repository: ${{ github.repository }} | |
| benchmark-data-dir-path: wiki/stats/benchmark/ | |
| auto-push: false | |
| # the counters are no measurement of speed, so they are uploaded on their own, where nothing alerts on | |
| # them: a release with more linting rules must never read as a performance regression. The page merges | |
| # the two suites back into one, see `mergeInfoSuites` in `wiki/stats/benchmark/stats.js`. | |
| - name: "🧮 Merge and Produce the Counters (artificial, tree-sitter)" | |
| uses: benchmark-action/github-action-benchmark@v1 | |
| with: | |
| name: '"artificial" Benchmark Suite (tree-sitter) [info]' | |
| tool: 'customSmallerIsBetter' | |
| output-file-path: benchmark-ai-tree-sitter/artificial/artificial-summarized-graph-info.json | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| fail-on-alert: false | |
| summary-always: false | |
| comment-on-alert: false | |
| comment-always: false | |
| alert-threshold: '100000%' | |
| skip-fetch-gh-pages: true | |
| max-items-in-chart: 100 | |
| ref: "refs/heads/main" | |
| gh-repository: ${{ github.repository }} | |
| benchmark-data-dir-path: wiki/stats/benchmark/ | |
| auto-push: false | |
| - name: "⬇️ Get benchmark Artifacts (real-world, tree-sitter)" | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: benchmark-results-real-world-tree-sitter | |
| path: benchmark-rw-tree-sitter | |
| - name: "🧪 Add the Test Counters (real-world-tree-sitter)" | |
| run: bash .github/workflows/scripts/run-flowr-command.sh "benchmark:test-counts -- --results test-labels/flowr-test-results.json test-labels/flowr-test-details.json benchmark-rw-tree-sitter/real-world/real-world-summarized-graph-info.json" | |
| - name: "🥄 Merge and Produce Performance Results (real-world, tree-sitter)" | |
| uses: benchmark-action/github-action-benchmark@v1 | |
| with: | |
| name: '"real-world" Benchmark Suite (tree-sitter)' | |
| tool: 'customSmallerIsBetter' | |
| output-file-path: benchmark-rw-tree-sitter/real-world/real-world-summarized-graph.json | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| fail-on-alert: false | |
| summary-always: false | |
| comment-on-alert: true | |
| comment-always: true | |
| skip-fetch-gh-pages: true | |
| max-items-in-chart: 100 | |
| ref: "refs/heads/main" | |
| gh-repository: ${{ github.repository }} | |
| benchmark-data-dir-path: wiki/stats/benchmark/ | |
| auto-push: false | |
| # the counters are no measurement of speed, so they are uploaded on their own, where nothing alerts on | |
| # them: a release with more linting rules must never read as a performance regression. The page merges | |
| # the two suites back into one, see `mergeInfoSuites` in `wiki/stats/benchmark/stats.js`. | |
| - name: "🧮 Merge and Produce the Counters (real-world, tree-sitter)" | |
| uses: benchmark-action/github-action-benchmark@v1 | |
| with: | |
| name: '"real-world" Benchmark Suite (tree-sitter) [info]' | |
| tool: 'customSmallerIsBetter' | |
| output-file-path: benchmark-rw-tree-sitter/real-world/real-world-summarized-graph-info.json | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| fail-on-alert: false | |
| summary-always: false | |
| comment-on-alert: false | |
| comment-always: false | |
| alert-threshold: '100000%' | |
| skip-fetch-gh-pages: true | |
| max-items-in-chart: 100 | |
| ref: "refs/heads/main" | |
| gh-repository: ${{ github.repository }} | |
| benchmark-data-dir-path: wiki/stats/benchmark/ | |
| auto-push: false | |
| - name: "⬆️ Upload Benchmark Graph Data" | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark-results-graph-data | |
| path: wiki/stats/benchmark/ | |
| retention-days: 1 |