name: E2E # Releases, and anything proposed for one. NOT every push. # # This is the expensive workflow in the repository — it builds the app's own # FrankenPHP image, brings up Postgres, a Mercure hub and the app, downloads a # Chromium, and runs PHPStan, PHPUnit and ~690 browser tests — and running it on # every commit to a branch nobody is releasing from spends most of the account's # capacity proving the same thing repeatedly. # # ("~250" is what this said for a long time. It is ~690 across ~95 files as of # v0.2.16 — counted with `npx playwright test --list`, which is worth doing # before reasoning about shard counts from a number in a comment.) # # Pushes to main are not run, and that costs nothing since main became # protected (2026-10-15): every commit there comes from a pull request whose # jobs below are required checks. That includes a release, whose `Release vX.Y.Z` # pull request is the suite run behind its tag (see CONTRIBUTING.md, Releasing). # The protection does not require a pull request to be up to date with main, so # two that pass apart can still fail together. Run it by hand with # `workflow_dispatch` on any branch you are unsure of — or locally, which is # faster than either: # # npm run test:unit:docker && npm run test:e2e:docker # # ── WHY IT IS SIX JOBS AND NOT ONE ────────────────────────────────────────── # # It was one job that did the setup once and then ran PHPStan, then PHPUnit, # then the browser suite, in that order, on a runner with two cores. Everything # waited for everything, and the browser suite — which is the great majority of # it — waited for two static checks that have no bearing on whether a page # renders. # # Now the browser suite is split with Playwright's own `--shard`, and the pieces # run at once on runners of their own. The static checks run beside them rather # than in front of them. # # WHAT THIS COSTS, said plainly: the preparation — image, dependencies, stack, # browser — is per JOB, so six jobs pay it six times. That is more machine # minutes for less wall-clock, which is the trade sharding always is. It is only # worth it because the preparation is nearly all cache: the image is built once # in `image` below and read from the layer cache by everyone else, and Composer # and the browser come from actions/cache. A cold cache makes this workflow # slower than the one it replaced, which is a thing to know before blaming a # shard. # # WHY EACH SHARD GETS ITS OWN STACK rather than sharing one. Two reasons, and # the second is the interesting one. A single app container was already the # ceiling on worker count, so pointing eight browsers at one of them would have # bought nothing. And a stack per job is what finally lets the four specs that # own install-wide state — the AI settings singleton, the integration provider # rows, the Mercure hub that one spec restarts — run at the same time as # everything else instead of queueing behind all of it in a single worker. They # were never slow; they were serialised, because there was one install to # collide over. on: push: tags: ['v*'] # Kept, and cheap: a pull request is somebody proposing a change, which is # exactly when a second opinion from the suite is worth having, and there # are few of them. # # `wip/**` as well: a feature too big for one pull request is built on a # `wip/` branch and reviewed in phases, each phase a pull request into that # branch (issue #28 is the first). Those merges are where the phases meet, # so they get the same second opinion a pull request into main does. A # push to a wip branch still does not — the pull request is the check. # # `milestone/**` too: a release's work is collected on a milestone branch # (`milestone/v0.4.0`) before it reaches main, and a pull request into one # is the same proposal a pull request into main is. pull_request: branches: [main, 'wip/**', 'milestone/**'] workflow_dispatch: # Six runners per push adds up, so a superseded pull request stops paying for # itself. Pushing a second commit to a branch under review cancels the run for # the first — nobody is waiting on an answer about a tree that no longer exists. # # Tags and manual runs are NOT cancelled, which is what the expression is for # rather than a bare `true`: a release is the one run whose result is kept and # looked at later, and two tags in a row are two different questions. concurrency: group: e2e-${{ github.ref }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} env: # compose.test.yaml's own default, and playwright.config.ts falls back to # the same number. Stated here only to be explicit. TEST_HTTP_PORT: "8001" jobs: # Nothing but the image, so that the five jobs below start from a warm layer # cache instead of five cold builds racing to write the same scope. # # It is the ONLY writer — every other job reads. Six `cache-to`s on one # scope is not a faster cache, it is six uploads overwriting each other, and # the winner is whichever finished last rather than whichever is most # complete. # # The cost is honest: on a warm cache this adds its own minute to the front # of the run. What it buys is that the minute is paid once instead of five # times over, and that a cold cache is filled by one build rather than by # five simultaneous ones. image: name: Image runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 - uses: ./.github/actions/test-stack with: write-cache: "true" stack: "false" # PHPStan and PHPUnit, beside the browser suite rather than in front of it. # # Both need the stack: phpstan-symfony resolves service ids out of the # compiled test container, and PHPUnit wants a database. Neither needs a # browser, which is why this job does not download one. # # Deliberately NOT sharded. It is a single PHP process and it finishes in # about a minute; splitting it across runners would spend more time starting # stacks than it saved, and the point of this workflow is wall-clock rather # than the appearance of parallelism. # THESE JOB NAMES ARE BRANCH PROTECTION. `main` requires "PHPStan and # PHPUnit", every "Playwright N/M" shard and "Playwright (install-wide)" by # name (Settings → Branches). Renaming one, or changing the shard count, # leaves a required check that never reports, and every pull request waits # on it for ever — change the rule in the same breath. static: name: PHPStan and PHPUnit runs-on: ubuntu-latest needs: image steps: - uses: actions/checkout@v7 - uses: ./.github/actions/test-stack # After the stack is up, not before: phpstan-symfony resolves # service ids out of the compiled test container, and the kernel # boot app-init did while migrating and seeding is what produced it. - name: Run PHPStan run: | docker compose -f compose.test.yaml run --rm --no-deps \ --entrypoint docker-php-entrypoint app composer stan # `npm run test:unit:docker` minus the `up`, which has already # happened. PHPUnit used to run bare on the runner while every # developer ran it in a container, so the two could disagree about # anything the image decides: PHP version, extensions, upload limits. - name: Run PHPUnit tests run: | docker compose -f compose.test.yaml run --rm --no-deps \ --entrypoint docker-php-entrypoint app sh bin/test-unit.sh - name: Collect the stack's log if: ${{ !cancelled() }} run: | mkdir -p var/log docker compose -f compose.test.yaml logs --no-color --timestamps > var/log/static-server.log - name: Upload the stack's log if: ${{ !cancelled() }} uses: actions/upload-artifact@v7 with: name: log-static path: var/log/static-server.log retention-days: 14 # The browser suite, in quarters. browser: name: Playwright ${{ matrix.shard }}/${{ matrix.shardTotal }} runs-on: ubuntu-latest needs: image strategy: # A failing shard must not cancel its siblings. They are testing # different things, the report at the end is assembled from all of # them, and a cancelled shard contributes no blob — so fail-fast # would answer "one shard failed" with "and we no longer know about # the other three". fail-fast: false matrix: # THE TWO NUMBERS HAVE TO AGREE. `shard` is the list of indices # and `shardTotal` is what they are out of; adding a fifth index # without changing the total gives shard 5 of 4, which Playwright # refuses. Kept in one block, as Playwright's own documentation # writes it, so they are read together. # # Four, because the ceiling is not the split but the runner: # each of these has two cores, the app container is on it, and # playwright.config.ts already caps CI at two workers for that # reason. Four shards is eight browsers against four app # containers. # # MEASURED, so the next person tuning this starts from data. # Four workers on a developer machine, whole suite 4m55s: # # 1/4 168 tests 2.8m account-*, admin-*, appearance*, calendar* # 2/4 169 tests 1.3m # 3/4 155 tests 1.4m # 4/4 137 tests 0.7m # exclusive 23 1.1m its own job, below # # Playwright divides by test COUNT, and the counts are even. The # DURATIONS are not: the first quarter holds the calendar and # appearance specs, which navigate and measure layout, so it # takes four times what the last one does. Since the run is only # as fast as its slowest shard, that is where a further gain is # — and more shards is the lever, since there is no # duration-aware split to reach for. Six would cut that block in # two, for two more runners' worth of setup. Four is the trade # taken here, not a limit. shard: [1, 2, 3, 4] shardTotal: [4] steps: - uses: actions/checkout@v7 - uses: ./.github/actions/test-stack with: playwright: "true" # --project=chromium, so the four install-wide specs are left to the # job below. Their project declares `dependencies: ["chromium"]`, # which under sharding would mean every shard running the whole of # chromium first — naming the project is what keeps that from # happening as well as what divides the work. # # `blob`, not html: a shard's HTML report describes a quarter of the # suite and cannot say anything about the rest. The blobs are merged # into one report at the end. It also replaces the `github` # reporter for the duration — annotations come from the merge, where # they can be attributed to the whole run. - name: Run Playwright tests run: npx playwright test --project=chromium --shard=${{ matrix.shard }}/${{ matrix.shardTotal }} --reporter=blob # The web server's own log, which is the only place a PHP fatal that # never reached the kernel is recorded — a trace cannot show what was # never a response. The whole project, not just `app`: when the # failure is that the stack never came up, the line explaining why is # in app-init's migration output or in Postgres'. - name: Collect the stack's log if: ${{ !cancelled() }} run: | mkdir -p var/log docker compose -f compose.test.yaml logs --no-color --timestamps > var/log/e2e-server.log - name: Upload the blob report if: ${{ !cancelled() }} uses: actions/upload-artifact@v7 with: name: blob-chromium-${{ matrix.shard }} path: blob-report/ retention-days: 14 - name: Upload the stack's log if: ${{ !cancelled() }} uses: actions/upload-artifact@v7 with: name: log-chromium-${{ matrix.shard }} path: var/log/e2e-server.log retention-days: 14 # The four specs that own install-wide state, on an install of their own. # # In the single-job workflow this project ran last, alone, on one worker, # after everything else had finished — because AiSettings, the integration # provider rows and the Mercure hub are one per INSTALLATION and no # per-worker fixture can isolate them. On its own runner with its own stack # there is nothing left to collide with, so it runs beside the shards and # its wall-clock cost falls to whatever it takes on its own. # # --no-deps is the other half of that. The project declares # `dependencies: ["chromium"]`, which is what enforces the ordering for # anybody running the suite locally against ONE stack, where it is still # exactly right. Here the stack is not shared, so the dependency would only # mean running the entire rest of the suite again before starting. browser-exclusive: name: Playwright (install-wide) runs-on: ubuntu-latest needs: image steps: - uses: actions/checkout@v7 - uses: ./.github/actions/test-stack with: playwright: "true" - name: Run Playwright tests run: npx playwright test --project=chromium-exclusive --no-deps --reporter=blob - name: Collect the stack's log if: ${{ !cancelled() }} run: | mkdir -p var/log docker compose -f compose.test.yaml logs --no-color --timestamps > var/log/e2e-server.log - name: Upload the blob report if: ${{ !cancelled() }} uses: actions/upload-artifact@v7 with: name: blob-exclusive path: blob-report/ retention-days: 14 - name: Upload the stack's log if: ${{ !cancelled() }} uses: actions/upload-artifact@v7 with: name: log-exclusive path: var/log/e2e-server.log retention-days: 14 # One report for the whole suite, assembled from the shards' blobs. # # Without this a run leaves five partial reports and no single answer to # "what failed" — which is worse than the unsharded workflow was, and would # be a poor trade for the minutes sharding saves. The merge also emits the # `github` annotations the shards deliberately did not, so a failure is # attributed once rather than four times. # # `always()` rather than `success()`: this is at its most useful exactly # when a shard has failed. report: name: Report runs-on: ubuntu-latest needs: [browser, browser-exclusive] if: ${{ !cancelled() }} steps: - uses: actions/checkout@v7 - uses: actions/setup-node@v7 with: node-version-file: .nvmrc cache: npm - name: Install Node dependencies run: npm ci # Every shard's blob into one directory. `merge-multiple` flattens # them, which is what merge-reports expects — it reads a directory # of blobs, not a directory of directories. - name: Download the blob reports uses: actions/download-artifact@v8 with: path: blob-report pattern: blob-* merge-multiple: true - name: Merge into one report run: npx playwright merge-reports --reporter=html,github ./blob-report - name: Upload the Playwright report uses: actions/upload-artifact@v7 with: name: playwright-report path: playwright-report/ retention-days: 14