chore(workflows): silently run playwright axe tests

Author: plumdumpling

.github/workflows/playwright.yml

@@ -1,27 +1,61 @@
-name: Playwright Tests
+name: Playwright Accessibility Tests
+
 on:
-  push:
-    branches: [ main ]
   pull_request:
-    branches: [ main ]
+    types: [opened, synchronize, reopened]
+
 jobs:
-  test:
+  playwright:
+    name: Playwright
+    runs-on: ubuntu-latest
     timeout-minutes: 60
+    outputs:
+      issue_count: ${{ steps.playwright.outputs.issue_count }}
+    steps:
+      - name: Checkout repository from GitHub
+        uses: actions/checkout@v4
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Install Playwright Browsers
+        run: npx playwright install --with-deps
+
+      - name: Build app
+        run: npm run build
+
+      - name: Serve preview
+        run: npm run preview & npx wait-on http://localhost:4173
+
+      - name: Run Playwright tests
+        run: npx playwright test || true
+
+      - name: Count Playwright accessibility issues
+        id: playwright
+        run: |
+          issue_count=$(jq '. | length' playwright-a11y-violations.json)
+          echo "issue_count=$issue_count" >> $GITHUB_OUTPUT
+
+      - name: Upload report artifact
+        uses: actions/upload-artifact@v4
+        if: ${{ !cancelled() }}
+        with:
+          name: playwright-report
+          path: playwright-report/
+          retention-days: 30
+
+  comment-on-pr:
+    needs: playwright
+    if: needs.playwright.outputs.issue_count != '0'
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
-    - uses: actions/setup-node@v4
-      with:
-        node-version: lts/*
-    - name: Install dependencies
-      run: npm ci
-    - name: Install Playwright Browsers
-      run: npx playwright install --with-deps
-    - name: Run Playwright tests
-      run: npx playwright test
-    - uses: actions/upload-artifact@v4
-      if: ${{ !cancelled() }}
-      with:
-        name: playwright-report
-        path: playwright-report/
-        retention-days: 30
+      - name: Comment on PR
+        uses: thollander/actions-comment-pull-request@v3
+        with:
+          message: "♿ Playwright detected ${{ needs.playwright.outputs.issue_count }} accessibility issue(s) – please review [here](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})."
+          github-token: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -29,3 +29,4 @@ dist-ssr
 /blob-report/
 /playwright/.cache/
 /playwright/.auth/
+playwright-a11y-violations.json

e2e/basic.spec.ts

@@ -1,13 +1,28 @@
+import fs from 'fs';
 import AxeBuilder from '@axe-core/playwright';
-import {test, expect} from '@playwright/test';
+import {test} from '@playwright/test';
 
-test.only('BASIC', async ({page}) => {
-  await page.goto('localhost:5173');
+test('BASIC', async ({page}, testInfo) => {
+  await page.goto('http://localhost:4173');
   await page.locator('#root').waitFor();
 
   // Check the whole page
   const axeBuilder = await new AxeBuilder({page})
     .withTags(['wcag2a', 'wcag2aa', 'wcag21a', 'wcag21aa'])
     .analyze();
-  expect(axeBuilder.violations).toEqual([]);
+  const violations = axeBuilder.violations;
+
+  // Write violations to json file
+  fs.writeFileSync(
+    'playwright-a11y-violations.json',
+    JSON.stringify(violations, null, 2)
+  );
+
+  // Add an info annotation if there are violations
+  if (violations.length > 0) {
+    testInfo.annotations.push({
+      type: 'info',
+      description: `Accessibility violations found: ${violations.length}`
+    });
+  }
 });