WIP: WEBDEV-6730 fix flaky tests - fixmes

.env.sample

@@ -1,9 +1,16 @@
 BROWSERSTACK_USERNAME="<retrieve_from_browserstack_account>"
 BROWSERSTACK_ACCESS_KEY="<retrieve_from_browserstack_account>"
 
-A_EMAIL="<priv_email>"
-A_PASSWORD="<priv_password>"
-B_EMAIL="<reg_email>"
-B_PASSWORD="<reg_password>"
+ARCHIVE_EMAIL="<priv_email>"
+ARCHIVE_PASSWORD="<priv_password>"
+PATRON_EMAIL="<reg_email>"
+PATRON_PASSWORD="<reg_password>"
 
-CATEGORY="all"
+# default to `plugin` - indicating that testRun was triggered by VSCode Playwright plugin
+CATEGORY="plugin"
+
+# Set to true if whathost debugging is needed every testRun.
+# Each fixtures will call an API service that will return a response with www<node> details,
+# which can help in determining what host node you are connected to.
+# This might be helpful in debugging if a www<node> is causing an issue/failing test
+WHATHOST="false"

.github/workflows/donation-tests.yml → .github/actions/setup-env.yml

@@ -1,11 +1,6 @@
-name: Donation banner tests
+name: Setup GH actions and install dependencies
 
-on:
-  push:
-    branches: [ main, master ]
-
-  pull_request:
-    branches: [ main, master ]
+on: workflow_call
 
 jobs:
   test:
@@ -24,6 +19,3 @@ jobs:
 
       - name: Install Playwright Browsers
         run: npx playwright install --with-deps
-
-      - name: Run Donation Tests
-        run: npm run test:donation

.github/workflows/call-run-individual-tests.yml

@@ -0,0 +1,86 @@
+name: Install deps and browsers, run tests, and upload artifacts
+
+on: workflow_call
+
+jobs:
+  # setup: 
+  #   name: Setup test environment
+  #   uses: ./.github/actions/setup-env.yml
+
+  test:
+    runs-on: ubuntu-latest
+    # strategy:
+    #   matrix:
+    #     os: [macos-latest, ubuntu-latest]
+    # runs-on: ${{ matrix.os }}
+
+    env:
+      ARCHIVE_EMAIL: ${{ secrets.ARCHIVE_EMAIL }}
+      ARCHIVE_PASSWORD: ${{ secrets.ARCHIVE_PASSWORD }}
+      PATRON_EMAIL: ${{ secrets.PATRON_EMAIL }}
+      PATRON_PASSWORD: ${{ secrets.PATRON_PASSWORD }}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Install Playwright Browsers
+        run: npx playwright install --with-deps
+
+      - name: Run about tests
+        if: always()
+        run: npm run test:about
+
+      - name: Run av tests
+        if: always()
+        run: npm run test:av
+
+      - name: Run books tests
+        if: always()
+        run: npm run test:books
+
+      - name: Run collection tests
+        if: always()
+        run: npm run test:collection
+
+      - name: Run details tests
+        if: always()
+        run: npm run test:details
+
+      - name: Run donation tests
+        if: always()
+        run: npm run test:donation
+
+      - name: Run home Tests
+        if: always()
+        run: npm run test:home
+
+      - name: Run login tests
+        if: always()
+        run: npm run test:login
+
+      - name: Run music tests
+        if: always()
+        run: npm run test:music
+
+      - name: Run profile tests
+        if: always()
+        run: npm run test:profile
+
+      - name: Run search tests
+        if: always()
+        run: npm run test:search
+
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: playwright-report
+          path: playwright-report/
+          retention-days: 1

.github/workflows/call-run-tests.yml

@@ -7,18 +7,19 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - uses: actions/setup-node@v3
-        with:
-          node-version: 20
-          cache: 'npm'
+      - name: Setup test environment
+        uses: ./.github/actions/setup-env
+
+      # - name: Run tests in Chromium browser
+      #   run: npm run test:chrome
 
-      - name: Install dependencies
-        run: npm ci
+      # - name: Run tests in Firefox browser
+      #   run: npm run test:firefox
 
-      - name: Install Playwright Browsers
-        run: npx playwright install --with-deps
+      # - name: Run tests in Webkit browser
+      #   run: npm run test:webkit
 
       - name: Run Playwright tests
         run: ./run-tests.sh

.github/workflows/main.yml

@@ -18,5 +18,9 @@ jobs:
   #     SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
 
   test:
-    name: Run tests
-    uses: ./.github/workflows/call-run-tests.yml
+    name: Run tests individually
+    uses: ./.github/workflows/call-run-individual-tests.yml
+
+  # test:
+  #   name: Run all tests
+  #   uses: ./.github/workflows/call-run-tests.yml

.github/workflows/manual-trigger-individual-test.yml

@@ -0,0 +1,9 @@
+# This is a basic workflow that is manually triggered
+name: Manual workflow - just for testing
+
+on: workflow_dispatch
+
+jobs:
+  test:
+    name: Run tests individually
+    uses: ./.github/workflows/call-run-individual-tests.yml

.github/workflows/manual-trigger-specific-test.yml

@@ -0,0 +1,31 @@
+# This is a basic workflow that is manually triggered
+
+name: Manual workflow - just for testing
+
+# Controls when the action will run. Workflow runs when manually triggered using the UI
+# or API.
+on:
+  workflow_dispatch:
+    # Inputs the workflow accepts.
+    input_command:
+      name:
+        description: 'CLI command you want to execute in the pipeline'
+
+jobs:
+  manual:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup test environment
+        uses: ./.github/actions/setup-env
+
+      - shell: bash
+        env:
+          ARCHIVE_EMAIL: ${{ secrets.ARCHIVE_EMAIL }}
+          ARCHIVE_PASSWORD: ${{ secrets.ARCHIVE_PASSWORD }}
+          PATRON_EMAIL: ${{ secrets.PATRON_EMAIL }}
+          PATRON_PASSWORD: ${{ secrets.PATRON_PASSWORD }}
+        run: |
+          ${{ github.event.inputs.input_command }}

.gitignore

@@ -4,6 +4,11 @@ node_modules/
 /playwright/.cache
 /playwright/.auth
 
+# json reporter summary
+/ctrf
+/playwright-summary
+*-summary.json
+
 .env
 *.err
 *.log

README.md

@@ -2,6 +2,7 @@
 
 # End to end tests for Archive.org using [Playwright](https://playwright.dev/)
 
+
 ## (Optional) BrowserStack Local Setup
 
 - login to BrowserStack and retrieve the account username and access key, see documentation [here](https://www.browserstack.com/docs/automate/playwright/getting-started/nodejs/test-runner)
@@ -17,20 +18,28 @@
 
     `npx playwright install`
 
+- create a `.env` file by copying the contents from `.env.sample` and add the respective values you want to use for testing
+
 - run all the tests in headless mode and generate 1 whole test report:
 
     `npm run test`
 
-- run all the tests in headless mode by each category and generate test report by category:
+- run all the tests in headless mode by each category:
 
     `./run-tests.sh`
 
+- run all the tests in headless mode by browser:
+
+    - chrome: `./run-tests.sh -b chrome`
+
+    - firefox: `./run-tests.sh -b firefox`
+
+    - webkit: `./run-tests.sh -b webkit`
+
 - run all the tests in headed mode (this will load multiple browsers):
 
     `npm run test:headed`
 
-- create a `.env` file by copying the contents from `.env.sample` and add the respective values you want to use for testing
-
 
 ## Running individual tests by category (headless mode)
 

config/index.ts

@@ -4,11 +4,11 @@ dotenv.config();
 export const config = {
   baseURL: process.env.BASE_URL ?? "https://archive.org",
   patronUser: {
-    email: process.env.B_EMAIL || '',
-    password: process.env.B_PASSWORD || ''
+    email: process.env.PATRON_EMAIL || '',
+    password: process.env.PATRON_PASSWORD || ''
   },
   privUser: {
-    email: process.env.A_EMAIL || '',
-    password: process.env.A_PASSWORD || '',
+    email: process.env.ARCHIVE_EMAIL || '',
+    password: process.env.ARCHIVE_PASSWORD || '',
   },
 }