diff --git a/.github/workflows/call-run-tests.yml b/.github/workflows/call-run-tests.yml index 3d9718d4..712e1767 100644 --- a/.github/workflows/call-run-tests.yml +++ b/.github/workflows/call-run-tests.yml @@ -7,25 +7,32 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 with: node-version: 20 cache: 'npm' + - name: Check npm version installed + run: | + npm -v + npx -v + - name: Install dependencies - run: npm ci + run: | + npm ci + npm install -D @playwright/test@latest - name: Install Playwright Browsers run: npx playwright install --with-deps - name: Run Playwright tests - run: ./run-tests.sh - - - uses: actions/upload-artifact@v3 - if: always() - with: - name: playwright-report - path: playwright-report/ - retention-days: 1 + run: npm run test + + # - uses: actions/upload-artifact@v4 + # if: always() + # with: + # name: playwright-report + # path: playwright-report/ + # retention-days: 1 diff --git a/.github/workflows/donation-tests.yml b/.github/workflows/donation-tests.yml index 993d68a1..12ffdc98 100644 --- a/.github/workflows/donation-tests.yml +++ b/.github/workflows/donation-tests.yml @@ -26,4 +26,4 @@ jobs: run: npx playwright install --with-deps - name: Run Donation Tests - run: npm run test:donation + run: npm run execute -- --test=donation diff --git a/README.md b/README.md index 819214dd..4e0eca5b 100644 --- a/README.md +++ b/README.md @@ -2,62 +2,167 @@ # End to end tests for Archive.org using [Playwright](https://playwright.dev/) + ## (Optional) BrowserStack Local Setup -- login to BrowserStack and retrieve the account username and access key, see documentation [here](https://www.browserstack.com/docs/automate/playwright/getting-started/nodejs/test-runner) +- Login to BrowserStack and retrieve the account username and access key, see documentation [here](https://www.browserstack.com/docs/automate/playwright/getting-started/nodejs/test-runner) + + +## Local Setup + +- Install dependencies: + + ```bash + npm i + ``` + +- Install Playwright browser libs: + + ```bash + npx playwright install + ``` + +- Create a `.env` file by copying the contents from `.env.sample` and add the respective values you want to use for testing. This is required to run tests with loggedIn flows + +## Run all tests -## Running tests locally +- Run command: -- install dependencies: + ```bash + npm run test + ``` - `npm i` -- install Playwright browser libs: +## Running Tests with Custom Parameters - `npx playwright install` +- To run a custom script with flexible shell script arguments/parameters, follow these steps: -- run all the tests in headless mode and generate 1 whole test report: + 1. **Command Structure:** + - Ensure to use the `--` convention after the npm command to separate npm-specific arguments from those intended for the Node.js script + - Without `--`, arguments may not pass correctly to the script + + 2. **Flexibility in Argument Order:** + - You can pass script arguments in any order to suit your needs for flexibility and customization - `npm run test` -- run all the tests in headless mode by each category and generate test report by category: + ### Test Category (by Folder Structure) - `./run-tests.sh` + - Running tests from specific categories in the `tests` directory (except `tests/page-objects`): + - If no category is specified, all tests will run by default. + - Parameter: `--test` + - Example usage: + ```bash + npm run test -- --test=about + npm run test -- --test=books + npm run test -- --test=donation + npm run test -- --test=profile + ``` -- run all the tests in headed mode (this will load multiple browsers): + ### Test Execution Modes - `npm run test:headed` + - **Headed Mode:** + - Executes tests with a browser window visible on screen. + - Parameter: `--headed` + - Example usage: + ```bash + npm run test -- --test=about --headed + npm run test -- --headed --test=about + ``` -- create a `.env` file by copying the contents from `.env.sample` and add the respective values you want to use for testing + - **Debug Mode:** + - Executes tests with Playwright Inspector for step-by-step debugging. + - Parameter: `--debug` + - Example usage: + ```bash + npm run test -- --test=about --debug + npm run test -- --debug --test=books + ``` + - **UI Mode:** + - Executes tests with Playwright Inspector for UI-based debugging. + - Parameter: `--ui` + - Example usage: + ```bash + npm run test -- --test=about --ui + ``` -## Running individual tests by category (headless mode) + - **Trace Viewer Mode:** + - Records test runs for inclusion in the Playwright report. + - Parameter: `--trace` + - Example usage: + ```bash + npm run test -- --test=about --trace + ``` -- run about tests: `npm run test:about` -- run av tests: `npm run test:av` -- run books tests: `npm run test:books` -- run collection tests: `npm run test:collection` -- run details tests: `npm run test:details` -- run home tests: `npm run test:home` -- run login tests: `npm run test:login` -- run music tests: `npm run test:music` -- run search tests: `npm run test:search` -- run profile tests: `npm run test:profile` + ### Test Execution by Browser + - Running tests in specific browsers: + - By default, tests run in all available browsers. + - Parameter: `--browser` + - Accepted parameters: `chromium`, `firefox`, `webkit` + - Example usage: + ```bash + npm run test -- --test=about --browser=chromium + npm run test -- --test=about --browser=firefox --debug + npm run test -- --test=about --browser=webkit --headed + ``` -## Running individual tests by category (headed mode) + ### Test Execution by Test Title -- run about tests: `npm run test:about:headed` -- run av tests: `npm run test:av:headed` -- run books tests: `npm run test:books:headed` -- run collection tests: `npm run test:collection:headed` -- run details tests: `npm run test:details:headed` -- run home tests: `npm run test:home:headed` -- run login tests: `npm run test:login:headed` -- run music tests: `npm run test:music:headed` -- run search tests: `npm run test:search:headed` -- run profile tests: `npm run test:profile:headed` + - Running tests matching specific test case titles: + - Searches for test titles within `.spec.ts` files. + - Parameter: `--title` + - Example usage: + ```bash + npm run test -- --title="TV has borrow button" + npm run test -- --title="Canonical About page has correct title and text" --browser=chromium + npm run test -- --title="Canonical About page has correct title and text" --debug + ``` + + ### Use helper function to generate custom test commands + + - To generate and execute custom test commands using the `generateCommand.js` script, follow these steps: + + - **Setup script configuration:** + - Ensure you have `generateCommand.js` configured with the desired parameters (`test`, `browser`, `title`, `headed`, `trace`, `debug`, `ui`, `device`) + + - Adjust these parameters as needed for your specific testing scenarios + + Example configuration: + ```javascript + const sampleArgs = { + test: 'books', + browser: 'chromium', // Options: 'chromium', 'firefox', 'webkit'; default: all browsers + title: '', // Specify a test case title within "" + headed: false, // Set to true for headed mode + trace: false, // Set to true to enable trace mode + debug: false, // Set to true to enable debug mode + ui: true, // Set to true to enable UI mode + device: '' // Options: 'mobile', 'desktop'; default: desktop + }; + ``` + + - **Generating custom test command:** + + - Run the script to generate a command string that includes the specified parameters: + + ```bash + npm run generate-command + ``` + + - This will output a generated command that can be used to execute specific tests based on your configuration: + + ```bash + ➜ npm run generate:command + + > iaux-e2e-playwright-tests@1.0.0 generate:command + > node scripts/generateCommand.js + + Generated command: npm run test -- --test=books --browser=chromium --ui + ``` + + - Copy the generated command and execute it in your terminal to run tests with the specified configurations ## Running tests using VSCode Playwright plugin @@ -67,20 +172,45 @@ ## Running specific test spec by file: -- run command format: `npx playwright test ` -- sample: `npx playwright test tests/search/search-layout.spec.ts` -- headed: `npx playwright test tests/search/search-layout.spec.ts --headed` +- Command format: `npx playwright test ` +- Sample (headless): + ```bash + npx playwright test tests/search/search-layout.spec.ts + ``` + +- Sample (headed): + ```bash + npx playwright test tests/search/search-layout.spec.ts --headed + ``` ## Running specific test spec by file in debug mode: -- run command format: `npx playwright test --debug` -- sample: `npx playwright test tests/search/search-layout.spec.ts --debug` +- Command format: `npx playwright test --debug` + +- Sample: + ```bash + npx playwright test tests/search/search-layout.spec.ts --debug + ``` + + +## Running specific test case title: + +- Command format: `npx playwright test -g ""` + +- Sample: + ```bash + npx playwright test -g "TV has borrow button" + ``` ## View tests execution result -- run: `npm run show:report` +- Run command: + + ```bash + npm run show:report + ``` ## Reference guide for writing tests diff --git a/package-lock.json b/package-lock.json index 48caf170..2c285f66 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,14 +9,14 @@ "version": "1.0.0", "license": "ISC", "devDependencies": { - "@playwright/test": "^1.41.2", + "@playwright/test": "^1.44.1", "@types/node": "^20.11.17", "browserstack-local": "^1.4.8", "dotenv": "^16.0.3", "eslint": "^8.44.0", "eslint-config-prettier": "^8.8.0", "husky": "^8.0.3", - "playwright": "^1.41.2", + "minimist": "^1.2.8", "prettier": "^3.0.0" } }, @@ -154,12 +154,12 @@ } }, "node_modules/@playwright/test": { - "version": "1.41.2", - "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.41.2.tgz", - "integrity": "sha512-qQB9h7KbibJzrDpkXkYvsmiDJK14FULCCZgEcoe2AvFAS64oCirWTwzTlAYEbKaRxWs5TFesE1Na6izMv3HfGg==", + "version": "1.44.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.44.1.tgz", + "integrity": "sha512-1hZ4TNvD5z9VuhNJ/walIjvMVvYkZKf71axoF/uiAqpntQJXpG64dlXhoDXE3OczPuTuvjf/M5KWFg5VAVUS3Q==", "dev": true, "dependencies": { - "playwright": "1.41.2" + "playwright": "1.44.1" }, "bin": { "playwright": "cli.js" @@ -953,6 +953,15 @@ "node": "*" } }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "dev": true, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -1070,12 +1079,12 @@ } }, "node_modules/playwright": { - "version": "1.41.2", - "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.41.2.tgz", - "integrity": "sha512-v0bOa6H2GJChDL8pAeLa/LZC4feoAMbSQm1/jF/ySsWWoaNItvrMP7GEkvEEFyCTUYKMxjQKaTSg5up7nR6/8A==", + "version": "1.44.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.44.1.tgz", + "integrity": "sha512-qr/0UJ5CFAtloI3avF95Y0L1xQo6r3LQArLIg/z/PoGJ6xa+EwzrwO5lpNr/09STxdHuUoP2mvuELJS+hLdtgg==", "dev": true, "dependencies": { - "playwright-core": "1.41.2" + "playwright-core": "1.44.1" }, "bin": { "playwright": "cli.js" @@ -1087,10 +1096,10 @@ "fsevents": "2.3.2" } }, - "node_modules/playwright/node_modules/playwright-core": { - "version": "1.41.2", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.41.2.tgz", - "integrity": "sha512-VaTvwCA4Y8kxEe+kfm2+uUUw5Lubf38RxF7FpBxLPmGe5sdNkSg5e3ChEigaGrX7qdqT3pt2m/98LiyvU2x6CA==", + "node_modules/playwright-core": { + "version": "1.44.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.44.1.tgz", + "integrity": "sha512-wh0JWtYTrhv1+OSsLPgFzGzt67Y7BE/ZS3jEqgGBlp2ppp1ZDj8c+9IARNW4dwf1poq5MgHreEM2KV/GuR4cFA==", "dev": true, "bin": { "playwright-core": "cli.js" @@ -1491,12 +1500,12 @@ } }, "@playwright/test": { - "version": "1.41.2", - "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.41.2.tgz", - "integrity": "sha512-qQB9h7KbibJzrDpkXkYvsmiDJK14FULCCZgEcoe2AvFAS64oCirWTwzTlAYEbKaRxWs5TFesE1Na6izMv3HfGg==", + "version": "1.44.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.44.1.tgz", + "integrity": "sha512-1hZ4TNvD5z9VuhNJ/walIjvMVvYkZKf71axoF/uiAqpntQJXpG64dlXhoDXE3OczPuTuvjf/M5KWFg5VAVUS3Q==", "dev": true, "requires": { - "playwright": "1.41.2" + "playwright": "1.44.1" } }, "@types/node": { @@ -1518,8 +1527,7 @@ "version": "5.3.2", "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", - "dev": true, - "requires": {} + "dev": true }, "agent-base": { "version": "6.0.2", @@ -1733,8 +1741,7 @@ "version": "8.8.0", "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-8.8.0.tgz", "integrity": "sha512-wLbQiFre3tdGgpDv67NQKnJuTlcUVYHas3k+DZCc2U2BadthoEY4B7hLPvAxaqdyOGCzuLfii2fqGph10va7oA==", - "dev": true, - "requires": {} + "dev": true }, "eslint-scope": { "version": "7.2.0", @@ -2092,6 +2099,12 @@ "brace-expansion": "^1.1.7" } }, + "minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "dev": true + }, "ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -2182,23 +2195,21 @@ } }, "playwright": { - "version": "1.41.2", - "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.41.2.tgz", - "integrity": "sha512-v0bOa6H2GJChDL8pAeLa/LZC4feoAMbSQm1/jF/ySsWWoaNItvrMP7GEkvEEFyCTUYKMxjQKaTSg5up7nR6/8A==", + "version": "1.44.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.44.1.tgz", + "integrity": "sha512-qr/0UJ5CFAtloI3avF95Y0L1xQo6r3LQArLIg/z/PoGJ6xa+EwzrwO5lpNr/09STxdHuUoP2mvuELJS+hLdtgg==", "dev": true, "requires": { "fsevents": "2.3.2", - "playwright-core": "1.41.2" - }, - "dependencies": { - "playwright-core": { - "version": "1.41.2", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.41.2.tgz", - "integrity": "sha512-VaTvwCA4Y8kxEe+kfm2+uUUw5Lubf38RxF7FpBxLPmGe5sdNkSg5e3ChEigaGrX7qdqT3pt2m/98LiyvU2x6CA==", - "dev": true - } + "playwright-core": "1.44.1" } }, + "playwright-core": { + "version": "1.44.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.44.1.tgz", + "integrity": "sha512-wh0JWtYTrhv1+OSsLPgFzGzt67Y7BE/ZS3jEqgGBlp2ppp1ZDj8c+9IARNW4dwf1poq5MgHreEM2KV/GuR4cFA==", + "dev": true + }, "prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", diff --git a/package.json b/package.json index e7ab4a72..88a1d50f 100644 --- a/package.json +++ b/package.json @@ -5,37 +5,9 @@ "main": "index.js", "scripts": { "show:report": "npx http-server ./playwright-report", - "test": "CATEGORY=all npx playwright test", + "test": "node scripts/executeTests.js", + "generate-command": "node scripts/generateCommand.js", "test:codegen": "npx playwright codegen", - "test:chromium": "CATEGORY=allnpx playwright test --project=chromium", - "test:debug": "CATEGORY=all npx playwright test --debug", - "test:trace": "CATEGORY=all npx playwright test --config=./playwright.config.ts --trace on", - "test:ui": "CATEGORY=all npx playwright test --ui", - "test:headed": "CATEGORY=all npx playwright test --headed", - "test:about": "CATEGORY=about npx playwright test tests/about", - "test:av": "CATEGORY=av npx playwright test tests/av", - "test:books": "CATEGORY=books npx playwright test tests/books", - "test:collection": "CATEGORY=collection npx playwright test tests/collection", - "test:details": "CATEGORY=details npx playwright test tests/details", - "test:details-lending-bar": "CATEGORY=details npx playwright test tests/details/lending-bar", - "test:home": "CATEGORY=home npx playwright test tests/home", - "test:login": "CATEGORY=login npx playwright test tests/login", - "test:music": "CATEGORY=music npx playwright test tests/music", - "test:search": "CATEGORY=search npx playwright test tests/search", - "test:profile": "CATEGORY=profile npx playwright test tests/profile", - "test:about:headed": "CATEGORY=about npx playwright test tests/about --headed", - "test:av:headed": "CATEGORY=av npx playwright test tests/av --headed", - "test:books:headed": "CATEGORY=books npx playwright test tests/books --headed", - "test:collection:headed": "CATEGORY=collection npx playwright test tests/collection --headed", - "test:details:headed": "CATEGORY=details npx playwright test tests/details --headed", - "test:details-lending-bar:headed": "CATEGORY=details npx playwright test tests/details/lending-bar --headed", - "test:home:headed": "CATEGORY=home npx playwright test tests/home --headed", - "test:login:headed": "CATEGORY=login npx playwright test tests/login --headed", - "test:music:headed": "CATEGORY=music npx playwright test tests/music --headed", - "test:search:headed": "CATEGORY=search npx playwright test tests/search --headed", - "test:profile:headed": "CATEGORY=profile npx playwright test tests/profile --headed", - "test:donation:headed": "CATEGORY=donation npx playwright test tests/donation --headed", - "test:donation": "CATEGORY=donation npx playwright test tests/donation", "format": "prettier --write \"tests/**/*.ts\"", "lint": "prettier --check \"tests/**/*.ts\"", "typecheck": "node node_modules/typescript/bin/tsc --noEmit", @@ -45,14 +17,14 @@ "author": "", "license": "ISC", "devDependencies": { - "@playwright/test": "^1.41.2", + "@playwright/test": "^1.44.1", "@types/node": "^20.11.17", "browserstack-local": "^1.4.8", "dotenv": "^16.0.3", "eslint": "^8.44.0", "eslint-config-prettier": "^8.8.0", "husky": "^8.0.3", - "playwright": "^1.41.2", + "minimist": "^1.2.8", "prettier": "^3.0.0" }, "eslintConfig": { @@ -66,22 +38,14 @@ ], "rules": { "no-unused-vars": "off", - "@typescript-eslint/no-unused-vars": [ - "error" - ], + "@typescript-eslint/no-unused-vars": 2, "no-shadow": "off", - "@typescript-eslint/no-shadow": [ - "error" - ], + "@typescript-eslint/no-shadow": 2, "class-methods-use-this": "off", "import/no-unresolved": "off", "import/extensions": [ "off", - "ignorePackages", - { - "js": "never", - "ts": "never" - } + "ignorePackages" ], "no-unsafe-optional-chaining": "warn", "default-param-last": "warn" diff --git a/scripts/executeTests.js b/scripts/executeTests.js new file mode 100644 index 00000000..5a648a64 --- /dev/null +++ b/scripts/executeTests.js @@ -0,0 +1,72 @@ +const { execSync } = require('child_process'); + +const capitalizeFirstLetter = str => str.charAt(0).toUpperCase() + str.slice(1); + +const buildCommand = (options) => { + const { category, trace, headed, debug, ui, title, browser, platform } = options; + + let command = `CATEGORY=${category} npx playwright test`; + + if (category !== 'all' && category !== 'grep') { + command += ` tests/${category}`; + } + + if (trace) command += ' --trace on'; + if (headed) command += ' --headed'; + if (debug) command += ' --debug'; + if (ui) command += ' --ui'; + if (title) command += ` -g "${title}"`; + if (browser) { + const capitalizedBrowser = capitalizeFirstLetter(browser); + const capitalizedPlatform = capitalizeFirstLetter(platform); + command += ` --project='${capitalizedPlatform} - ${capitalizedBrowser}'`; + } + + return command; +} + +const parseArguments = (args) => { + const validCategories = ['about', 'av', 'books', 'collection', 'details', 'donation', 'home', 'login', 'music', 'profile', 'search']; + const validBrowserDevices = ['chromium', 'firefox', 'webkit']; + const validPlatforms = ['mobile', 'desktop']; + + const { test: testCategory, browser, title, headed, trace, debug, ui, device } = args; + + let category = testCategory && validCategories.includes(testCategory) ? testCategory : 'all'; + let selectedBrowser = validBrowserDevices.includes(browser) ? browser : ''; + let platform = validPlatforms.includes(device) ? device : 'desktop'; + + if (title) category = 'grep'; + + return { + category, + browser: selectedBrowser, + title, + headed: !!headed, + trace: !!trace, + debug: !!debug, + ui: !!ui, + platform + }; +} + +const executeCommand = (command) => { + try { + console.log(`Executing command: ${command}`); + const output = execSync(command, { stdio: 'inherit' }); + if (output !== null) { + console.log(output.toString()); + } + } catch (error) { + console.error('Error:', error.stderr ? error.stderr.toString() : error.toString()); + process.exit(1); + } +} + +const executeTests = (options) => { + executeCommand(buildCommand(options)); +} + +const args = require('minimist')(process.argv.slice(2)); +const parsedArgs = parseArguments(args); +executeTests(parsedArgs); diff --git a/scripts/generateCommand.js b/scripts/generateCommand.js new file mode 100644 index 00000000..af519c9a --- /dev/null +++ b/scripts/generateCommand.js @@ -0,0 +1,47 @@ +const generateCommand = (args) => { + // Format the arguments into a command string + const commandArgs = Object.entries(args) + .filter(([_, value]) => { + // Remove blank arguments + if (value === '' || value === undefined || value === false) { + return false; + } + return true; + }) + .map(([key, value]) => { + if (typeof value === 'boolean') { + return value ? `--${key}` : ''; + } else { + // If the argument is a title, surround it with "" + if (key === 'title') { + return `--${key}="${value}"`; + } else { + return `--${key}=${value}`; + } + } + }) + .join(' '); // Join the arguments with a space + + // Return the formatted command + return `npm run test -- ${commandArgs}`; +}; + +/** + * Test category options: 'about', 'av', 'books', 'collection', 'details', 'donation', 'home', 'login', 'music', 'profile', 'search' + * Browser options: 'chromium', 'firefox', 'webkit' + * Device options: 'mobile', 'desktop' + * Title: '' + */ +const sampleArgs = { + test: 'books', + browser: 'chromium', // default/blank: all browsers + title: '', + headed: false, + trace: false, + debug: false, + ui: true, + device: '' // default/blank: desktop +}; + +const sampleCommand = generateCommand(sampleArgs); +console.log('Generated command:', sampleCommand);