chore(e2e-tests): various improvements to test execution

- moved saml data input to slower method to help firefox & webkit to pass - made release tag tests more robust against retries - allowed for more browser speed differences - aligned webkit terminal fixture with updated layout - multiple lint cleanups Signed-off-by: Manuel Zedel <[email protected]>
mendersoftware · Oct 27, 2023 · 1db9d68 · 1db9d68
1 parent a9782b8
commit 1db9d68
Show file tree

Hide file tree

Showing 10 changed files with 88 additions and 52 deletions.
diff --git a/tests/e2e_tests/fixtures/terminalContent-webkit.png b/tests/e2e_tests/fixtures/terminalContent-webkit.png
diff --git a/tests/e2e_tests/integration/01-login.spec.ts b/tests/e2e_tests/integration/01-login.spec.ts
@@ -109,7 +109,7 @@ test.describe('Login', () => {
     await isLoggedIn(page);
     let loginVisible = await page.getByRole('button', { name: /log in/i }).isVisible();
     expect(loginVisible).toBeFalsy();
-    await page.getByText('Help & support').click();
+    await page.getByText(/Releases/i).click();
     const cookies = await context.cookies();
     await context.storageState({ path: 'storage.json' });
 

diff --git a/tests/e2e_tests/integration/02-layoutAssertions.spec.ts b/tests/e2e_tests/integration/02-layoutAssertions.spec.ts
@@ -41,7 +41,7 @@ test.describe('Layout assertions', () => {
     await page.click(`.leftNav :text('Devices')`);
     let hasAcceptedDevice = false;
     try {
-      hasAcceptedDevice = await page.isVisible('.deviceListItem');
+      hasAcceptedDevice = await page.isVisible(selectors.deviceListItem);
     } catch (e) {
       console.log(`no accepted device present so far`);
     }
@@ -55,10 +55,10 @@ test.describe('Layout assertions', () => {
     }
     await page.locator(`input:near(:text("Status:"))`).first().click({ force: true });
     await page.click(`css=.MuiPaper-root >> text=/Accepted/i`);
-    await page.waitForSelector(`css=.deviceListItem >> text=/original/`, { timeout: timeouts.sixtySeconds });
-    const element = await page.textContent('.deviceListItem');
+    await page.waitForSelector(`css=${selectors.deviceListItem} >> text=/original/`, { timeout: timeouts.sixtySeconds });
+    const element = await page.textContent(selectors.deviceListItem);
     expect(element.includes('original')).toBeTruthy();
-    await page.click(`.deviceListItem div:last-child`);
+    await page.click(`${selectors.deviceListItem} div:last-child`);
     await page.waitForSelector(`text=/Device information for/i`);
     expect(await page.isVisible('text=Authentication status')).toBeTruthy();
   });
@@ -78,6 +78,6 @@ test.describe('Layout assertions', () => {
     await page.click(`.grouplist:has-text('All devices')`);
     await page.click(selectors.deviceListCheckbox);
     await page.click(`.grouplist:has-text('testgroup')`);
-    expect(await page.locator(`css=.deviceListItem >> text=/original/`)).toBeVisible();
+    expect(await page.locator(`css=${selectors.deviceListItem} >> text=/original/`)).toBeVisible();
   });
 });
diff --git a/tests/e2e_tests/integration/03-files.spec.ts b/tests/e2e_tests/integration/03-files.spec.ts
@@ -43,6 +43,10 @@ test.describe('Files', () => {
     await page.click(`.leftNav :text('Releases')`);
     await page.getByText(/demo-artifact/i).click();
     expect(await page.getByRole('heading', { name: /Release notes/i }).isVisible()).toBeTruthy();
+    const hasNotes = await page.getByText('foo notes').isVisible();
+    if (hasNotes) {
+      return;
+    }
     // layout based locators are not an option here, since the edit button is also visible on the nearby tags section
     // and the selector would get confused due to the proximity - so instead we loop over all the divs
     await page
@@ -54,7 +58,7 @@ test.describe('Files', () => {
     await input.fill('foo notes');
     await page.getByTestId('CheckIcon').click();
     expect(input).not.toBeVisible();
-    expect(await page.getByText('foo notes').isVisible()).toBeTruthy();
+    expect(hasNotes).toBeTruthy();
   });
 
   test('allows release tags manipulation', async ({ baseUrl, loggedInPage: page }) => {
@@ -75,8 +79,10 @@ test.describe('Files', () => {
     const input = await page.getByPlaceholder(/enter release tags/i);
     await input.fill('some,tags');
     await page.getByTestId('CheckIcon').click();
+    await page.waitForTimeout(timeouts.oneSecond);
     expect(input).not.toBeVisible();
     await page.goto(`${baseUrl}ui/releases`);
+    await page.waitForSelector('text="some, tags"', { timeout: timeouts.oneSecond });
     expect(await page.getByText('some, tags').isVisible()).toBeTruthy();
   });
 
@@ -91,30 +97,41 @@ test.describe('Files', () => {
     await editButton.click();
     const alreadyTagged = await page.getByRole('button', { name: 'some' }).isVisible();
     if (alreadyTagged) {
-      await page.getByRole('button', { name: 'some' }).getByTestId('CancelIcon').click();
-      await page.getByRole('button', { name: 'tags' }).getByTestId('CancelIcon').click();
+      await Promise.all(
+        ['some', 'tags'].map(async name => {
+          const foundTag = await page.getByRole('button', { name });
+          if (!(await foundTag.isVisible())) {
+            return Promise.resolve();
+          }
+          return await foundTag.getByTestId('CancelIcon').click();
+        })
+      );
       await page.getByTestId('CheckIcon').click();
-      expect(await page.getByText('add release tags').isVisible()).toBeTruthy();
+      // await page.waitForTimeout(timeouts.oneSecond);
+      expect(await page.getByPlaceholder(/add release tags/i).isVisible({ timeout: timeouts.oneSecond })).toBeTruthy();
       await editButton.click();
     }
     await page.getByPlaceholder(/enter release tags/i).fill(releaseTag);
     await page.getByTestId('CheckIcon').click();
     await page.press('body', 'Escape');
-    expect(await page.getByText(releaseTag).isVisible()).toBeTruthy();
+    await page.waitForSelector('text=Upload', { timeout: timeouts.oneSecond });
+    await page.press('body', 'Escape');
+    expect(await page.getByText(releaseTag, { exact: false }).isVisible({ timeout: timeouts.oneSecond })).toBeTruthy();
   });
 
   test('allows release tags filtering', async ({ loggedInPage: page }) => {
     await page.click(`.leftNav :text('Releases')`);
     expect(await page.getByText(releaseTag.toLowerCase()).isVisible()).toBeTruthy();
     await page.getByPlaceholder(/select tags/i).fill('foo,');
-    await page.waitForTimeout(timeouts.oneSecond);
-    expect(await page.getByText(/There are no Releases/i).isVisible()).toBeTruthy();
+    const releasesNote = await page.getByText(/There are no Releases*/i);
+    releasesNote.waitFor({ timeout: timeouts.default });
+    await page.getByText(/mender-demo-artifact*/i).waitFor({ timeout: timeouts.default, state: 'detached' });
     await page.getByText(/Clear filter/i).click();
-    await page.waitForTimeout(timeouts.oneSecond);
+    await page.waitForSelector('text=/mender-demo-artifact*/i');
     expect(await page.getByText(releaseTag.toLowerCase()).isVisible()).toBeTruthy();
     await page.getByPlaceholder(/select tags/i).fill(`${releaseTag.toLowerCase()},`);
-    await page.waitForTimeout(timeouts.oneSecond);
-    expect(await page.getByText(/There are no Releases/i).isVisible()).not.toBeTruthy();
+    await page.getByText(/mender-demo-artifact*/i).waitFor({ timeout: timeouts.default });
+    expect(await releasesNote.isVisible()).toBeFalsy();
   });
 
   // test('allows uploading custom file creations', () => {
@@ -156,7 +173,7 @@ test.describe('Files', () => {
     // TODO adjust test to better work with webkit, for now it should be good enough to assume file transfers work there too if the remote terminal works
     test.skip(!['enterprise', 'staging'].includes(environment) || ['webkit'].includes(browserName));
     await page.click(`.leftNav :text('Devices')`);
-    await page.click(`.deviceListItem div:last-child`);
+    await page.click(`${selectors.deviceListItem} div:last-child`);
     await page.click(`text=/troubleshooting/i`);
     // the deviceconnect connection might not be established right away
     await page.waitForSelector('text=/Session status/i', { timeout: timeouts.tenSeconds });

diff --git a/tests/e2e_tests/integration/04-deployments.spec.ts b/tests/e2e_tests/integration/04-deployments.spec.ts
@@ -35,14 +35,15 @@ test.describe('Deployments', () => {
     await page.click('.MuiSpeedDial-fab');
     await page.click('[aria-label="deploy"]');
     await page.waitForSelector(selectors.deviceGroupSelect, { timeout: timeouts.fiveSeconds });
-    await page.focus(selectors.deviceGroupSelect);
-    await page.getByPlaceholder(/select a device group/i).fill('All');
+    const deviceGroupSelect = await page.getByPlaceholder(/select a device group/i);
+    await deviceGroupSelect.focus();
+    await deviceGroupSelect.fill('All');
     await page.click(`#deployment-device-group-selection-listbox li:has-text('All devices')`);
-    const creationButton = await page.waitForSelector(selectors.deploymentCreation);
+    const creationButton = await page.getByRole('button', { name: /create deployment/i });
     await creationButton.scrollIntoViewIfNeeded();
     await creationButton.click();
     await page.waitForSelector(selectors.deploymentListItem, { timeout: timeouts.tenSeconds });
-    await page.click(`[role="tab"]:has-text('Finished')`);
+    await page.getByRole('tab', { name: /finished/i }).click();
     await page.waitForSelector(selectors.deploymentListItemContent, { timeout: timeouts.sixtySeconds });
     const datetime = await page.getAttribute(`${selectors.deploymentListItemContent} time`, 'datetime');
     const time = dayjs(datetime);
@@ -58,7 +59,7 @@ test.describe('Deployments', () => {
     await page.click('.MuiSpeedDial-fab');
     await page.click('[aria-label="create-deployment"]');
     await page.waitForSelector(selectors.releaseSelect, { timeout: timeouts.fiveSeconds });
-    const releaseSelect = await page.locator(selectors.releaseSelect);
+    const releaseSelect = await page.getByPlaceholder(/select a release/i);
     await releaseSelect.focus();
     await releaseSelect.fill('mender-demo');
     await page.click(`#deployment-release-selection-listbox li`);
@@ -68,11 +69,11 @@ test.describe('Deployments', () => {
     await releaseSelect.focus();
     await releaseSelect.fill('mender-demo');
     await page.click(`#deployment-release-selection-listbox li`);
-    const creationButton = await page.waitForSelector(selectors.deploymentCreation);
+    const creationButton = await page.getByRole('button', { name: /create deployment/i });
     await creationButton.scrollIntoViewIfNeeded();
     await creationButton.click();
-    await page.waitForSelector(selectors.deploymentListItem, { timeout: timeouts.tenSeconds });
-    await page.click(`[role="tab"]:has-text('Finished')`);
+    await expect(page.getByText(/Select a Release to deploy/i)).toHaveCount(0, { timeout: timeouts.tenSeconds });
+    await page.getByRole('tab', { name: /finished/i }).click();
     await page.waitForSelector(selectors.deploymentListItemContent, { timeout: timeouts.sixtySeconds });
     const datetime = await page.getAttribute(`${selectors.deploymentListItemContent} time`, 'datetime');
     const time = dayjs(datetime);
@@ -86,21 +87,22 @@ test.describe('Deployments', () => {
     await page.click(`button:has-text('Create a deployment')`);
 
     await page.waitForSelector(selectors.releaseSelect, { timeout: timeouts.fiveSeconds });
-    const releaseSelect = await page.locator(selectors.releaseSelect);
+    const releaseSelect = await page.getByPlaceholder(/select a release/i);
     await releaseSelect.focus();
     await releaseSelect.fill('mender');
     await page.click(`#deployment-release-selection-listbox li:has-text('mender-demo-artifact')`);
 
     await page.waitForSelector(selectors.deviceGroupSelect, { timeout: timeouts.fiveSeconds });
-    const deviceGroupSelect = await page.locator(selectors.deviceGroupSelect);
+    const deviceGroupSelect = await page.getByPlaceholder(/select a device group/i);
     await deviceGroupSelect.focus();
     await deviceGroupSelect.fill('test');
     await page.click(`#deployment-device-group-selection-listbox li:has-text('testgroup')`);
-    const creationButton = await page.waitForSelector(selectors.deploymentCreation);
+    const creationButton = await page.getByRole('button', { name: /create deployment/i });
     await creationButton.scrollIntoViewIfNeeded();
     await creationButton.click();
+    await expect(page.getByText(/Select a Release to deploy/i)).toHaveCount(0, { timeout: timeouts.tenSeconds });
     await page.waitForSelector(selectors.deploymentListItem, { timeout: timeouts.tenSeconds });
-    await page.click(`[role="tab"]:has-text('Finished')`);
+    await page.getByRole('tab', { name: /finished/i }).click();
     await page.waitForSelector(selectors.deploymentListItemContent, { timeout: timeouts.sixtySeconds });
   });
 });
diff --git a/tests/e2e_tests/integration/05-deviceDetails.spec.ts b/tests/e2e_tests/integration/05-deviceDetails.spec.ts
@@ -29,7 +29,7 @@ test.describe('Device details', () => {
 
   test('has basic inventory', async ({ demoDeviceName, loggedInPage: page }) => {
     await page.click(`.leftNav :text('Devices')`);
-    await page.click(`.deviceListItem div:last-child`);
+    await page.click(`${selectors.deviceListItem} div:last-child`);
     await page.click(`text=/inventory/i`);
     expect(await page.isVisible(`css=.expandedDevice >> text=Linux`)).toBeTruthy();
     expect(await page.isVisible(`css=.expandedDevice >> text=mac`)).toBeTruthy();
@@ -39,11 +39,11 @@ test.describe('Device details', () => {
   test('can be found', async ({ demoDeviceName, loggedInPage: page }) => {
     const searchField = await page.getByPlaceholder(/search devices/i);
     await searchField.fill(demoDeviceName);
-    await page.waitForSelector('.deviceListItem');
+    await page.waitForSelector(selectors.deviceListItem);
     const slideOut = await page.locator('.MuiPaper-root');
     expect(await slideOut.locator(`:text("${demoDeviceName}"):below(:text("clear search"))`).isVisible()).toBeTruthy();
     expect(await slideOut.getByText('1-1 of 1').isVisible()).toBeTruthy();
-    await page.click(`.deviceListItem`);
+    await page.click(selectors.deviceListItem);
     await page.waitForSelector('text=/device information/i');
     expect(await page.getByText(/Authorization sets/i).isVisible()).toBeTruthy();
     await page.click('[aria-label="close"]');
@@ -53,31 +53,42 @@ test.describe('Device details', () => {
     expect(await page.getByText(/device found/i).isVisible()).toBeTruthy();
   });
 
-  test('can be filtered', async ({ demoDeviceName, environment, loggedInPage: page }) => {
+  test('can be filtered', async ({ browserName, demoDeviceName, loggedInPage: page }) => {
+    test.setTimeout(2 * timeouts.fifteenSeconds);
     await page.click(`.leftNav :text('Devices')`);
     await page.getByRole('button', { name: /filters/i }).click();
     await page.getByLabel(/attribute/i).fill(rootfs);
     const nameInput = await page.getByLabel(/value/i);
     await nameInput.fill(demoDeviceName);
     await page.waitForTimeout(timeouts.oneSecond);
     await nameInput.press('Enter');
-    expect(await page.getByRole('button', { name: `${rootfs} = ${demoDeviceName}` }).isVisible()).toBeTruthy();
-    await page.waitForSelector('.deviceListItem');
-    expect(await page.getByText('1-1 of 1').isVisible()).toBeTruthy();
-    await page.getByText(/clear filter/i).click();
-    if (['enterprise', 'staging'].includes(environment)) {
-      await page.getByLabel(/attribute/i).fill(rootfs);
-      await page.getByText(/equals/i).click();
-      await page.getByText(`doesn't exist`).click();
+    if (browserName === 'webkit') {
       await page.waitForTimeout(timeouts.fiveSeconds);
-      expect(await page.getByRole('button', { name: `${rootfs} doesn't exist` }).isVisible()).toBeTruthy();
-      expect(await page.getByText('No devices found').isVisible()).toBeTruthy();
     }
+    expect(await page.getByRole('button', { name: `${rootfs} = ${demoDeviceName}` }).isVisible({ timeout: timeouts.default })).toBeTruthy();
+    await page.waitForSelector(selectors.deviceListItem);
+  });
+
+  test('can be filtered into non-existence', async ({ environment, loggedInPage: page }) => {
+    test.skip(!['enterprise', 'staging'].includes(environment), 'not available in OS');
+    test.setTimeout(2 * timeouts.fifteenSeconds);
+    await page.click(`.leftNav :text('Devices')`);
+    await page.getByRole('button', { name: /filters/i }).click();
+    await page.getByLabel(/attribute/i).fill(rootfs);
+    await page.getByText(/equals/i).click();
+    await page.getByText(`doesn't exist`).click();
+    await page.waitForTimeout(timeouts.fiveSeconds);
+    expect(await page.getByRole('button', { name: `${rootfs} doesn't exist` }).isVisible()).toBeTruthy();
+    expect(await page.getByText('No devices found').isVisible()).toBeTruthy();
+    await page.getByText(/clear filter/i).click();
+    await page.waitForSelector(selectors.deviceListItem);
+    const paginationVisible = await page.getByText('1-1').isVisible({ timeout: timeouts.default });
+    expect(paginationVisible).toBeTruthy();
   });
 
   test('can open a terminal', async ({ browserName, loggedInPage: page }) => {
     await page.click(`.leftNav :text('Devices')`);
-    await page.click(`.deviceListItem div:last-child`);
+    await page.click(`${selectors.deviceListItem} div:last-child`);
     await page.click(`text=/troubleshooting/i`);
     // the deviceconnect connection might not be established right away
     await page.waitForSelector('text=/Session status/i', { timeout: timeouts.tenSeconds });

diff --git a/tests/e2e_tests/integration/06-auditlogs.spec.ts b/tests/e2e_tests/integration/06-auditlogs.spec.ts
@@ -26,7 +26,7 @@ test.describe('Auditlogs', () => {
   test('will track remote terminal sessions', async ({ environment, loggedInPage: page }) => {
     test.skip(!['enterprise', 'staging'].includes(environment));
     await page.click(`.leftNav :text('Devices')`);
-    await page.click(`.deviceListItem div:last-child`);
+    await page.click(`${selectors.deviceListItem} div:last-child`);
     await page.click(`text=/troubleshooting/i`);
     // the deviceconnect connection might not be established right away
     await page.waitForSelector('text=/Session status/i', { timeout: timeouts.tenSeconds });

diff --git a/tests/e2e_tests/integration/07-saml.spec.ts b/tests/e2e_tests/integration/07-saml.spec.ts
@@ -59,7 +59,7 @@ test.describe('SAML Login via sso/id/login', () => {
   });
 
   // Setups the SAML/SSO login with samltest.id Identity Provider
-  test('Set up SAML', async ({ context, environment, baseUrl, page }) => {
+  test('Set up SAML', async ({ browserName, context, environment, baseUrl, loggedInPage: page }) => {
     test.skip(environment !== 'staging');
     // allow a lot of time to enter metadata + then some to handle uploading the config to the external service
     test.setTimeout(5 * timeouts.sixtySeconds + timeouts.fifteenSeconds);
@@ -76,7 +76,12 @@ test.describe('SAML Login via sso/id/login', () => {
       await page.locator('text=input with the text editor').click();
 
       const textfield = await page.locator('[aria-label="Editor content\\;Press Alt\\+F1 for Accessibility Options\\."]');
-      await textfield.fill(metadata.replace(/(?:\r\n|\r|\n)/g, ''));
+      const cleanedMetaData = metadata.replace(/(?:\r\n|\r|\n)/g, '');
+      if (browserName === 'firefox') {
+        await textfield.pressSequentially(cleanedMetaData);
+      } else {
+        await textfield.fill(cleanedMetaData);
+      }
       console.log('typing metadata done.');
       // The screenshot saves the view of the typed metadata
       await page.screenshot({ 'path': 'saml-edit-saving.png' });
@@ -122,7 +127,7 @@ test.describe('SAML Login via sso/id/login', () => {
   });
 
   // Creates a user with login that matches Identity privder (samltest.id) user email
-  test('Creates a user without a password', async ({ environment, browserName, baseUrl, page }) => {
+  test('Creates a user without a password', async ({ environment, browserName, baseUrl, loggedInPage: page }) => {
     test.skip(environment !== 'staging');
     await page.goto(`${baseUrl}ui/settings/user-management`);
     const userExists = await page.isVisible(`text=${samlSettings.credentials[browserName].email}`);
@@ -147,7 +152,7 @@ test.describe('SAML Login via sso/id/login', () => {
   // and verifies that login is successful.
   test('User can login via sso/login endpoint', async ({ environment, browserName, baseUrl, browser, loggedInPage }) => {
     test.skip(environment !== 'staging');
-    test.setTimeout(2 * timeouts.fifteenSeconds);
+    test.setTimeout(3 * timeouts.fifteenSeconds);
 
     await loggedInPage.goto(`${baseUrl}ui/settings/organization-and-billing`);
     await loggedInPage.waitForSelector('text=View metadata in the text editor', { timeout: timeouts.tenSeconds });

diff --git a/tests/e2e_tests/integration/08-rbac.spec.ts b/tests/e2e_tests/integration/08-rbac.spec.ts
@@ -111,7 +111,7 @@ test.describe('RBAC functionality', () => {
     expect(await page.isVisible(`css=button >> text=Upload`)).toBeFalsy();
 
     await page.click(`.leftNav :text('Devices')`);
-    await page.click(`.deviceListItem div:last-child`);
+    await page.click(`${selectors.deviceListItem} div:last-child`);
     // the created role does have permission to configure devices, so the section should be visible
     await page.click(`text=/configuration/i`);
     await page.waitForSelector('text=/Device configuration/i', { timeout: timeouts.tenSeconds });