ci/e2e tests against mobile (#55347)

Co-authored-by: sembauke <semboot699@gmail.com>

.github/workflows/e2e-playwright.yml

@@ -77,7 +77,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        browsers: [chromium, firefox, webkit]
+        # Not Mobile Safari until we can get it working. Webkit and Mobile
+        # Chrome both work, so hopefully there are no Mobile Safari specific
+        # bugs.
+        browsers: [chromium, firefox, webkit, Mobile Chrome]
         node-version: [20.x]
 
     services:
@@ -143,7 +146,7 @@ jobs:
         run: |
           pnpm run start-ci &
           sleep 10
-          npx playwright test --project=${{ matrix.browsers }} --grep-invert 'third-party-donation.spec.ts'
+          npx playwright test --project="${{ matrix.browsers }}" --grep-invert 'third-party-donation.spec.ts'
 
       - uses: actions/upload-artifact@v4
         if: ${{ !cancelled() }}

client/src/templates/Challenges/classic/mobile-layout.tsx

@@ -285,6 +285,7 @@ class MobileLayout extends Component<MobileLayoutProps, MobileLayoutState> {
             <TabsContent
               tabIndex={-1}
               className='tab-content'
+              data-playwright-test-label='preview-pane'
               value={tabs.preview}
               forceMount
               // forceMount causes the preview tabpanel to never be hidden,

e2e/challenge-reset-modal.spec.ts

@@ -44,14 +44,14 @@ test('should render the modal content correctly', async ({ page }) => {
 });
 
 test('User can reset challenge', async ({ page, isMobile, browserName }) => {
-  const initialText = 'CatPhotoApp';
-  const initialFrame = page
-    .frameLocator('iframe[title="challenge preview"]')
+  const initialText = '    <h2>Cat Photos</h2>';
+  const initialEditorText = page
+    .getByTestId('editor-container-indexhtml')
     .getByText(initialText);
 
   const updatedText = 'Only Dogs';
-  const updatedFrame = page
-    .frameLocator('iframe[title="challenge preview"]')
+  const updatedEditorText = page
+    .getByTestId('editor-container-indexhtml')
     .getByText(updatedText);
 
   await page.goto(
@@ -59,14 +59,14 @@ test('User can reset challenge', async ({ page, isMobile, browserName }) => {
   );
 
   // Building the preview can take a while
-  await expect(initialFrame).toBeVisible({ timeout: 10000 });
+  await expect(initialEditorText).toBeVisible();
 
   // Modify the text in the editor pane, clearing first, otherwise the existing
   // text will be selected before typing
   await focusEditor({ page, isMobile });
   await clearEditor({ page, browserName });
   await getEditors(page).fill(updatedText);
-  await expect(updatedFrame).toBeVisible({ timeout: 10000 });
+  await expect(updatedEditorText).toBeVisible();
 
   // Run the tests so the lower jaw updates (later we confirm that the update
   // are reset)
@@ -89,7 +89,7 @@ test('User can reset challenge', async ({ page, isMobile, browserName }) => {
     .click();
 
   // Check it's back to the initial state
-  await expect(initialFrame).toBeVisible({ timeout: 10000 });
+  await expect(initialEditorText).toBeVisible();
   await expect(
     page.getByText(translations.learn['sorry-keep-trying'])
   ).not.toBeVisible();
@@ -101,7 +101,7 @@ test('User can reset classic challenge', async ({ page, isMobile }) => {
   );
 
   const challengeSolution = '// This is in-line comment';
-
+  await focusEditor({ page, isMobile });
   await getEditors(page).fill(challengeSolution);
 
   const submitButton = page.getByRole('button', {
@@ -112,16 +112,25 @@ test('User can reset classic challenge', async ({ page, isMobile }) => {
   await expect(
     page.locator('.view-lines').getByText(challengeSolution)
   ).toBeVisible();
+
+  if (isMobile) {
+    await page
+      .getByText(translations.learn['editor-tabs'].instructions)
+      .click();
+  }
+
   await expect(
     page.getByLabel(translations.icons.passed).locator('circle')
   ).toBeVisible();
-  await expect(
-    page.getByText(translations.learn['tests-completed'])
-  ).toBeVisible();
 
   await page
-    .getByRole('button', { name: translations.buttons['reset-lesson'] })
+    .getByRole('button', {
+      name: !isMobile
+        ? translations.buttons['reset-lesson']
+        : translations.buttons.reset
+    })
     .click();
+
   await page
     .getByRole('button', { name: translations.buttons['reset-lesson'] })
     .click();
@@ -135,6 +144,11 @@ test('User can reset classic challenge', async ({ page, isMobile }) => {
   await expect(
     page.getByText(translations.learn['tests-completed'])
   ).not.toBeVisible();
+
+  if (isMobile) {
+    await page.getByText(translations.learn['editor-tabs'].console).click();
+  }
+
   await expect(page.getByText(translations.learn['test-output'])).toBeVisible();
 });
 

e2e/code-storage.spec.ts

@@ -4,6 +4,7 @@ import { getEditors } from './utils/editor';
 
 test.use({ storageState: 'playwright/.auth/certified-user.json' });
 test.describe('Challenge with editor', function () {
+  test.skip(({ isMobile }) => isMobile);
   test('the shortcut "Ctrl + S" saves the code', async ({ page }) => {
     await page.goto(
       '/learn/2022/responsive-web-design/learn-html-by-building-a-cat-photo-app/step-2'

e2e/editor.spec.ts

@@ -1,6 +1,6 @@
 import { APIRequestContext, expect, test } from '@playwright/test';
 
-import { clearEditor, focusEditor } from './utils/editor';
+import { clearEditor, focusEditor, getEditors } from './utils/editor';
 import { authedRequest } from './utils/request';
 
 const setTheme = async (
@@ -41,15 +41,22 @@ test.describe('Python Terminal', () => {
     );
 
     await focusEditor({ page, isMobile });
-    await clearEditor({ page, browserName });
+    await clearEditor({ page, browserName, isMobile });
     // Then enter invalid code
-    await page.keyboard.insertText('def');
+    await getEditors(page).fill('def');
+
+    if (isMobile) {
+      await page.getByRole('tab', { name: 'Preview' }).click();
+    }
+
     const preview = page.getByTestId('preview-pane');
 
     // While it's displayed on multiple lines, the string itself has no newlines, hence:
     const error = `Traceback (most recent call last):  File "main.py", line 1    def       ^SyntaxError: invalid syntax`;
     // It shouldn't take this long, but the Python worker can be slow to respond.
-    await expect(preview).toContainText(error, { timeout: 15000 });
+    await expect(preview.getByText(error)).toContainText(error, {
+      timeout: 15000
+    });
   });
 });
 

e2e/help-button.spec.ts

@@ -55,7 +55,7 @@ test.describe('help-button tests for a page with a reset and help button', () =>
     page
   }) => {
     await page.goto(
-      'learn/2022/responsive-web-design/learn-html-by-building-a-cat-photo-app/step-8'
+      'learn/2022/responsive-web-design/learn-html-by-building-a-cat-photo-app/step-3'
     );
     const checkButton = page.getByTestId('lowerJaw-check-button');
     await checkButton.click();

e2e/hotkeys.spec.ts

@@ -86,6 +86,9 @@ test.afterEach(
     })
 );
 
+// TODO: handle keyboard shortcuts on mobile
+test.skip(({ isMobile }) => isMobile, 'Only test on desktop');
+
 test('User can use shortcuts in and around the editor', async ({ page }) => {
   await page.goto(links.basicJS1);
 

e2e/multifile-cert-projects.spec.ts

@@ -26,12 +26,16 @@ test.describe('multifileCertProjects', () => {
     await page.keyboard.type('save1text');
     await expect(page.getByText('save1text')).toBeVisible();
 
-    await page.getByRole('button', { name: 'Save your Code' }).click();
+    await page
+      .getByRole('button', { name: !isMobile ? 'Save your Code' : 'Save' })
+      .click();
 
     await expect(page.getByTestId('flash-message')).toContainText(success);
 
     await page.reload();
 
+    await focusEditor({ page, isMobile });
+
     await expect(page.getByText('save1text')).toBeVisible();
   });
 
@@ -40,6 +44,8 @@ test.describe('multifileCertProjects', () => {
     isMobile,
     browserName
   }) => {
+    test.skip(isMobile);
+
     await focusEditor({ page, isMobile });
     await clearEditor({ page, browserName });
 
@@ -58,7 +64,6 @@ test.describe('multifileCertProjects', () => {
     await page.reload();
 
     await expect(page.getByText('save2text')).toBeVisible();
-
     await focusEditor({ page, isMobile });
 
     await page.keyboard.down('Control');

e2e/output.spec.ts

@@ -1,7 +1,7 @@
 import { test, expect, type Page } from '@playwright/test';
 
 import translations from '../client/i18n/locales/english/translations.json';
-import { clearEditor, getEditors } from './utils/editor';
+import { clearEditor, focusEditor, getEditors } from './utils/editor';
 
 const outputTexts = {
   default: `
@@ -25,29 +25,14 @@ interface InsertTextParameters {
   text: string;
 }
 
-const insertTextInCodeEditor = async ({
-  page,
-  isMobile,
-  text
-}: InsertTextParameters) => {
-  if (isMobile) {
-    await page
-      .getByRole('tab', { name: translations.learn['editor-tabs'].code })
-      .click();
-  }
+const insertTextInCodeEditor = async ({ page, text }: InsertTextParameters) => {
   await getEditors(page).fill(text);
-  if (isMobile) {
-    await page
-      .getByRole('tab', { name: translations.learn['editor-tabs'].console })
-      .click();
-  }
 };
 
 const runChallengeTest = async (page: Page, isMobile: boolean) => {
   if (isMobile) {
-    await page.getByRole('tab', { name: 'Code' }).click();
-    await page.getByText('Run').click();
     await page.getByRole('tab', { name: 'Console' }).click();
+    await page.getByText('Run').click();
   } else {
     await page.getByText('Run the Tests (Ctrl + Enter)').click();
   }
@@ -60,7 +45,11 @@ test.describe('For classic challenges', () => {
     );
   });
 
-  test('it renders the default output', async ({ page }) => {
+  test('it renders the default output', async ({ page, isMobile }) => {
+    if (isMobile) {
+      await page.getByRole('tab', { name: 'Console' }).click();
+    }
+
     await expect(
       page.getByRole('region', {
         name: translations.learn['editor-tabs'].console
@@ -77,8 +66,8 @@ test.describe('For classic challenges', () => {
     await expect(page).toHaveTitle(
       'Basic HTML and HTML5: Say Hello to HTML Elements |' + ' freeCodeCamp.org'
     );
-
-    await clearEditor({ browserName, page });
+    await focusEditor({ page, isMobile });
+    await clearEditor({ browserName, page, isMobile });
     await insertTextInCodeEditor({
       page,
       isMobile,
@@ -86,6 +75,7 @@ test.describe('For classic challenges', () => {
     });
     await runChallengeTest(page, isMobile);
     await closeButton.click();
+
     await expect(
       page.getByRole('region', {
         name: translations.learn['editor-tabs'].console
@@ -95,11 +85,11 @@ test.describe('For classic challenges', () => {
 
   test('shows test output when the tests are triggered by the keyboard', async ({
     page,
-    isMobile,
-    browserName
+    isMobile
   }) => {
+    test.skip(isMobile);
     const closeButton = page.getByRole('button', { name: 'Close' });
-    await clearEditor({ browserName, page });
+
     await insertTextInCodeEditor({
       page,
       isMobile,
@@ -107,6 +97,7 @@ test.describe('For classic challenges', () => {
     });
     await page.keyboard.press('Control+Enter');
     await closeButton.click();
+
     await expect(
       page.getByRole('region', {
         name: translations.learn['editor-tabs'].console
@@ -126,9 +117,11 @@ test.describe('Challenge Output Component Tests', () => {
     if (isMobile) {
       await page.getByRole('tab', { name: 'Console' }).click();
     }
+
     const outputConsole = page.getByRole('region', {
       name: translations.learn['editor-tabs'].console
     });
+
     await expect(outputConsole).toBeVisible();
     await expect(outputConsole).toHaveText(outputTexts.default);
   });
@@ -137,7 +130,13 @@ test.describe('Challenge Output Component Tests', () => {
     page,
     isMobile
   }) => {
+    await focusEditor({ page, isMobile });
     await insertTextInCodeEditor({ page, isMobile, text: 'var' });
+
+    if (isMobile) {
+      await page.getByRole('tab', { name: 'Console' }).click();
+    }
+
     await expect(
       page.getByRole('region', {
         name: translations.learn['editor-tabs'].console
@@ -151,7 +150,13 @@ test.describe('Challenge Output Component Tests', () => {
   }) => {
     const referenceErrorRegex =
       /ReferenceError: (myName is not defined|Can't find variable: myName)/;
+    await focusEditor({ page, isMobile });
     await insertTextInCodeEditor({ page, isMobile, text: 'myName' });
+
+    if (isMobile) {
+      await page.getByRole('tab', { name: 'Console' }).click();
+    }
+
     await expect(
       page.getByRole('region', {
         name: translations.learn['editor-tabs'].console
@@ -164,6 +169,7 @@ test.describe('Challenge Output Component Tests', () => {
     isMobile
   }) => {
     await runChallengeTest(page, isMobile);
+
     await expect(
       page.getByRole('region', {
         name: translations.learn['editor-tabs'].console
@@ -176,10 +182,11 @@ test.describe('Challenge Output Component Tests', () => {
     isMobile
   }) => {
     const closeButton = page.getByRole('button', { name: 'Close' });
-
+    await focusEditor({ page, isMobile });
     await insertTextInCodeEditor({ page, isMobile, text: 'var myName;' });
     await runChallengeTest(page, isMobile);
     await closeButton.click();
+
     await expect(
       page.getByRole('region', {
         name: translations.learn['editor-tabs'].console
@@ -190,11 +197,17 @@ test.describe('Challenge Output Component Tests', () => {
 
 test.describe('Jquery challenges', () => {
   test('Jquery challenge should render with default output', async ({
-    page
+    page,
+    isMobile
   }) => {
     await page.goto(
       '/learn/front-end-development-libraries/jquery/target-html-elements-with-selectors-using-jquery'
     );
+
+    if (isMobile) {
+      await page.getByRole('tab', { name: 'Console' }).click();
+    }
+
     await expect(
       page.getByRole('region', {
         name: translations.learn['editor-tabs'].console
@@ -218,17 +231,26 @@ test.describe('Custom output for Set and Map', () => {
     await page.goto(
       '/learn/javascript-algorithms-and-data-structures/basic-javascript/comment-your-javascript-code'
     );
+
+    await focusEditor({ page, isMobile });
+
     await insertTextInCodeEditor({
       page,
       isMobile,
       text: 'const set = new Set(); set.add(1); set.add("set"); set.add(10); console.log(set);'
     });
+
+    if (isMobile) {
+      await page.getByRole('tab', { name: 'Console' }).click();
+    }
+
     await expect(
       page.getByRole('region', {
         name: translations.learn['editor-tabs'].console
       })
     ).toContainText('Set(3) {1, set, 10}');
 
+    await focusEditor({ page, isMobile });
     await clearEditor({ browserName, page });
 
     await insertTextInCodeEditor({
@@ -237,6 +259,10 @@ test.describe('Custom output for Set and Map', () => {
       text: 'const map = new Map(); map.set(1, "one"); map.set("two", 2); console.log(map);'
     });
 
+    if (isMobile) {
+      await page.getByRole('tab', { name: 'Console' }).click();
+    }
+
     await expect(
       page.getByRole('region', {
         name: translations.learn['editor-tabs'].console

e2e/progress-bar.spec.ts

@@ -48,7 +48,10 @@ test.describe('Progress bar component', () => {
     await page.keyboard.insertText('var myName;');
 
     await page
-      .getByRole('button', { name: 'Run the Tests (Ctrl + Enter)' })
+      .getByRole('button', {
+        name: 'Run',
+        exact: false
+      })
       .click();
 
     await expect(page.locator('.completion-block-meta')).toContainText(

e2e/projects.spec.ts

@@ -239,8 +239,10 @@ test.describe('Completion modal should be shown after submitting a project', ()
 
   test('Ctrl + enter triggers the completion modal on multifile projects', async ({
     page,
-    context
+    context,
+    isMobile
   }) => {
+    test.skip(isMobile);
     await context.grantPermissions(['clipboard-read', 'clipboard-write']);
 
     const tributeContent = [

e2e/report-user.spec.ts

@@ -15,7 +15,8 @@ test.beforeEach(async () => {
 });
 
 test('should be possible to report a user from their profile page', async ({
-  page
+  page,
+  isMobile
 }) => {
   await page.goto('/twaha');
 
@@ -30,7 +31,11 @@ test('should be possible to report a user from their profile page', async ({
     page.getByText("Do you want to report twaha's portfolio for abuse?")
   ).toBeVisible();
 
-  await page.getByRole('textbox').nth(1).fill('Some details');
+  // On mobile, the texarea is the first element due to the searchbox not being present
+  await page
+    .getByRole('textbox')
+    .nth(isMobile ? 0 : 1)
+    .fill('Some details');
   await page.getByRole('button', { name: 'Submit the report' }).click();
   await expect(page).toHaveURL('/learn');
 

e2e/sass.spec.ts

@@ -7,7 +7,11 @@ test.describe('Sass Challenge', () => {
     );
   });
 
-  test('should render the sass preview', async ({ page }) => {
+  test('should render the sass preview', async ({ page, isMobile }) => {
+    if (isMobile) {
+      await page.getByRole('tab', { name: 'Preview' }).click();
+    }
+
     const frame = page.frameLocator('.challenge-preview iframe');
     expect(frame).not.toBeNull();
 

e2e/tool-panel.spec.ts

@@ -13,7 +13,8 @@ test.describe('Tool Panel', () => {
   }) => {
     await page
       .getByRole('button', {
-        name: 'Run the Test'
+        name: 'Run',
+        exact: false
       })
       .click();
 

e2e/utils/editor.ts

@@ -26,13 +26,15 @@ export const focusEditor = async ({
 
 export async function clearEditor({
   page,
-  browserName
+  browserName,
+  isMobile = false
 }: {
   page: Page;
   browserName: string;
+  isMobile?: boolean;
 }) {
   // TODO: replace with ControlOrMeta when it's supported
-  if (browserName === 'webkit') {
+  if (browserName === 'webkit' && !isMobile) {
     await page.keyboard.press('Meta+a');
   } else {
     await page.keyboard.press('Control+a');