feat: enhance E2E testing setup with Cloudflare Access support and improved authentication handling

- Added support for Cloudflare Access headers in Playwright configuration and teardown. - Updated global setup for E2E tests to validate authentication credentials and handle login more robustly. - Enhanced README with authentication configuration details and supported methods. - Updated Playwright reporter configuration to include JSON output for test results.
2025-12-19 17:27:16 -05:00 · 2025-12-16 14:15:09 +08:00
parent 3863894072
commit 47724ec764
5 changed files with 146 additions and 41 deletions
--- a/web/e2e/README.md
+++ b/web/e2e/README.md
@@ -28,6 +28,28 @@ E2E_SKIP_WEB_SERVER=true

 # API URL (optional, defaults to http://localhost:5001/console/api)
 NEXT_PUBLIC_API_PREFIX=http://localhost:5001/console/api
+
+# Authentication Configuration
+# Test user credentials
+NEXT_PUBLIC_E2E_USER_EMAIL=test@example.com
+NEXT_PUBLIC_E2E_USER_PASSWORD=your-password
+```
+
+### Authentication Methods
+
+Dify supports multiple login methods, but not all are suitable for E2E testing:
+
+| Method | E2E Support | Configuration |
+|--------|-------------|---------------|
+| **Email + Password** | ✅ Recommended | Set `NEXT_PUBLIC_E2E_USER_EMAIL` and `NEXT_PUBLIC_E2E_USER_PASSWORD` |
+
+#### Email + Password (Default)
+
+The most reliable method for E2E testing. Simply set the credentials:
+
+```env
+NEXT_PUBLIC_E2E_USER_EMAIL=test@example.com
+NEXT_PUBLIC_E2E_USER_PASSWORD=your-password
 ```

 ### 3. Run Tests
--- a/web/e2e/global.setup.ts
+++ b/web/e2e/global.setup.ts
@@ -4,75 +4,124 @@ import path from 'node:path'

 const authFile = path.join(__dirname, '.auth/user.json')

+/**
+ * Supported authentication methods for E2E tests
+ * - password: Email + Password login (default, recommended)
+ *
+ * OAuth (GitHub/Google) and SSO are not supported in E2E tests
+ * as they require third-party authentication which cannot be reliably automated.
+ */
+
 /**
 * Global setup for E2E tests
 *
 * This runs before all tests and handles authentication.
 * The authenticated state is saved and reused across all tests.
 *
- * Based on signin implementation:
- * - web/app/signin/components/mail-and-password-auth.tsx
+ * Environment variables:
+ * - NEXT_PUBLIC_E2E_USER_EMAIL: Test user email (required)
+ * - NEXT_PUBLIC_E2E_USER_PASSWORD: Test user password (required for 'password' method)
 */
 setup('authenticate', async ({ page }) => {
-  // Get test user credentials from environment
  const email = process.env.NEXT_PUBLIC_E2E_USER_EMAIL
  const password = process.env.NEXT_PUBLIC_E2E_USER_PASSWORD

-  if (!email || !password) {
+  // Validate required credentials based on auth method
+  if (!email) {
    console.warn(
-      '⚠️  NEXT_PUBLIC_E2E_USER_EMAIL or NEXT_PUBLIC_E2E_USER_PASSWORD not set.',
+      '⚠️  NEXT_PUBLIC_E2E_USER_EMAIL not set.',
      'Creating empty auth state. Tests requiring auth will fail.',
    )
-    // Create empty auth state directory if it doesn't exist
-    const authDir = path.dirname(authFile)
-    if (!fs.existsSync(authDir))
-      fs.mkdirSync(authDir, { recursive: true })
+    await saveEmptyAuthState(page)
+    return
+  }

-    // Save empty state
-    await page.context().storageState({ path: authFile })
+  if (!password) {
+    console.warn(
+      '⚠️  NEXT_PUBLIC_E2E_USER_PASSWORD not set for password auth method.',
+      'Creating empty auth state. Tests requiring auth will fail.',
+    )
+    await saveEmptyAuthState(page)
    return
  }

  // Navigate to login page
  await page.goto('/signin')
-
-  // Wait for the page to load
  await page.waitForLoadState('networkidle')

-  // Fill in login form using actual Dify selectors
+  // Execute login
+  await loginWithPassword(page, email, password!)
+
+  // Wait for successful redirect to /apps
+  await expect(page).toHaveURL(/\/apps/, { timeout: 30000 })
+
+  // Save authenticated state
+  await page.context().storageState({ path: authFile })
+  console.log('✅ Authentication successful, state saved.')
+})
+
+/**
+ * Save empty auth state when credentials are not available
+ */
+async function saveEmptyAuthState(page: import('@playwright/test').Page): Promise<void> {
+  const authDir = path.dirname(authFile)
+  if (!fs.existsSync(authDir))
+    fs.mkdirSync(authDir, { recursive: true })
+  await page.context().storageState({ path: authFile })
+}
+
+/**
+ * Login using email and password
+ * Based on: web/app/signin/components/mail-and-password-auth.tsx
+ */
+async function loginWithPassword(
+  page: import('@playwright/test').Page,
+  email: string,
+  password: string,
+): Promise<void> {
+  console.log('📧 Logging in with email and password...')
+
+  // Fill in login form
  // Email input has id="email"
  await page.locator('#email').fill(email)
  // Password input has id="password"
  await page.locator('#password').fill(password)

  // Wait for button to be enabled (form validation passes)
-  const signInButton = page.getByRole('button', { name: 'Sign in' })
+  const signInButton = page.getByRole('button', { name: /sign in/i })
  await expect(signInButton).toBeEnabled({ timeout: 5000 })

-  // Click login button and wait for the login API response
-  const [response] = await Promise.all([
-    page.waitForResponse(resp =>
-      resp.url().includes('/login') && resp.request().method() === 'POST',
-    ),
-    signInButton.click(),
-  ])
+  // Click login button and wait for navigation or API response
+  // The app uses ky library which follows redirects automatically
+  // Some environments may have WAF/CDN that adds extra redirects
+  // So we use a more flexible approach: wait for either URL change or API response
+  const responsePromise = page.waitForResponse(
+    resp => resp.url().includes('login') && resp.request().method() === 'POST',
+    { timeout: 15000 },
+  ).catch(() => null) // Don't fail if we can't catch the response

-  // Check if login request was successful
-  const status = response.status()
-  if (status === 200) {
-    // Redirect response means login successful (server-side redirect)
-    console.log('✅ Login successful (redirect response)')
-    // Wait for navigation to complete (redirect to /apps)
-    // See: mail-and-password-auth.tsx line 71 - router.replace(redirectUrl || '/apps')
-    await expect(page).toHaveURL(/\/apps/, { timeout: 30000 })
+  await signInButton.click()
+
+  // Try to get the response, but don't fail if we can't
+  const response = await responsePromise
+  if (response) {
+    const status = response.status()
+    console.log(`📡 Login API response status: ${status}`)
+    // 200 = success, 302 = redirect (some WAF/CDN setups)
+    if (status !== 200 && status !== 302) {
+      // Try to get error details
+      try {
+        const body = await response.json()
+        console.error('❌ Login failed:', body)
+      }
+      catch {
+        console.error(`❌ Login failed with status ${status}`)
+      }
+    }
  }
  else {
-    // Other status codes indicate failure
-    throw new Error(`Login request failed with status ${status}`)
+    console.log('⚠️ Could not capture login API response, will verify via URL redirect')
  }

-  // Save authenticated state
-  await page.context().storageState({ path: authFile })
-
-  console.log('✅ Authentication successful, state saved.')
-})
+  console.log('✅ Password login request sent')
+}
--- a/web/e2e/global.teardown.ts
+++ b/web/e2e/global.teardown.ts
@@ -20,6 +20,17 @@ import { request, test as teardown } from '@playwright/test'
 // Ensure baseURL ends with '/' for proper path concatenation
 const API_BASE_URL = (process.env.NEXT_PUBLIC_API_PREFIX || 'http://localhost:5001/console/api').replace(/\/?$/, '/')

+// Cloudflare Access headers (for protected environments).
+// Prefer environment variables to avoid hardcoding secrets in repo.
+const CF_ACCESS_CLIENT_ID = process.env.CF_ACCESS_CLIENT_ID
+const CF_ACCESS_CLIENT_SECRET = process.env.CF_ACCESS_CLIENT_SECRET
+
+const cfAccessHeaders: Record<string, string> = {}
+if (CF_ACCESS_CLIENT_ID && CF_ACCESS_CLIENT_SECRET) {
+  cfAccessHeaders['CF-Access-Client-Id'] = CF_ACCESS_CLIENT_ID
+  cfAccessHeaders['CF-Access-Client-Secret'] = CF_ACCESS_CLIENT_SECRET
+}
+
 // Test data prefixes - used to identify test-created data
 // Should match the prefix used in generateTestId()
 const TEST_DATA_PREFIXES = ['e2e-', 'test-']
@@ -87,7 +98,10 @@ teardown('cleanup test data', async () => {
      return
    }
    // Extract CSRF token from cookies for API requests
-    const csrfCookie = authState.cookies.find((c: { name: string }) => c.name === 'csrf_token')
+    // Cookie name may be 'csrf_token' or '__Host-csrf_token' depending on environment
+    const csrfCookie = authState.cookies.find((c: { name: string }) =>
+      c.name === 'csrf_token' || c.name === '__Host-csrf_token',
+    )
    csrfToken = csrfCookie?.value || ''
  }
  catch {
@@ -103,6 +117,7 @@ teardown('cleanup test data', async () => {
      storageState: authPath,
      extraHTTPHeaders: {
        'X-CSRF-Token': csrfToken,
+        ...cfAccessHeaders,
      },
    })

--- a/web/playwright.config.ts
+++ b/web/playwright.config.ts
@@ -25,6 +25,17 @@ const BASE_URL = process.env.E2E_BASE_URL || 'http://localhost:3000'
 // - CI/CD with deployed env: true (use existing server)
 const SKIP_WEB_SERVER = process.env.E2E_SKIP_WEB_SERVER === 'true'

+// Cloudflare Access headers (for protected environments).
+// Prefer environment variables to avoid hardcoding secrets in repo.
+const CF_ACCESS_CLIENT_ID = process.env.CF_ACCESS_CLIENT_ID
+const CF_ACCESS_CLIENT_SECRET = process.env.CF_ACCESS_CLIENT_SECRET
+
+const cfAccessHeaders: Record<string, string> = {}
+if (CF_ACCESS_CLIENT_ID && CF_ACCESS_CLIENT_SECRET) {
+  cfAccessHeaders['CF-Access-Client-Id'] = CF_ACCESS_CLIENT_ID
+  cfAccessHeaders['CF-Access-Client-Secret'] = CF_ACCESS_CLIENT_SECRET
+}
+
 export default defineConfig({
  // Directory containing test files
  testDir: './e2e/tests',
@@ -43,7 +54,7 @@ export default defineConfig({

  // Reporter to use
  reporter: process.env.CI
-    ? [['html', { open: 'never' }], ['github']]
+    ? [['html', { open: 'never', outputFolder: 'playwright-report' }], ['github'], ['json', { outputFile: 'e2e/test-results/results.json' }]]
    : [['html', { open: 'on-failure' }], ['list']],

  // Shared settings for all the projects below
@@ -51,6 +62,13 @@ export default defineConfig({
    // Base URL for all page.goto() calls
    baseURL: BASE_URL,

+    // Extra headers for all requests made by the browser context.
+    extraHTTPHeaders: cfAccessHeaders,
+
+    // Bypass Content Security Policy to allow test automation
+    // This is needed when testing against environments with strict CSP headers
+    bypassCSP: true,
+
    // Collect trace when retrying the failed test
    trace: 'on-first-retry',

--- a/web/pnpm-lock.yaml
+++ b/web/pnpm-lock.yaml
@@ -386,6 +386,9 @@ importers:
      '@next/mdx':
        specifier: 15.5.9
        version: 15.5.9(@mdx-js/loader@3.1.1(webpack@5.103.0(esbuild@0.25.0)(uglify-js@3.19.3)))(@mdx-js/react@3.1.1(@types/react@19.2.7)(react@19.2.3))
+      '@playwright/test':
+        specifier: ^1.56.1
+        version: 1.57.0
      '@rgrove/parse-xml':
        specifier: ^4.2.0
        version: 4.2.0
@@ -6899,7 +6902,6 @@ packages:
  next@15.5.9:
    resolution: {integrity: sha512-agNLK89seZEtC5zUHwtut0+tNrc0Xw4FT/Dg+B/VLEo9pAcS9rtTKpek3V6kVcVwsB2YlqMaHdfZL4eLEVYuCg==}
    engines: {node: ^18.18.0 || ^19.8.0 || >= 20.0.0}
-    deprecated: This version has a security vulnerability. Please upgrade to a patched version. See https://nextjs.org/blog/security-update-2025-12-11 for more details.
    hasBin: true
    peerDependencies:
      '@opentelemetry/api': ^1.1.0
@@ -11463,7 +11465,6 @@ snapshots:
  '@playwright/test@1.57.0':
    dependencies:
      playwright: 1.57.0
-    optional: true

  '@pmmmwh/react-refresh-webpack-plugin@0.5.17(react-refresh@0.14.2)(type-fest@4.2.0)(webpack-hot-middleware@2.26.1)(webpack@5.103.0(esbuild@0.25.0)(uglify-js@3.19.3))':
    dependencies: