feat: enhance E2E testing setup with Cloudflare Access support and improved authentication handling

- Added support for Cloudflare Access headers in Playwright configuration and teardown.
- Updated global setup for E2E tests to validate authentication credentials and handle login more robustly.
- Enhanced README with authentication configuration details and supported methods.
- Updated Playwright reporter configuration to include JSON output for test results.

web/e2e/README.md

@@ -28,6 +28,28 @@ E2E_SKIP_WEB_SERVER=true
 
 # API URL (optional, defaults to http://localhost:5001/console/api)
 NEXT_PUBLIC_API_PREFIX=http://localhost:5001/console/api
+
+# Authentication Configuration
+# Test user credentials
+NEXT_PUBLIC_E2E_USER_EMAIL=test@example.com
+NEXT_PUBLIC_E2E_USER_PASSWORD=your-password
+```
+
+### Authentication Methods
+
+Dify supports multiple login methods, but not all are suitable for E2E testing:
+
+| Method | E2E Support | Configuration |
+|--------|-------------|---------------|
+| **Email + Password** | ✅ Recommended | Set `NEXT_PUBLIC_E2E_USER_EMAIL` and `NEXT_PUBLIC_E2E_USER_PASSWORD` |
+
+#### Email + Password (Default)
+
+The most reliable method for E2E testing. Simply set the credentials:
+
+```env
+NEXT_PUBLIC_E2E_USER_EMAIL=test@example.com
+NEXT_PUBLIC_E2E_USER_PASSWORD=your-password
 ```
 
 ### 3. Run Tests

web/e2e/global.setup.ts

@@ -4,75 +4,124 @@ import path from 'node:path'
 
 const authFile = path.join(__dirname, '.auth/user.json')
 
+/**
+ * Supported authentication methods for E2E tests
+ * - password: Email + Password login (default, recommended)
+ *
+ * OAuth (GitHub/Google) and SSO are not supported in E2E tests
+ * as they require third-party authentication which cannot be reliably automated.
+ */
+
 /**
  * Global setup for E2E tests
  *
  * This runs before all tests and handles authentication.
  * The authenticated state is saved and reused across all tests.
  *
- * Based on signin implementation:
+ * Environment variables:
- * - web/app/signin/components/mail-and-password-auth.tsx
+ * - NEXT_PUBLIC_E2E_USER_EMAIL: Test user email (required)
+ * - NEXT_PUBLIC_E2E_USER_PASSWORD: Test user password (required for 'password' method)
  */
 setup('authenticate', async ({ page }) => {
-  // Get test user credentials from environment
   const email = process.env.NEXT_PUBLIC_E2E_USER_EMAIL
   const password = process.env.NEXT_PUBLIC_E2E_USER_PASSWORD
 
-  if (!email || !password) {
+  // Validate required credentials based on auth method
+  if (!email) {
     console.warn(
-      '⚠️  NEXT_PUBLIC_E2E_USER_EMAIL or NEXT_PUBLIC_E2E_USER_PASSWORD not set.',
+      '⚠️  NEXT_PUBLIC_E2E_USER_EMAIL not set.',
       'Creating empty auth state. Tests requiring auth will fail.',
     )
-    // Create empty auth state directory if it doesn't exist
+    await saveEmptyAuthState(page)
-    const authDir = path.dirname(authFile)
+    return
-    if (!fs.existsSync(authDir))
+  }
-      fs.mkdirSync(authDir, { recursive: true })
 
-    // Save empty state
+  if (!password) {
-    await page.context().storageState({ path: authFile })
+    console.warn(
+      '⚠️  NEXT_PUBLIC_E2E_USER_PASSWORD not set for password auth method.',
+      'Creating empty auth state. Tests requiring auth will fail.',
+    )
+    await saveEmptyAuthState(page)
     return
   }
 
   // Navigate to login page
   await page.goto('/signin')
-
-  // Wait for the page to load
   await page.waitForLoadState('networkidle')
 
-  // Fill in login form using actual Dify selectors
+  // Execute login
+  await loginWithPassword(page, email, password!)
+
+  // Wait for successful redirect to /apps
+  await expect(page).toHaveURL(/\/apps/, { timeout: 30000 })
+
+  // Save authenticated state
+  await page.context().storageState({ path: authFile })
+  console.log('✅ Authentication successful, state saved.')
+})
+
+/**
+ * Save empty auth state when credentials are not available
+ */
+async function saveEmptyAuthState(page: import('@playwright/test').Page): Promise<void> {
+  const authDir = path.dirname(authFile)
+  if (!fs.existsSync(authDir))
+    fs.mkdirSync(authDir, { recursive: true })
+  await page.context().storageState({ path: authFile })
+}
+
+/**
+ * Login using email and password
+ * Based on: web/app/signin/components/mail-and-password-auth.tsx
+ */
+async function loginWithPassword(
+  page: import('@playwright/test').Page,
+  email: string,
+  password: string,
+): Promise<void> {
+  console.log('📧 Logging in with email and password...')
+
+  // Fill in login form
   // Email input has id="email"
   await page.locator('#email').fill(email)
   // Password input has id="password"
   await page.locator('#password').fill(password)
 
   // Wait for button to be enabled (form validation passes)
-  const signInButton = page.getByRole('button', { name: 'Sign in' })
+  const signInButton = page.getByRole('button', { name: /sign in/i })
   await expect(signInButton).toBeEnabled({ timeout: 5000 })
 
-  // Click login button and wait for the login API response
+  // Click login button and wait for navigation or API response
-  const [response] = await Promise.all([
+  // The app uses ky library which follows redirects automatically
-    page.waitForResponse(resp =>
+  // Some environments may have WAF/CDN that adds extra redirects
-      resp.url().includes('/login') && resp.request().method() === 'POST',
+  // So we use a more flexible approach: wait for either URL change or API response
-    ),
+  const responsePromise = page.waitForResponse(
-    signInButton.click(),
+    resp => resp.url().includes('login') && resp.request().method() === 'POST',
-  ])
+    { timeout: 15000 },
+  ).catch(() => null) // Don't fail if we can't catch the response
 
-  // Check if login request was successful
+  await signInButton.click()
-  const status = response.status()
+
-  if (status === 200) {
+  // Try to get the response, but don't fail if we can't
-    // Redirect response means login successful (server-side redirect)
+  const response = await responsePromise
-    console.log('✅ Login successful (redirect response)')
+  if (response) {
-    // Wait for navigation to complete (redirect to /apps)
+    const status = response.status()
-    // See: mail-and-password-auth.tsx line 71 - router.replace(redirectUrl || '/apps')
+    console.log(`📡 Login API response status: ${status}`)
-    await expect(page).toHaveURL(/\/apps/, { timeout: 30000 })
+    // 200 = success, 302 = redirect (some WAF/CDN setups)
+    if (status !== 200 && status !== 302) {
+      // Try to get error details
+      try {
+        const body = await response.json()
+        console.error('❌ Login failed:', body)
+      }
+      catch {
+        console.error(`❌ Login failed with status ${status}`)
+      }
+    }
   }
   else {
-    // Other status codes indicate failure
+    console.log('⚠️ Could not capture login API response, will verify via URL redirect')
-    throw new Error(`Login request failed with status ${status}`)
   }
 
-  // Save authenticated state
+  console.log('✅ Password login request sent')
-  await page.context().storageState({ path: authFile })
+}
-
-  console.log('✅ Authentication successful, state saved.')
-})

web/e2e/global.teardown.ts

@@ -20,6 +20,17 @@ import { request, test as teardown } from '@playwright/test'
 // Ensure baseURL ends with '/' for proper path concatenation
 const API_BASE_URL = (process.env.NEXT_PUBLIC_API_PREFIX || 'http://localhost:5001/console/api').replace(/\/?$/, '/')
 
+// Cloudflare Access headers (for protected environments).
+// Prefer environment variables to avoid hardcoding secrets in repo.
+const CF_ACCESS_CLIENT_ID = process.env.CF_ACCESS_CLIENT_ID
+const CF_ACCESS_CLIENT_SECRET = process.env.CF_ACCESS_CLIENT_SECRET
+
+const cfAccessHeaders: Record<string, string> = {}
+if (CF_ACCESS_CLIENT_ID && CF_ACCESS_CLIENT_SECRET) {
+  cfAccessHeaders['CF-Access-Client-Id'] = CF_ACCESS_CLIENT_ID
+  cfAccessHeaders['CF-Access-Client-Secret'] = CF_ACCESS_CLIENT_SECRET
+}
+
 // Test data prefixes - used to identify test-created data
 // Should match the prefix used in generateTestId()
 const TEST_DATA_PREFIXES = ['e2e-', 'test-']
@@ -87,7 +98,10 @@ teardown('cleanup test data', async () => {
       return
     }
     // Extract CSRF token from cookies for API requests
-    const csrfCookie = authState.cookies.find((c: { name: string }) => c.name === 'csrf_token')
+    // Cookie name may be 'csrf_token' or '__Host-csrf_token' depending on environment
+    const csrfCookie = authState.cookies.find((c: { name: string }) =>
+      c.name === 'csrf_token' || c.name === '__Host-csrf_token',
+    )
     csrfToken = csrfCookie?.value || ''
   }
   catch {
@@ -103,6 +117,7 @@ teardown('cleanup test data', async () => {
       storageState: authPath,
       extraHTTPHeaders: {
         'X-CSRF-Token': csrfToken,
+        ...cfAccessHeaders,
       },
     })
 

web/playwright.config.ts

@@ -25,6 +25,17 @@ const BASE_URL = process.env.E2E_BASE_URL || 'http://localhost:3000'
 // - CI/CD with deployed env: true (use existing server)
 const SKIP_WEB_SERVER = process.env.E2E_SKIP_WEB_SERVER === 'true'
 
+// Cloudflare Access headers (for protected environments).
+// Prefer environment variables to avoid hardcoding secrets in repo.
+const CF_ACCESS_CLIENT_ID = process.env.CF_ACCESS_CLIENT_ID
+const CF_ACCESS_CLIENT_SECRET = process.env.CF_ACCESS_CLIENT_SECRET
+
+const cfAccessHeaders: Record<string, string> = {}
+if (CF_ACCESS_CLIENT_ID && CF_ACCESS_CLIENT_SECRET) {
+  cfAccessHeaders['CF-Access-Client-Id'] = CF_ACCESS_CLIENT_ID
+  cfAccessHeaders['CF-Access-Client-Secret'] = CF_ACCESS_CLIENT_SECRET
+}
+
 export default defineConfig({
   // Directory containing test files
   testDir: './e2e/tests',
@@ -43,7 +54,7 @@ export default defineConfig({
 
   // Reporter to use
   reporter: process.env.CI
-    ? [['html', { open: 'never' }], ['github']]
+    ? [['html', { open: 'never', outputFolder: 'playwright-report' }], ['github'], ['json', { outputFile: 'e2e/test-results/results.json' }]]
     : [['html', { open: 'on-failure' }], ['list']],
 
   // Shared settings for all the projects below
@@ -51,6 +62,13 @@ export default defineConfig({
     // Base URL for all page.goto() calls
     baseURL: BASE_URL,
 
+    // Extra headers for all requests made by the browser context.
+    extraHTTPHeaders: cfAccessHeaders,
+
+    // Bypass Content Security Policy to allow test automation
+    // This is needed when testing against environments with strict CSP headers
+    bypassCSP: true,
+
     // Collect trace when retrying the failed test
     trace: 'on-first-retry',
 

web/pnpm-lock.yaml

@@ -386,6 +386,9 @@ importers:
       '@next/mdx':
         specifier: 15.5.9
         version: 15.5.9(@mdx-js/loader@3.1.1(webpack@5.103.0(esbuild@0.25.0)(uglify-js@3.19.3)))(@mdx-js/react@3.1.1(@types/react@19.2.7)(react@19.2.3))
+      '@playwright/test':
+        specifier: ^1.56.1
+        version: 1.57.0
       '@rgrove/parse-xml':
         specifier: ^4.2.0
         version: 4.2.0
@@ -6899,7 +6902,6 @@ packages:
   next@15.5.9:
     resolution: {integrity: sha512-agNLK89seZEtC5zUHwtut0+tNrc0Xw4FT/Dg+B/VLEo9pAcS9rtTKpek3V6kVcVwsB2YlqMaHdfZL4eLEVYuCg==}
     engines: {node: ^18.18.0 || ^19.8.0 || >= 20.0.0}
-    deprecated: This version has a security vulnerability. Please upgrade to a patched version. See https://nextjs.org/blog/security-update-2025-12-11 for more details.
     hasBin: true
     peerDependencies:
       '@opentelemetry/api': ^1.1.0
@@ -11463,7 +11465,6 @@ snapshots:
   '@playwright/test@1.57.0':
     dependencies:
       playwright: 1.57.0
-    optional: true
 
   '@pmmmwh/react-refresh-webpack-plugin@0.5.17(react-refresh@0.14.2)(type-fest@4.2.0)(webpack-hot-middleware@2.26.1)(webpack@5.103.0(esbuild@0.25.0)(uglify-js@3.19.3))':
     dependencies: