From b1b371c72a7adffa980ee16158bb979fe2a59306 Mon Sep 17 00:00:00 2001 From: Ahmad Abdolsaheb Date: Tue, 17 Sep 2024 04:39:55 +0300 Subject: [PATCH] feat: AB test initial donation modal interval (#56078) --- .../config/growthbook-features-default.json | 35 +++- .../growth-book-redux-connector.tsx | 36 ++-- client/src/redux/action-types.js | 2 +- client/src/redux/actions.js | 5 +- client/src/redux/index.js | 6 +- client/src/redux/selectors.js | 20 ++- client/src/utils/random-between.ts | 3 + e2e/donation-modal.spec.ts | 169 +++++++----------- e2e/landing.spec.ts | 19 +- e2e/utils/add-growthbook-cookie.ts | 19 ++ 10 files changed, 161 insertions(+), 153 deletions(-) create mode 100644 client/src/utils/random-between.ts create mode 100644 e2e/utils/add-growthbook-cookie.ts diff --git a/client/config/growthbook-features-default.json b/client/config/growthbook-features-default.json index 2be8bb28fb1..f73e3d55181 100644 --- a/client/config/growthbook-features-default.json +++ b/client/config/growthbook-features-default.json @@ -30,6 +30,37 @@ "name": "tests the conversion rate of the new design comparing to the old one" } ] + }, + "show-modal-randomly": { + "defaultValue": false, + "rules": [ + { + "coverage": 1, + "hashAttribute": "id", + "seed": "show-modal-randomly", + "hashVersion": 2, + "variations": [ + false, + true + ], + "weights": [ + 0.5, + 0.5 + ], + "key": "show-modal-randomly", + "meta": [ + { + "key": "0", + "name": "Control" + }, + { + "key": "1", + "name": "Variation 1" + } + ], + "phase": "0", + "name": "stg show modal randomly" + } + ] } -} - +} \ No newline at end of file diff --git a/client/src/components/growth-book/growth-book-redux-connector.tsx b/client/src/components/growth-book/growth-book-redux-connector.tsx index 7c7aabe1c7c..460eb898f87 100644 --- a/client/src/components/growth-book/growth-book-redux-connector.tsx +++ b/client/src/components/growth-book/growth-book-redux-connector.tsx @@ -5,49 +5,49 @@ import { connect } from 'react-redux'; import { createSelector } from 'reselect'; import { isSignedInSelector, - showMultipleProgressModalsSelector, + isRandomCompletionThresholdSelector, userIdSelector, userFetchStateSelector } from '../../redux/selectors'; -import { setShowMultipleProgressModals } from '../../redux/actions'; +import { setIsRandomCompletionThreshold } from '../../redux/actions'; import { UserFetchState } from '../../redux/prop-types'; import callGA from '../../analytics/call-ga'; const mapStateToProps = createSelector( isSignedInSelector, - showMultipleProgressModalsSelector, + isRandomCompletionThresholdSelector, userIdSelector, userFetchStateSelector, ( isSignedIn: boolean, - showMultipleProgressModals: boolean, + isRandomCompletionThreshold: boolean, userId: string, userFetchState: UserFetchState ) => ({ isSignedIn, - showMultipleProgressModals, + isRandomCompletionThreshold, userId, userFetchState }) ); type StateProps = ReturnType; -type DispatchProps = { setShowMultipleProgressModals: (arg: boolean) => void }; +type DispatchProps = { setIsRandomCompletionThreshold: (arg: boolean) => void }; interface GrowthBookReduxConnector extends StateProps, DispatchProps { children: ReactNode; } const mapDispatchToProps = { - setShowMultipleProgressModals + setIsRandomCompletionThreshold }; const GrowthBookReduxConnector = ({ children, isSignedIn, - showMultipleProgressModals, + isRandomCompletionThreshold, userId, - setShowMultipleProgressModals, + setIsRandomCompletionThreshold, userFetchState }: GrowthBookReduxConnector) => { // Send user id to GA @@ -60,23 +60,17 @@ const GrowthBookReduxConnector = ({ } }, [userFetchState, userId, isSignedIn]); - const displayProgressModalMultipleTimes = useFeature( - 'display_progress_modal_multiple_times' - ).on; + const showModalsRandomly = useFeature('show-modal-randomly').on; useFeature('aa-test'); useEffect(() => { - if ( - isSignedIn && - displayProgressModalMultipleTimes && - !showMultipleProgressModals - ) { - setShowMultipleProgressModals(true); + if (isSignedIn && showModalsRandomly && !isRandomCompletionThreshold) { + setIsRandomCompletionThreshold(true); } }, [ isSignedIn, - showMultipleProgressModals, - displayProgressModalMultipleTimes, - setShowMultipleProgressModals + isRandomCompletionThreshold, + showModalsRandomly, + setIsRandomCompletionThreshold ]); return <>{children}; }; diff --git a/client/src/redux/action-types.js b/client/src/redux/action-types.js index ecf2cd649e4..f5022221e3c 100644 --- a/client/src/redux/action-types.js +++ b/client/src/redux/action-types.js @@ -9,7 +9,7 @@ export const actionTypes = createTypes( 'allowBlockDonationRequests', 'setRenderStartTime', 'preventBlockDonationRequests', - 'setShowMultipleProgressModals', + 'setIsRandomCompletionThreshold', 'openDonationModal', 'closeDonationModal', 'openSignoutModal', diff --git a/client/src/redux/actions.js b/client/src/redux/actions.js index 78188582dc5..bc369ee5922 100644 --- a/client/src/redux/actions.js +++ b/client/src/redux/actions.js @@ -17,9 +17,8 @@ export const openDonationModal = createAction(actionTypes.openDonationModal); export const preventBlockDonationRequests = createAction( actionTypes.preventBlockDonationRequests ); - -export const setShowMultipleProgressModals = createAction( - actionTypes.setShowMultipleProgressModals +export const setIsRandomCompletionThreshold = createAction( + actionTypes.setIsRandomCompletionThreshold ); export const updateDonationFormState = createAction( actionTypes.updateDonationFormState diff --git a/client/src/redux/index.js b/client/src/redux/index.js index a5290c55eb7..931c8b6044e 100644 --- a/client/src/redux/index.js +++ b/client/src/redux/index.js @@ -50,7 +50,7 @@ export const defaultDonationFormState = { const initialState = { appUsername: '', - showMultipleProgressModals: false, + isRandomCompletionThreshold: false, recentlyClaimedBlock: null, currentChallengeId: store.get(CURRENT_CHALLENGE_KEY), examInProgress: false, @@ -273,9 +273,9 @@ export const reducer = handleActions( ...state, recentlyClaimedBlock: null }), - [actionTypes.setShowMultipleProgressModals]: (state, { payload }) => ({ + [actionTypes.setIsRandomCompletionThreshold]: (state, { payload }) => ({ ...state, - showMultipleProgressModals: payload + isRandomCompletionThreshold: payload }), [actionTypes.resetUserData]: state => ({ ...state, diff --git a/client/src/redux/selectors.js b/client/src/redux/selectors.js index c8a56981a69..8dde7b31085 100644 --- a/client/src/redux/selectors.js +++ b/client/src/redux/selectors.js @@ -1,4 +1,5 @@ import { Certification } from '../../../shared/config/certification-settings'; +import { randomBetween } from '../utils/random-between'; import { getSessionChallengeData } from '../utils/session-storage'; import { ns as MainApp } from './action-types'; @@ -12,8 +13,8 @@ export const partiallyCompletedChallengesSelector = state => export const currentChallengeIdSelector = state => state[MainApp].currentChallengeId; export const completionCountSelector = state => state[MainApp].completionCount; -export const showMultipleProgressModalsSelector = state => - state[MainApp].showMultipleProgressModals; +export const isRandomCompletionThresholdSelector = state => + state[MainApp].isRandomCompletionThreshold; export const isDonatingSelector = state => userSelector(state).isDonating; export const isOnlineSelector = state => state[MainApp].isOnline; export const isServerOnlineSelector = state => state[MainApp].isServerOnline; @@ -36,6 +37,8 @@ export const shouldRequestDonationSelector = state => { const completedChallengeCount = completedChallengesSelector(state).length; const isDonating = isDonatingSelector(state); const recentlyClaimedBlock = recentlyClaimedBlockSelector(state); + const isRandomCompletionThreshold = + isRandomCompletionThresholdSelector(state); // don't request donation if already donating if (isDonating) return false; @@ -58,9 +61,16 @@ export const shouldRequestDonationSelector = state => { // not before the 11th challenge has mounted) if (completedChallengeCount < 10) return false; - // this will mean we have completed 3 or more challenges this browser session - // and enough challenges overall to not be new - return sessionChallengeData.currentCount >= 3; + /* + Show modal if user has completed 10 challanged in total + and 3 or more in this session. + The isRandomCompletionThreshold flag is used to AB test interval randomness + */ + if (isRandomCompletionThreshold) { + return sessionChallengeData.currentCount >= randomBetween(3, 7); + } else { + return sessionChallengeData.currentCount >= 3; + } }; export const userTokenSelector = state => { diff --git a/client/src/utils/random-between.ts b/client/src/utils/random-between.ts new file mode 100644 index 00000000000..42449314f30 --- /dev/null +++ b/client/src/utils/random-between.ts @@ -0,0 +1,3 @@ +// returns a random number between x and y +export const randomBetween = (x: number, y: number): number => + Math.floor(Math.random() * (y - x + 1)) + x; diff --git a/e2e/donation-modal.spec.ts b/e2e/donation-modal.spec.ts index 4cd48c5ac52..44b2a3fca44 100644 --- a/e2e/donation-modal.spec.ts +++ b/e2e/donation-modal.spec.ts @@ -1,5 +1,6 @@ import { execSync } from 'child_process'; import { test, expect, type Page } from '@playwright/test'; +import { addGrowthbookCookie } from './utils/add-growthbook-cookie'; import { clearEditor, focusEditor } from './utils/editor'; @@ -34,122 +35,74 @@ const completeFrontEndCert = async (page: Page) => { } }; -const completeThreeChallenges = async ({ - page, - browserName, - isMobile -}: { - page: Page; - browserName: string; - isMobile: boolean; -}) => { - await page.goto( - '/learn/javascript-algorithms-and-data-structures/basic-javascript/comment-your-javascript-code' - ); - - const challenges = [ - { - url: '/learn/javascript-algorithms-and-data-structures/basic-javascript/comment-your-javascript-code', - solution: `// some comment\n/* some comment */` - }, - { - url: '/learn/javascript-algorithms-and-data-structures/basic-javascript/declare-javascript-variables', - solution: 'var myName;' - }, - { - url: '/learn/javascript-algorithms-and-data-structures/basic-javascript/storing-values-with-the-assignment-operator', - solution: `// Setup\nvar a;\n\n// Only change code below this line\na = 7;` - } - ]; - - for (const challenge of challenges) { - await page.waitForURL(challenge.url); - - await focusEditor({ page, isMobile }); - await clearEditor({ page, browserName }); - - await page.evaluate( - async contents => await navigator.clipboard.writeText(contents), - challenge.solution - ); - await page.keyboard.press('ControlOrMeta+V'); - - await page.getByRole('button', { name: 'Run' }).click(); - await expect(page.getByRole('dialog')).toBeVisible(); // completion dialog - await page.getByRole('button', { name: 'Submit' }).click(); +const challenges = [ + { + url: '/learn/javascript-algorithms-and-data-structures/basic-javascript/comment-your-javascript-code', + solution: `// some comment\n/* some comment */` + }, + { + url: '/learn/javascript-algorithms-and-data-structures/basic-javascript/declare-javascript-variables', + solution: 'var myName;' + }, + { + url: '/learn/javascript-algorithms-and-data-structures/basic-javascript/storing-values-with-the-assignment-operator', + solution: `// Setup\nvar a;\n\n// Only change code below this line\na = 7;` + }, + { + url: '/learn/javascript-algorithms-and-data-structures/basic-javascript/assigning-the-value-of-one-variable-to-another', + solution: `// Setup\nvar a;\na = 7;\nvar b;\n\n// Only change code below this line\nb = a;` + }, + { + url: '/learn/javascript-algorithms-and-data-structures/basic-javascript/initializing-variables-with-the-assignment-operator', + solution: 'var a = 9;' + }, + { + url: '/learn/javascript-algorithms-and-data-structures/basic-javascript/declare-string-variables', + solution: `var myFirstName = 'foo';\nvar myLastName = 'bar';` + }, + { + url: '/learn/javascript-algorithms-and-data-structures/basic-javascript/understanding-uninitialized-variables', + solution: `// Only change code below this line\nvar a = 5;\nvar b = 10;\nvar c = 'I am a';\n// Only change code above this line\n\na = a + 1;\nb = b + 5;\nc = c + " String!";` + }, + { + url: '/learn/javascript-algorithms-and-data-structures/basic-javascript/understanding-case-sensitivity-in-variables', + solution: `// Variable declarations\nvar studlyCapVar;\nvar properCamelCase;\nvar titleCaseOver;\n\n// Variable assignments\nstudlyCapVar = 10;\nproperCamelCase = "A String";\ntitleCaseOver = 9000;` + }, + { + url: '/learn/javascript-algorithms-and-data-structures/basic-javascript/explore-differences-between-the-var-and-let-keywords', + solution: `let catName = "Oliver";\nlet catSound = "Meow!";` + }, + { + url: '/learn/javascript-algorithms-and-data-structures/basic-javascript/declare-a-read-only-variable-with-the-const-keyword', + solution: `const FCC = "freeCodeCamp";\n// Change this line\nlet fact = "is cool!";\n// Change this line\nfact = "is awesome!";\nconsole.log(FCC, fact);\n// Change this line` } -}; +]; -const completeTenChallenges = async ({ +const completeChallenges = async ({ page, browserName, - isMobile + isMobile, + number }: { page: Page; browserName: string; isMobile: boolean; + number: number; }) => { - await page.goto( - '/learn/javascript-algorithms-and-data-structures/basic-javascript/comment-your-javascript-code' - ); - - const challenges = [ - { - url: '/learn/javascript-algorithms-and-data-structures/basic-javascript/comment-your-javascript-code', - solution: `// some comment\n/* some comment */` - }, - { - url: '/learn/javascript-algorithms-and-data-structures/basic-javascript/declare-javascript-variables', - solution: 'var myName;' - }, - { - url: '/learn/javascript-algorithms-and-data-structures/basic-javascript/storing-values-with-the-assignment-operator', - solution: `// Setup\nvar a;\n\n// Only change code below this line\na = 7;` - }, - { - url: '/learn/javascript-algorithms-and-data-structures/basic-javascript/assigning-the-value-of-one-variable-to-another', - solution: `// Setup\nvar a;\na = 7;\nvar b;\n\n// Only change code below this line\nb = a;` - }, - { - url: '/learn/javascript-algorithms-and-data-structures/basic-javascript/initializing-variables-with-the-assignment-operator', - solution: 'var a = 9;' - }, - { - url: '/learn/javascript-algorithms-and-data-structures/basic-javascript/declare-string-variables', - solution: `var myFirstName = 'foo';\nvar myLastName = 'bar';` - }, - { - url: '/learn/javascript-algorithms-and-data-structures/basic-javascript/understanding-uninitialized-variables', - solution: `// Only change code below this line\nvar a = 5;\nvar b = 10;\nvar c = 'I am a';\n// Only change code above this line\n\na = a + 1;\nb = b + 5;\nc = c + " String!";` - }, - { - url: '/learn/javascript-algorithms-and-data-structures/basic-javascript/understanding-case-sensitivity-in-variables', - solution: `// Variable declarations\nvar studlyCapVar;\nvar properCamelCase;\nvar titleCaseOver;\n\n// Variable assignments\nstudlyCapVar = 10;\nproperCamelCase = "A String";\ntitleCaseOver = 9000;` - }, - { - url: '/learn/javascript-algorithms-and-data-structures/basic-javascript/explore-differences-between-the-var-and-let-keywords', - solution: `let catName = "Oliver";\nlet catSound = "Meow!";` - }, - { - url: '/learn/javascript-algorithms-and-data-structures/basic-javascript/declare-a-read-only-variable-with-the-const-keyword', - solution: `const FCC = "freeCodeCamp";\n// Change this line\nlet fact = "is cool!";\n// Change this line\nfact = "is awesome!";\nconsole.log(FCC, fact);\n// Change this line` - } - ]; - - for (const challenge of challenges) { + await page.goto(challenges[0].url); + for (const challenge of challenges.slice(0, number)) { await page.waitForURL(challenge.url); - await focusEditor({ page, isMobile }); await clearEditor({ page, browserName }); - await page.evaluate( async contents => await navigator.clipboard.writeText(contents), challenge.solution ); await page.keyboard.press('ControlOrMeta+V'); - await page.getByRole('button', { name: 'Run' }).click(); - await expect(page.getByRole('dialog')).toBeVisible(); // completion dialog + await expect( + page.getByRole('dialog').filter({ hasText: 'Basic Javascript' }) + ).toBeVisible(); // completion dialog await page.getByRole('button', { name: 'Submit' }).click(); } }; @@ -160,6 +113,10 @@ test.skip( ); test.describe('Donation modal display', () => { + test.beforeEach(async ({ context }) => { + await addGrowthbookCookie({ context, variation: 'A' }); + }); + test.use({ storageState: 'playwright/.auth/certified-user.json' }); test('should display the content correctly and disable close when the animation is not complete', async ({ @@ -171,7 +128,7 @@ test.describe('Donation modal display', () => { await context.grantPermissions(['clipboard-read', 'clipboard-write']); test.setTimeout(40000); - await completeThreeChallenges({ page, browserName, isMobile }); + await completeChallenges({ page, browserName, isMobile, number: 3 }); const donationModal = page .getByRole('dialog') @@ -241,6 +198,9 @@ test.describe('Donation modal display', () => { test.describe('Donation modal appearance logic - New user', () => { test.use({ storageState: 'playwright/.auth/development-user.json' }); + test.beforeEach(async ({ context }) => { + await addGrowthbookCookie({ context, variation: 'B' }); + }); test.beforeEach(() => { execSync('node ./tools/scripts/seed/seed-demo-user'); @@ -259,7 +219,7 @@ test.describe('Donation modal appearance logic - New user', () => { await context.grantPermissions(['clipboard-read', 'clipboard-write']); // Development user doesn't have any completed challenges, we are completing the first 3. - await completeThreeChallenges({ page, browserName, isMobile }); + await completeChallenges({ page, browserName, isMobile, number: 3 }); const donationModal = page .getByRole('dialog') @@ -276,7 +236,7 @@ test.describe('Donation modal appearance logic - New user', () => { await context.grantPermissions(['clipboard-read', 'clipboard-write']); test.setTimeout(50000); - await completeTenChallenges({ page, isMobile, browserName }); + await completeChallenges({ page, isMobile, browserName, number: 10 }); const donationModal = page .getByRole('dialog') @@ -333,6 +293,9 @@ test.describe('Donation modal appearance logic - New user', () => { test.describe('Donation modal appearance logic - Certified user', () => { test.use({ storageState: 'playwright/.auth/certified-user.json' }); + test.beforeEach(async ({ context }) => { + await addGrowthbookCookie({ context, variation: 'A' }); + }); test('should appear if the user has completed 3 challenges and has more than 10 completed challenges in total', async ({ page, @@ -344,7 +307,7 @@ test.describe('Donation modal appearance logic - Certified user', () => { test.setTimeout(40000); // Certified user already has more than 10 completed challenges, we are just completing 3 more. - await completeThreeChallenges({ page, browserName, isMobile }); + await completeChallenges({ page, isMobile, browserName, number: 3 }); const donationModal = page .getByRole('dialog') @@ -390,7 +353,7 @@ test.describe('Donation modal appearance logic - Donor user', () => { }) => { await context.grantPermissions(['clipboard-read', 'clipboard-write']); - await completeThreeChallenges({ page, browserName, isMobile }); + await completeChallenges({ page, browserName, isMobile, number: 3 }); const donationModal = page .getByRole('dialog') diff --git a/e2e/landing.spec.ts b/e2e/landing.spec.ts index ec93697a00f..8eceb6b7cd8 100644 --- a/e2e/landing.spec.ts +++ b/e2e/landing.spec.ts @@ -1,7 +1,8 @@ -import { BrowserContext, expect, Page, test } from '@playwright/test'; +import { expect, Page, test } from '@playwright/test'; import intro from '../client/i18n/locales/english/intro.json'; import translations from '../client/i18n/locales/english/translations.json'; import { SuperBlocks } from '../shared/config/curriculum'; +import { addGrowthbookCookie } from './utils/add-growthbook-cookie'; const landingPageElements = { heading: 'landing-header', @@ -37,25 +38,13 @@ const superBlocks = [ intro[SuperBlocks.PythonForEverybody].title ]; -async function addDefaultCookies(context: BrowserContext, variation: string) { - await context.addCookies([ - { - name: 'gbuuid', - value: variation, - domain: 'localhost', - path: '/', - expires: Math.floor(Date.now() / 1000) + 400 * 24 * 60 * 60 // 400 days from now - } - ]); -} - async function goToLandingPage(page: Page) { await page.goto('/'); } test.describe('Landing Page - Variation B', () => { test.beforeEach(async ({ context, page }) => { - await addDefaultCookies(context, 'B'); + await addGrowthbookCookie({ context, variation: 'B' }); await goToLandingPage(page); }); @@ -132,7 +121,7 @@ test.describe('Landing Page - Variation B', () => { test.describe('Landing Page - Variation A', () => { test.beforeEach(async ({ context, page }) => { - await addDefaultCookies(context, 'A'); + await addGrowthbookCookie({ context, variation: 'A' }); await goToLandingPage(page); }); diff --git a/e2e/utils/add-growthbook-cookie.ts b/e2e/utils/add-growthbook-cookie.ts new file mode 100644 index 00000000000..437c6b47d43 --- /dev/null +++ b/e2e/utils/add-growthbook-cookie.ts @@ -0,0 +1,19 @@ +import { BrowserContext } from '@playwright/test'; + +export async function addGrowthbookCookie({ + context, + variation +}: { + context: BrowserContext; + variation: string; +}) { + await context.addCookies([ + { + name: 'gbuuid', + value: variation, + domain: 'localhost', + path: '/', + expires: Math.floor(Date.now() / 1000) + 400 * 24 * 60 * 60 // 400 days from now + } + ]); +}