refactor: split curriculum build in two (#63639)

client/package.json

@@ -22,6 +22,7 @@
     "prebuild": "pnpm run common-setup && pnpm run build:scripts --env production",
     "build": "NODE_OPTIONS=\"--max-old-space-size=7168 --no-deprecation\" gatsby build --prefix-paths",
     "build:scripts": "pnpm run -F=browser-scripts build",
+    "build:external-curriculum": "tsx ./tools/external-curriculum/build",
     "clean": "gatsby clean",
     "common-setup": "pnpm -w run compile:ts && pnpm run create:env && pnpm run create:trending && pnpm run create:search-placeholder",
     "create:env": "DEBUG=fcc:* tsx ./tools/create-env.ts",
@@ -175,6 +176,7 @@
     "monaco-editor-webpack-plugin": "7.0.1",
     "node-fetch": "2.7.0",
     "react-test-renderer": "17.0.2",
+    "readdirp": "3.6.0",
     "redux-saga-test-plan": "4.0.6",
     "serve": "13.0.4",
     "vitest": "^3.2.4",

client/tools/external-curriculum/build-external-curricula-data-v1.test.ts

@@ -1,9 +1,10 @@
 import path from 'path';
-import fs, { readFileSync } from 'fs';
+import fs from 'fs';
 
 import readdirp from 'readdirp';
 import { describe, test, expect } from 'vitest';
 
+import intros from '../../i18n/locales/english/intro.json';
 import {
   SuperBlocks,
   SuperBlockStage,
@@ -15,18 +16,11 @@ import {
 } from './external-data-schema-v1';
 import {
   type Curriculum,
-  type CurriculumIntros,
   type GeneratedCurriculumProps,
   orderedSuperBlockInfo
 } from './build-external-curricula-data-v1';
 
 const VERSION = 'v1';
-const intros = JSON.parse(
-  readFileSync(
-    path.resolve(__dirname, '../../../client/i18n/locales/english/intro.json'),
-    'utf-8'
-  )
-) as CurriculumIntros;
 
 describe('external curriculum data build', () => {
   const clientStaticPath = path.resolve(__dirname, '../../../client/static');
@@ -129,8 +123,13 @@ describe('external curriculum data build', () => {
       const randomBlock = blocks[randomBlockIndex];
 
       expect(fileContent[superBlock].intro).toEqual(intros[superBlock].intro);
-      expect(fileContent[superBlock].blocks[randomBlock].desc).toEqual(
-        intros[superBlock].blocks[randomBlock].intro
+      expect(fileContent[superBlock].blocks[randomBlock]?.desc).toEqual(
+        (
+          intros[superBlock].blocks as unknown as Record<
+            string,
+            { intro: unknown }
+          >
+        )[randomBlock].intro
       );
     });
   });

client/tools/external-curriculum/build-external-curricula-data-v1.ts

@@ -2,7 +2,6 @@ import { mkdirSync, writeFileSync, readFileSync } from 'fs';
 import { resolve, dirname } from 'path';
 import { omit } from 'lodash';
 import { submitTypes } from '../../../shared-dist/config/challenge-types';
-import { type ChallengeNode } from '../../../client/src/redux/prop-types';
 import { SuperBlocks } from '../../../shared-dist/config/curriculum';
 import { patchBlock } from './patches';
 
@@ -22,7 +21,7 @@ export type Curriculum<T> = {
 
 export interface CurriculumProps {
   intro: string[];
-  blocks: Record<string, Block<ChallengeNode['challenge'][]>>;
+  blocks: Record<string, Block<{ id: string }[]>>;
 }
 
 export interface GeneratedCurriculumProps {
@@ -126,14 +125,14 @@ export function buildExtCurriculumDataV1(
 
         superBlock[superBlockKey]['blocks'][blockName]['challenges'] =
           patchBlock(
-            omit(curriculum[superBlockKey]['blocks'][blockName]['meta'], [
+            omit(curriculum[superBlockKey]['blocks'][blockName]?.meta, [
               'chapter',
               'module'
             ])
           );
 
         const blockChallenges =
-          curriculum[superBlockKey]['blocks'][blockName]['challenges'];
+          curriculum[superBlockKey]['blocks'][blockName]?.challenges;
 
         for (const challenge of blockChallenges) {
           const challengeId = challenge.id;

client/tools/external-curriculum/build-external-curricula-data-v2.test.ts

@@ -112,7 +112,9 @@ describe('external curriculum data build', () => {
         'utf-8'
       );
 
-      const result = validateSuperBlock(JSON.parse(fileContent));
+      const result = validateSuperBlock(
+        JSON.parse(fileContent) as Record<string, unknown>
+      );
 
       expect(result.error?.details).toBeUndefined();
       expect(result.error).toBeFalsy();
@@ -280,7 +282,7 @@ describe('external curriculum data build', () => {
     expect(stages).not.toContain('upcoming');
 
     for (const stage of stages) {
-      const superBlockDashedNames = orderedSuperBlockInfo[stage].map(
+      const superBlockDashedNames = orderedSuperBlockInfo[stage]?.map(
         superBlock => superBlock.dashedName
       );
 

client/tools/external-curriculum/build-external-curricula-data-v2.ts

@@ -2,7 +2,7 @@ import { mkdirSync, writeFileSync, readFileSync } from 'fs';
 import { resolve, dirname } from 'path';
 import { omit } from 'lodash';
 import { submitTypes } from '../../../shared-dist/config/challenge-types';
-import { type ChallengeNode } from '../../../client/src/redux/prop-types';
+import { type ChallengeNode } from '../../src/redux/prop-types';
 import {
   SuperBlocks,
   chapterBasedSuperBlocks

client/tools/external-curriculum/build.ts

@@ -0,0 +1,29 @@
+import curriculum from '../../../shared-dist/config/curriculum.json';
+import {
+  buildExtCurriculumDataV1,
+  Curriculum as CurriculumV1,
+  CurriculumProps as CurriculumPropsV1
+} from './build-external-curricula-data-v1';
+import {
+  buildExtCurriculumDataV2,
+  Curriculum as CurriculumV2,
+  CurriculumProps as CurriculumPropsV2
+} from './build-external-curricula-data-v2';
+
+const isSelectiveBuild =
+  process.env.FCC_SUPERBLOCK ||
+  process.env.FCC_BLOCK ||
+  process.env.FCC_CHALLENGE_ID;
+
+if (isSelectiveBuild) {
+  console.log(
+    'Skipping external curriculum build (selective build mode active)'
+  );
+} else {
+  buildExtCurriculumDataV1(
+    curriculum as unknown as CurriculumV1<CurriculumPropsV1>
+  );
+  buildExtCurriculumDataV2(
+    curriculum as unknown as CurriculumV2<CurriculumPropsV2>
+  );
+}

client/tools/external-curriculum/external-data-schema-v1.ts

@@ -1,7 +1,5 @@
-const Joi = require('joi');
-const {
-  chapterBasedSuperBlocks
-} = require('../../../shared-dist/config/curriculum');
+import Joi from 'joi';
+import { chapterBasedSuperBlocks } from '../../../shared-dist/config/curriculum';
 
 const blockSchema = Joi.object({}).keys({
   desc: Joi.array().min(1),
@@ -88,8 +86,8 @@ const availableSuperBlocksSchema = Joi.object({
   )
 });
 
-exports.superblockSchemaValidator = () => superblock =>
+export const superblockSchemaValidator = () => (superblock: unknown) =>
   schema.validate(superblock);
 
-exports.availableSuperBlocksValidator = () => data =>
+export const availableSuperBlocksValidator = () => (data: unknown) =>
   availableSuperBlocksSchema.validate(data);

client/tools/external-curriculum/external-data-schema-v2.ts

@@ -1,7 +1,5 @@
-const Joi = require('joi');
-const {
-  chapterBasedSuperBlocks
-} = require('../../../shared-dist/config/curriculum');
+import Joi from 'joi';
+import { chapterBasedSuperBlocks } from '../../../shared-dist/config/curriculum';
 
 const slugRE = new RegExp('^[a-z0-9-]+$');
 
@@ -121,15 +119,16 @@ const availableSuperBlocksSchema = Joi.object({
   )
 });
 
-exports.superblockSchemaValidator = () => superBlock => {
-  const superBlockName = Object.keys(superBlock)[0];
+export const superblockSchemaValidator =
+  () => (superBlock: Record<string, unknown>) => {
+    const superBlockName = Object.keys(superBlock)[0];
 
-  if (chapterBasedSuperBlocks.includes(superBlockName)) {
-    return chapterBasedCurriculumSchema.validate(superBlock);
-  }
+    if (chapterBasedSuperBlocks.includes(superBlockName)) {
+      return chapterBasedCurriculumSchema.validate(superBlock);
+    }
 
-  return blockBasedCurriculumSchema.validate(superBlock);
-};
+    return blockBasedCurriculumSchema.validate(superBlock);
+  };
 
-exports.availableSuperBlocksValidator = () => data =>
+export const availableSuperBlocksValidator = () => (data: unknown) =>
   availableSuperBlocksSchema.validate(data);

curriculum/package.json

@@ -18,7 +18,7 @@
   "homepage": "https://github.com/freeCodeCamp/freeCodeCamp#readme",
   "author": "freeCodeCamp <team@freecodecamp.org>",
   "scripts": {
-    "build": "tsx --tsconfig ../tsconfig.json ../tools/scripts/build/build-curriculum",
+    "build": "tsx ./src/generate/build-curriculum",
     "create-empty-steps": "CALLING_DIR=$INIT_CWD tsx --tsconfig ../tsconfig.json ../tools/challenge-helper-scripts/create-empty-steps",
     "create-next-challenge": "CALLING_DIR=$INIT_CWD tsx --tsconfig ../tsconfig.json ../tools/challenge-helper-scripts/create-next-challenge",
     "create-this-challenge": "CALLING_DIR=$INIT_CWD tsx --tsconfig ../tsconfig.json ../tools/challenge-helper-scripts/create-this-challenge",
@@ -58,7 +58,6 @@
     "ora": "5.4.1",
     "polka": "^0.5.2",
     "puppeteer": "22.12.1",
-    "readdirp": "3.6.0",
     "sirv": "^3.0.2",
     "string-similarity": "4.0.4",
     "vitest": "^3.2.4"

curriculum/src/generate/build-curriculum.ts

@@ -0,0 +1,12 @@
+import fs from 'fs';
+import path from 'path';
+
+import { getChallengesForLang } from '../get-challenges';
+
+const globalConfigPath = path.resolve(__dirname, '../../../shared-dist/config');
+
+void getChallengesForLang('english')
+  .then(JSON.stringify)
+  .then(json => {
+    fs.writeFileSync(`${globalConfigPath}/curriculum.json`, json);
+  });

package.json

@@ -25,7 +25,7 @@
     "build": "npm-run-all -p build:*",
     "build-workers": "cd ./client && pnpm run prebuild",
     "build:client": "cd ./client && pnpm run build",
-    "build:curriculum": "cd ./curriculum && pnpm run build",
+    "build:curriculum": "pnpm -F=curriculum run build && pnpm -F=client run build:external-curriculum",
     "build:api": "cd ./api && pnpm run build",
     "challenge-editor": "npm-run-all -p challenge-editor:*",
     "challenge-editor:client": "cd ./tools/challenge-editor/client && pnpm start",
@@ -73,7 +73,6 @@
     "test": "NODE_OPTIONS='--max-old-space-size=7168' run-s compile:ts build:curriculum build-workers test:**",
     "test:api": "cd api && pnpm test",
     "test:tools:challenge-helper-scripts": "cd ./tools/challenge-helper-scripts && pnpm test run",
-    "test:tools:scripts-build": "cd ./tools/scripts/build && pnpm test run",
     "test:tools:scripts-lint": "cd ./tools/scripts/lint && pnpm test run",
     "test:tools:challenge-parser": "cd ./tools/challenge-parser && pnpm test run",
     "test:curriculum:content": "cd ./curriculum && pnpm test run",

pnpm-lock.yaml

@@ -652,6 +652,9 @@ importers:
       react-test-renderer:
         specifier: 17.0.2
         version: 17.0.2(react@17.0.2)
+      readdirp:
+        specifier: 3.6.0
+        version: 3.6.0
       redux-saga-test-plan:
         specifier: 4.0.6
         version: 4.0.6(@redux-saga/is@1.1.3)(@redux-saga/symbols@1.1.3)(redux-saga@1.2.3)
@@ -724,9 +727,6 @@ importers:
       puppeteer:
         specifier: 22.12.1
         version: 22.12.1(typescript@5.8.2)
-      readdirp:
-        specifier: 3.6.0
-        version: 3.6.0
       sirv:
         specifier: ^3.0.2
         version: 3.0.2
@@ -815,6 +815,9 @@ importers:
         specifier: 4.0.3
         version: 4.0.3
     devDependencies:
+      '@total-typescript/ts-reset':
+        specifier: ^0.6.1
+        version: 0.6.1
       '@types/cors':
         specifier: ^2.8.13
         version: 2.8.18
@@ -1063,9 +1066,6 @@ importers:
       chokidar:
         specifier: 3.6.0
         version: 3.6.0
-      readdirp:
-        specifier: 3.6.0
-        version: 3.6.0
 
   tools/daily-challenges:
     devDependencies:
@@ -1082,24 +1082,6 @@ importers:
         specifier: 5.8.2
         version: 5.8.2
 
-  tools/scripts/build:
-    devDependencies:
-      '@total-typescript/ts-reset':
-        specifier: ^0.5.0
-        version: 0.5.1
-      '@vitest/ui':
-        specifier: ^3.2.4
-        version: 3.2.4(vitest@3.2.4)
-      joi:
-        specifier: 17.12.2
-        version: 17.12.2
-      readdirp:
-        specifier: 3.6.0
-        version: 3.6.0
-      vitest:
-        specifier: ^3.2.4
-        version: 3.2.4(@types/debug@4.1.12)(@types/node@20.12.8)(@vitest/ui@3.2.4)(jiti@2.6.1)(jsdom@26.1.0)(msw@2.8.7(@types/node@20.12.8)(typescript@5.8.2))(terser@5.28.1)(tsx@4.19.1)(yaml@2.8.0)
-
   tools/scripts/lint:
     devDependencies:
       '@vitest/ui':

pnpm-workspace.yaml

@@ -10,7 +10,6 @@ packages:
   - 'tools/client-plugins/*'
   - 'tools/crowdin'
   - 'tools/daily-challenges'
-  - 'tools/scripts/build'
   - 'tools/scripts/lint'
   - 'tools/scripts/seed'
   - 'tools/scripts/seed-exams'

tools/challenge-editor/api/package.json

@@ -15,6 +15,7 @@
     "gray-matter": "4.0.3"
   },
   "devDependencies": {
+    "@total-typescript/ts-reset": "^0.6.1",
     "@types/cors": "^2.8.13",
     "@types/express": "4.17.21",
     "dotenv": "16.4.5",

tools/client-plugins/gatsby-source-challenges/package.json

@@ -19,7 +19,6 @@
   "author": "freeCodeCamp <team@freecodecamp.org>",
   "main": "gatsby-node.js",
   "devDependencies": {
-    "chokidar": "3.6.0",
-    "readdirp": "3.6.0"
+    "chokidar": "3.6.0"
   }
 }

tools/daily-challenges/helpers.ts

@@ -126,7 +126,7 @@ export function combineChallenges({
   return challengeData;
 }
 
-export function handleError(err: Error, client: MongoClient) {
+export function handleError(err: unknown, client: MongoClient) {
   if (err) {
     console.error('Oh noes!! Error seeding Daily Challenges.');
     console.error(err);

tools/scripts/build/build-curriculum.ts

@@ -1,43 +0,0 @@
-import fs from 'fs';
-import path from 'path';
-
-import { getChallengesForLang } from '../../../curriculum/src/get-challenges';
-import {
-  buildExtCurriculumDataV1,
-  type Curriculum as CurriculumV1,
-  type CurriculumProps as CurriculumPropsV1
-} from './build-external-curricula-data-v1';
-import {
-  buildExtCurriculumDataV2,
-  type Curriculum as CurriculumV2,
-  type CurriculumProps as CurriculumPropsV2
-} from './build-external-curricula-data-v2';
-
-const globalConfigPath = path.resolve(__dirname, '../../../shared-dist/config');
-
-const isSelectiveBuild =
-  process.env.FCC_SUPERBLOCK ||
-  process.env.FCC_BLOCK ||
-  process.env.FCC_CHALLENGE_ID;
-
-void getChallengesForLang('english')
-  .then(result => {
-    if (!isSelectiveBuild) {
-      console.log('Building external curriculum data...');
-      buildExtCurriculumDataV1(
-        result as unknown as CurriculumV1<CurriculumPropsV1>
-      );
-      buildExtCurriculumDataV2(
-        result as unknown as CurriculumV2<CurriculumPropsV2>
-      );
-    } else {
-      console.log(
-        'Skipping external curriculum build (selective build mode active)'
-      );
-    }
-    return result;
-  })
-  .then(JSON.stringify)
-  .then(json => {
-    fs.writeFileSync(`${globalConfigPath}/curriculum.json`, json);
-  });

tools/scripts/build/package.json

@@ -1,31 +0,0 @@
-{
-  "name": "@freecodecamp/scripts-build",
-  "version": "0.0.1",
-  "description": "The freeCodeCamp.org open-source codebase and curriculum",
-  "license": "BSD-3-Clause",
-  "private": true,
-  "engines": {
-    "node": ">=16",
-    "pnpm": ">=10"
-  },
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/freeCodeCamp/freeCodeCamp.git"
-  },
-  "bugs": {
-    "url": "https://github.com/freeCodeCamp/freeCodeCamp/issues"
-  },
-  "homepage": "https://github.com/freeCodeCamp/freeCodeCamp#readme",
-  "author": "freeCodeCamp <team@freecodecamp.org>",
-  "main": "none",
-  "scripts": {
-    "test": "vitest"
-  },
-  "devDependencies": {
-    "@total-typescript/ts-reset": "^0.5.0",
-    "@vitest/ui": "^3.2.4",
-    "joi": "17.12.2",
-    "readdirp": "3.6.0",
-    "vitest": "^3.2.4"
-  }
-}