mirror of
https://github.com/freeCodeCamp/freeCodeCamp.git
synced 2026-01-06 15:03:08 -05:00
560 lines
17 KiB
JavaScript
560 lines
17 KiB
JavaScript
const fs = require('fs');
|
|
const path = require('path');
|
|
const util = require('util');
|
|
const yaml = require('js-yaml');
|
|
const { findIndex } = require('lodash');
|
|
const readDirP = require('readdirp');
|
|
const stringSimilarity = require('string-similarity');
|
|
|
|
const { curriculum: curriculumLangs } =
|
|
require('../shared/config/i18n').availableLangs;
|
|
const { parseMD } = require('../tools/challenge-parser/parser');
|
|
|
|
const {
|
|
translateCommentsInChallenge
|
|
} = require('../tools/challenge-parser/translation-parser');
|
|
|
|
const { isAuditedSuperBlock } = require('../shared/utils/is-audited');
|
|
const { createPoly } = require('../shared/utils/polyvinyl');
|
|
const { chapterBasedSuperBlocks } = require('../shared/config/curriculum');
|
|
const {
|
|
getSuperOrder,
|
|
getSuperBlockFromDir,
|
|
getChapterFromBlock,
|
|
getModuleFromBlock,
|
|
getBlockOrder
|
|
} = require('./utils');
|
|
const { metaSchemaValidator } = require('./schema/meta-schema');
|
|
const {
|
|
assertSuperBlockStructure
|
|
} = require('./schema/superblock-structure-schema');
|
|
|
|
const fullStackSuperBlockStructure = require('./superblock-structure/full-stack.json');
|
|
|
|
assertSuperBlockStructure(fullStackSuperBlockStructure);
|
|
|
|
const access = util.promisify(fs.access);
|
|
|
|
const ENGLISH_CHALLENGES_DIR = path.resolve(__dirname, 'challenges');
|
|
const ENGLISH_DICTIONARIES_DIR = path.resolve(__dirname, 'dictionaries');
|
|
const META_DIR = path.resolve(ENGLISH_CHALLENGES_DIR, '_meta');
|
|
|
|
// This is to allow English to build without having to download the i18n files.
|
|
// It fails when trying to resolve the i18n-curriculum path if they don't exist.
|
|
const curriculumLocale = process.env.CURRICULUM_LOCALE ?? 'english';
|
|
const I18N_CURRICULUM_DIR = path.resolve(
|
|
__dirname,
|
|
curriculumLocale === 'english' ? '.' : 'i18n-curriculum/curriculum'
|
|
);
|
|
|
|
const I18N_CHALLENGES_DIR = path.resolve(I18N_CURRICULUM_DIR, 'challenges');
|
|
const I18N_DICTIONARIES_DIR = path.resolve(I18N_CURRICULUM_DIR, 'dictionaries');
|
|
|
|
exports.ENGLISH_CHALLENGES_DIR = ENGLISH_CHALLENGES_DIR;
|
|
exports.META_DIR = META_DIR;
|
|
exports.I18N_CHALLENGES_DIR = I18N_CHALLENGES_DIR;
|
|
|
|
const COMMENT_TRANSLATIONS = createCommentMap(
|
|
I18N_DICTIONARIES_DIR,
|
|
ENGLISH_DICTIONARIES_DIR
|
|
);
|
|
|
|
function createCommentMap(dictionariesDir, englishDictionariesDir) {
|
|
const languages = fs.readdirSync(dictionariesDir);
|
|
|
|
// get all their dictionaries
|
|
const dictionaries = languages.reduce(
|
|
(acc, lang) => ({
|
|
...acc,
|
|
[lang]: require(path.resolve(dictionariesDir, lang, 'comments.json'))
|
|
}),
|
|
{}
|
|
);
|
|
|
|
// get the english dicts
|
|
const COMMENTS_TO_TRANSLATE = require(
|
|
path.resolve(englishDictionariesDir, 'english', 'comments.json')
|
|
);
|
|
|
|
const COMMENTS_TO_NOT_TRANSLATE = require(
|
|
path.resolve(
|
|
englishDictionariesDir,
|
|
'english',
|
|
'comments-to-not-translate.json'
|
|
)
|
|
);
|
|
|
|
// map from english comment text to translations
|
|
const translatedCommentMap = Object.entries(COMMENTS_TO_TRANSLATE).reduce(
|
|
(acc, [id, text]) => {
|
|
return {
|
|
...acc,
|
|
[text]: getTranslationEntry(dictionaries, { engId: id, text })
|
|
};
|
|
},
|
|
{}
|
|
);
|
|
|
|
// map from english comment text to itself
|
|
const untranslatableCommentMap = Object.values(
|
|
COMMENTS_TO_NOT_TRANSLATE
|
|
).reduce((acc, text) => {
|
|
const englishEntry = languages.reduce(
|
|
(acc, lang) => ({
|
|
...acc,
|
|
[lang]: text
|
|
}),
|
|
{}
|
|
);
|
|
return {
|
|
...acc,
|
|
[text]: englishEntry
|
|
};
|
|
}, {});
|
|
|
|
const allComments = { ...translatedCommentMap, ...untranslatableCommentMap };
|
|
|
|
// the english entries need to be added here, because english is not in
|
|
// languages
|
|
Object.keys(allComments).forEach(comment => {
|
|
allComments[comment].english = comment;
|
|
});
|
|
|
|
return allComments;
|
|
}
|
|
|
|
exports.createCommentMap = createCommentMap;
|
|
|
|
function getTranslationEntry(dicts, { engId, text }) {
|
|
return Object.keys(dicts).reduce((acc, lang) => {
|
|
const entry = dicts[lang][engId];
|
|
if (entry) {
|
|
return { ...acc, [lang]: entry };
|
|
} else {
|
|
// default to english
|
|
return { ...acc, [lang]: text };
|
|
}
|
|
}, {});
|
|
}
|
|
|
|
function getChallengesDirForLang(lang) {
|
|
if (lang === 'english') {
|
|
return path.resolve(ENGLISH_CHALLENGES_DIR, `${lang}`);
|
|
} else {
|
|
return path.resolve(I18N_CHALLENGES_DIR, `${lang}`);
|
|
}
|
|
}
|
|
|
|
function getMetaForBlock(block) {
|
|
return JSON.parse(
|
|
fs.readFileSync(path.resolve(META_DIR, `${block}/meta.json`), 'utf8')
|
|
);
|
|
}
|
|
|
|
function parseCert(filePath) {
|
|
return yaml.load(fs.readFileSync(filePath, 'utf8'));
|
|
}
|
|
|
|
exports.getChallengesDirForLang = getChallengesDirForLang;
|
|
exports.getMetaForBlock = getMetaForBlock;
|
|
|
|
// This recursively walks the directories starting at root, and calls cb for
|
|
// each file/directory and only resolves once all the callbacks do.
|
|
const walk = (root, target, options, cb) => {
|
|
return new Promise(resolve => {
|
|
let running = 1;
|
|
function done() {
|
|
if (--running === 0) {
|
|
resolve(target);
|
|
}
|
|
}
|
|
readDirP(root, options)
|
|
.on('data', file => {
|
|
running++;
|
|
cb(file, target).then(done);
|
|
})
|
|
.on('end', done);
|
|
});
|
|
};
|
|
|
|
exports.getChallengesForLang = async function getChallengesForLang(
|
|
lang,
|
|
filters
|
|
) {
|
|
const invalidLang = !curriculumLangs.includes(lang);
|
|
if (invalidLang)
|
|
throw Error(`${lang} is not a accepted language.
|
|
Accepted languages are ${curriculumLangs.join(', ')}`);
|
|
// english determines the shape of the curriculum, all other languages mirror
|
|
// it.
|
|
const root = getChallengesDirForLang('english');
|
|
// scaffold the curriculum, first set up the superblocks, then recurse into
|
|
// the blocks
|
|
const curriculum = await walk(
|
|
root,
|
|
{},
|
|
{ type: 'directories', depth: 0 },
|
|
buildSuperBlocks
|
|
);
|
|
|
|
const superBlocks = Object.keys(curriculum);
|
|
const blocksWithParent = Object.entries(curriculum).flatMap(
|
|
([key, superBlock]) => {
|
|
const blocks = Object.entries(superBlock.blocks);
|
|
return blocks.map(([block, blockData]) => ({
|
|
block,
|
|
blockData,
|
|
superBlock: key
|
|
}));
|
|
}
|
|
);
|
|
|
|
const blocks = blocksWithParent.map(({ block }) => block);
|
|
|
|
let filteredCurriculum = curriculum;
|
|
const updatedFilters = { ...filters };
|
|
if (filters?.superBlock) {
|
|
const target = stringSimilarity.findBestMatch(
|
|
filters.superBlock,
|
|
superBlocks
|
|
).bestMatch.target;
|
|
|
|
console.log('superBlock being tested:', target);
|
|
|
|
filteredCurriculum = {
|
|
[target]: curriculum[target]
|
|
};
|
|
updatedFilters.superBlock = target;
|
|
} else if (filters?.block) {
|
|
const target = stringSimilarity.findBestMatch(filters.block, blocks)
|
|
.bestMatch.target;
|
|
|
|
console.log('block being tested:', target);
|
|
const targetBlock = blocksWithParent.find(({ block }) => block === target);
|
|
|
|
filteredCurriculum = {
|
|
[targetBlock.superBlock]: {
|
|
blocks: {
|
|
[targetBlock.block]: targetBlock.blockData
|
|
}
|
|
}
|
|
};
|
|
updatedFilters.block = targetBlock.block;
|
|
} else if (filters?.challengeId) {
|
|
const blocksWithMeta = blocksWithParent.filter(
|
|
({ blockData }) => blockData.meta
|
|
);
|
|
const container = blocksWithMeta.filter(({ blockData }) => {
|
|
return blockData.meta.challengeOrder.some(
|
|
({ id }) => id === filters.challengeId
|
|
);
|
|
});
|
|
|
|
if (container.length === 0) {
|
|
throw new Error(`No block found with challengeId ${filters.challengeId}`);
|
|
}
|
|
if (container.length > 1) {
|
|
throw new Error(
|
|
`Multiple blocks found with challengeId ${filters.challengeId}`
|
|
);
|
|
}
|
|
const targetBlock = container[0];
|
|
filteredCurriculum = {
|
|
[targetBlock.superBlock]: {
|
|
blocks: {
|
|
[targetBlock.block]: targetBlock.blockData
|
|
}
|
|
}
|
|
};
|
|
updatedFilters.block = targetBlock.block;
|
|
updatedFilters.superBlock = targetBlock.superBlock;
|
|
}
|
|
|
|
const cb = (file, curriculum) =>
|
|
buildChallenges(file, curriculum, lang, updatedFilters);
|
|
// fill the scaffold with the challenges
|
|
return walk(
|
|
root,
|
|
filteredCurriculum,
|
|
{ type: 'files', fileFilter: ['*.md', '*.yml'] },
|
|
cb
|
|
);
|
|
};
|
|
|
|
async function buildBlocks(file, curriculum, superBlock) {
|
|
const { basename: blockName } = file;
|
|
const metaPath = path.resolve(META_DIR, `${blockName}/meta.json`);
|
|
const isCertification = !fs.existsSync(metaPath);
|
|
const isEmptyDir = fs.readdirSync(file.fullPath).length === 0;
|
|
if (isEmptyDir) {
|
|
throw Error(
|
|
`Block directory, ${file.fullPath}, is empty.
|
|
If this block should exist, please add challenge files to it.
|
|
If this block should not exist, please remove the directory.`
|
|
);
|
|
}
|
|
if (isCertification && superBlock !== 'certifications') {
|
|
throw Error(
|
|
`superblock ${superBlock} is missing meta.json for ${blockName}`
|
|
);
|
|
}
|
|
|
|
if (isCertification) {
|
|
curriculum['certifications'].blocks[blockName] = { challenges: [] };
|
|
} else {
|
|
const blockMeta = JSON.parse(fs.readFileSync(metaPath));
|
|
|
|
const validateMeta = metaSchemaValidator(blockMeta);
|
|
if (validateMeta.error) {
|
|
throw Error(
|
|
`${validateMeta.error} in meta.json for block '${blockName}'`
|
|
);
|
|
}
|
|
|
|
const { isUpcomingChange } = blockMeta;
|
|
|
|
if (!isUpcomingChange || process.env.SHOW_UPCOMING_CHANGES === 'true') {
|
|
// add the block to the superBlock
|
|
const blockInfo = { meta: blockMeta, challenges: [] };
|
|
curriculum[superBlock].blocks[blockName] = blockInfo;
|
|
}
|
|
}
|
|
}
|
|
|
|
async function buildSuperBlocks({ path, fullPath }, curriculum) {
|
|
const superBlock = getSuperBlockFromDir(getBaseDir(path));
|
|
curriculum[superBlock] = { blocks: {} };
|
|
|
|
const cb = (file, curriculum) => buildBlocks(file, curriculum, superBlock);
|
|
return walk(fullPath, curriculum, { depth: 1, type: 'directories' }, cb);
|
|
}
|
|
|
|
async function buildChallenges({ path: filePath }, curriculum, lang, filters) {
|
|
// path is relative to getChallengesDirForLang(lang)
|
|
const block = getBlockNameFromPath(filePath);
|
|
if (filters?.block && block !== filters.block) {
|
|
return;
|
|
}
|
|
const superBlockDir = getBaseDir(filePath);
|
|
const superBlock = getSuperBlockFromDir(superBlockDir);
|
|
if (filters?.superBlock && superBlock !== filters.superBlock) {
|
|
return;
|
|
}
|
|
let challengeBlock;
|
|
|
|
// TODO: this try block and process exit can all go once errors terminate the
|
|
// tests correctly.
|
|
try {
|
|
challengeBlock = curriculum[superBlock].blocks[block];
|
|
if (!challengeBlock) {
|
|
// this should only happen when a isUpcomingChange block is skipped
|
|
return;
|
|
}
|
|
} catch (e) {
|
|
console.error(e);
|
|
console.log(`failed to create superBlock from ${superBlockDir}`);
|
|
process.exit(1);
|
|
}
|
|
const { meta } = challengeBlock;
|
|
const isCert = path.extname(filePath) === '.yml';
|
|
const englishPath = path.resolve(
|
|
__dirname,
|
|
ENGLISH_CHALLENGES_DIR,
|
|
'english',
|
|
filePath
|
|
);
|
|
const i18nPath = path.resolve(__dirname, I18N_CHALLENGES_DIR, lang, filePath);
|
|
const createChallenge = generateChallengeCreator(lang, englishPath, i18nPath);
|
|
|
|
await assertHasEnglishSource(filePath, lang, englishPath);
|
|
const challenge = isCert
|
|
? await parseCert(englishPath)
|
|
: await createChallenge(filePath, meta);
|
|
|
|
// this builds the entire block, even if we only want one challenge, which is
|
|
// inefficient, but finding the next challenge without building the whole
|
|
// block is fiddly.
|
|
challengeBlock.challenges = [...challengeBlock.challenges, challenge];
|
|
}
|
|
|
|
// This is a slightly weird abstraction, but it lets us define helper functions
|
|
// without passing around a ton of arguments.
|
|
function generateChallengeCreator(lang, englishPath, i18nPath) {
|
|
function addMetaToChallenge(challenge, meta) {
|
|
function addChapterAndModuleToChallenge(challenge) {
|
|
if (chapterBasedSuperBlocks.includes(challenge.superBlock)) {
|
|
const chapter = getChapterFromBlock(
|
|
challenge.block,
|
|
fullStackSuperBlockStructure
|
|
);
|
|
|
|
if (!meta.isUpcomingChange && chapter.comingSoon) {
|
|
throw Error(
|
|
`The '${chapter.dashedName}' chapter is 'comingSoon', but its '${meta.dashedName}' block is not hidden. Set 'isUpcomingChange' to 'true' in the 'meta.json' for the block to hide it.`
|
|
);
|
|
}
|
|
|
|
challenge.chapter = chapter.dashedName;
|
|
|
|
const module = getModuleFromBlock(
|
|
challenge.block,
|
|
fullStackSuperBlockStructure
|
|
);
|
|
|
|
if (!meta.isUpcomingChange && module.comingSoon) {
|
|
throw Error(
|
|
`The '${chapter.dashedName}' module is 'comingSoon', but its '${meta.dashedName}' block is not hidden. Set 'isUpcomingChange' to 'true' in the 'meta.json' for the block to hide it.`
|
|
);
|
|
}
|
|
|
|
challenge.module = module.dashedName;
|
|
}
|
|
}
|
|
const challengeOrder = findIndex(
|
|
meta.challengeOrder,
|
|
({ id }) => id === challenge.id
|
|
);
|
|
|
|
const isLastChallengeInBlock =
|
|
meta.challengeOrder.length - 1 === challengeOrder;
|
|
|
|
const isObjectIdFilename = /\/[a-z0-9]{24}\.md$/.test(englishPath);
|
|
if (isObjectIdFilename) {
|
|
const filename = englishPath.split('/').pop();
|
|
if (filename !== `${challenge.id}.md`) {
|
|
throw Error(
|
|
`Filename matches MongoDB ObjectID pattern, but ${filename} does not match challenge id ${challenge.id}`
|
|
);
|
|
}
|
|
}
|
|
|
|
challenge.block = meta.dashedName;
|
|
challenge.blockType = meta.blockType;
|
|
challenge.blockLayout = meta.blockLayout;
|
|
challenge.hasEditableBoundaries = !!meta.hasEditableBoundaries;
|
|
challenge.order = chapterBasedSuperBlocks.includes(meta.superBlock)
|
|
? getBlockOrder(meta.dashedName, fullStackSuperBlockStructure)
|
|
: meta.order;
|
|
|
|
if (!challenge.description) challenge.description = '';
|
|
if (!challenge.instructions) challenge.instructions = '';
|
|
if (!challenge.questions) challenge.questions = [];
|
|
|
|
// const superOrder = getSuperOrder(meta.superBlock);
|
|
// NOTE: Use this version when a super block is in beta.
|
|
const superOrder = getSuperOrder(meta.superBlock);
|
|
if (superOrder !== null) challenge.superOrder = superOrder;
|
|
/* Since there can be more than one way to complete a certification (using the
|
|
legacy curriculum or the new one, for instance), we need a certification
|
|
field to track which certification this belongs to. */
|
|
const dupeCertifications = [
|
|
{
|
|
certification: 'responsive-web-design',
|
|
dupe: '2022/responsive-web-design'
|
|
}
|
|
];
|
|
const hasDupe = dupeCertifications.find(
|
|
cert => cert.dupe === meta.superBlock
|
|
);
|
|
challenge.certification = hasDupe ? hasDupe.certification : meta.superBlock;
|
|
challenge.superBlock = meta.superBlock;
|
|
challenge.challengeOrder = challengeOrder;
|
|
challenge.isLastChallengeInBlock = isLastChallengeInBlock;
|
|
challenge.isPrivate = challenge.isPrivate || meta.isPrivate;
|
|
challenge.required = (meta.required || []).concat(challenge.required || []);
|
|
challenge.template = meta.template;
|
|
challenge.helpCategory = challenge.helpCategory || meta.helpCategory;
|
|
challenge.usesMultifileEditor = !!meta.usesMultifileEditor;
|
|
challenge.disableLoopProtectTests = !!meta.disableLoopProtectTests;
|
|
challenge.disableLoopProtectPreview = !!meta.disableLoopProtectPreview;
|
|
|
|
addChapterAndModuleToChallenge(challenge);
|
|
}
|
|
|
|
function fixChallengeProperties(challenge) {
|
|
if (challenge.challengeFiles) {
|
|
// The client expects the challengeFiles to be an array of polyvinyls
|
|
challenge.challengeFiles = challengeFilesToPolys(
|
|
challenge.challengeFiles
|
|
);
|
|
}
|
|
if (challenge.solutions?.length) {
|
|
// The test runner needs the solutions to be arrays of polyvinyls so it
|
|
// can sort them correctly.
|
|
challenge.solutions = challenge.solutions.map(challengeFilesToPolys);
|
|
}
|
|
}
|
|
|
|
async function createChallenge(filePath, maybeMeta) {
|
|
const meta = maybeMeta
|
|
? maybeMeta
|
|
: require(
|
|
path.resolve(META_DIR, `${getBlockNameFromPath(filePath)}/meta.json`)
|
|
);
|
|
|
|
const isAudited = isAuditedSuperBlock(lang, meta.superBlock);
|
|
|
|
// If we can use the language, do so. Otherwise, default to english.
|
|
const langUsed = isAudited && fs.existsSync(i18nPath) ? lang : 'english';
|
|
|
|
const challenge = translateCommentsInChallenge(
|
|
await parseMD(langUsed === 'english' ? englishPath : i18nPath),
|
|
langUsed,
|
|
COMMENT_TRANSLATIONS
|
|
);
|
|
challenge.translationPending = lang !== 'english' && !isAudited;
|
|
addMetaToChallenge(challenge, meta);
|
|
fixChallengeProperties(challenge);
|
|
|
|
return challenge;
|
|
}
|
|
return createChallenge;
|
|
}
|
|
|
|
function challengeFilesToPolys(files) {
|
|
return files.reduce((challengeFiles, challengeFile) => {
|
|
return [
|
|
...challengeFiles,
|
|
{
|
|
...createPoly(challengeFile),
|
|
seed: challengeFile.contents.slice(0)
|
|
}
|
|
];
|
|
}, []);
|
|
}
|
|
|
|
async function assertHasEnglishSource(filePath, lang, englishPath) {
|
|
const missingEnglish =
|
|
lang !== 'english' &&
|
|
!(await hasEnglishSource(ENGLISH_CHALLENGES_DIR, filePath));
|
|
if (missingEnglish)
|
|
throw Error(`Missing English challenge for
|
|
${filePath}
|
|
It should be in
|
|
${englishPath}
|
|
`);
|
|
}
|
|
|
|
async function hasEnglishSource(basePath, translationPath) {
|
|
const englishRoot = path.resolve(__dirname, basePath, 'english');
|
|
return await access(
|
|
path.join(englishRoot, translationPath),
|
|
fs.constants.F_OK
|
|
)
|
|
.then(() => true)
|
|
.catch(() => false);
|
|
}
|
|
|
|
function getBaseDir(filePath) {
|
|
const [baseDir] = filePath.split(path.sep);
|
|
return baseDir;
|
|
}
|
|
|
|
function getBlockNameFromPath(filePath) {
|
|
const [, block] = filePath.split(path.sep);
|
|
return block;
|
|
}
|
|
|
|
exports.hasEnglishSource = hasEnglishSource;
|
|
exports.generateChallengeCreator = generateChallengeCreator;
|