jprdonnelly/freeCodeCamp
1
0
Fork 0
mirror of https://github.com/freeCodeCamp/freeCodeCamp.git synced 2026-05-14 10:01:58 -04:00
Code Issues Projects Releases Wiki Activity

feat(client): integrate new test runner (#60318)

Browse Source
This commit is contained in:
Oliver Eyton-Williams
2025-06-12 09:25:37 +02:00
committed by GitHub
parent 18e2f919c2
commit 49fbe88369
23 changed files with 922 additions and 1393 deletions
Download Patch File Download Diff File Expand all files Collapse all files

110
tools/client-plugins/browser-scripts/frame-runner.ts
View File

@@ -1,110 +0,0 @@
import jQuery from 'jquery';
import * as helpers from '@freecodecamp/curriculum-helpers';
import type { FrameDocument, FrameWindow, InitTestFrameArg } from '.';
(window as FrameWindow).$ = jQuery;
const frameDocument = document as FrameDocument;
frameDocument.__initTestFrame = initTestFrame;
async function initTestFrame(e: InitTestFrameArg = { code: {} }) {
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const code = (e.code.contents || '').slice();
const editableContents = (e.code.editableContents || '').slice();
// __testEditable allows test authors to run tests against a transitory dom
// element built using only the code in the editable region.
const __testEditable = (cb: () => () => unknown) => {
const div = frameDocument.createElement('div');
div.id = 'editable-only';
div.innerHTML = editableContents;
frameDocument.body.appendChild(div);
const out = cb();
frameDocument.body.removeChild(div);
return out;
};
/* eslint-disable @typescript-eslint/no-unused-vars */
// Hardcode Deep Freeze dependency
const DeepFreeze = (o: Record<string, unknown>) => {
Object.freeze(o);
Object.getOwnPropertyNames(o).forEach(function (prop) {
if (
Object.prototype.hasOwnProperty.call(o, prop) &&
o[prop] !== null &&
(typeof o[prop] === 'object' || typeof o[prop] === 'function') &&
!Object.isFrozen(o[prop])
) {
DeepFreeze(o[prop] as Record<string, unknown>);
}
});
return o;
};
const { default: chai } = await import(/* webpackChunkName: "chai" */ 'chai');
const assert = chai.assert;
const __helpers = helpers;
const __checkForBrowserExtensions = true;
/* eslint-enable @typescript-eslint/no-unused-vars */
let Enzyme;
if (e.loadEnzyme) {
/* eslint-disable prefer-const */
let Adapter16;
[{ default: Enzyme }, { default: Adapter16 }] = await Promise.all([
import(/* webpackChunkName: "enzyme" */ 'enzyme'),
import(/* webpackChunkName: "enzyme-adapter" */ 'enzyme-adapter-react-16')
]);
Enzyme.configure({ adapter: new Adapter16() });
/* eslint-enable prefer-const */
}
frameDocument.__runTest = async function runTests(testString: string) {
// uncomment the following line to inspect
// the frame-runner as it runs tests
// make sure the dev tools console is open
// debugger;
try {
// eval test string to actual JavaScript
// This return can be a function
// i.e. function() { assert(true, 'happy coding'); }
const testPromise = new Promise((resolve, reject) =>
// To avoid race conditions, we have to run the test in a final
// frameDocument ready:
$(() => {
try {
const test: unknown = eval(testString);
resolve(test);
} catch (err) {
reject(err as Error);
}
})
);
const test = await testPromise;
if (typeof test === 'function') {
// eslint-disable-next-line @typescript-eslint/no-unsafe-call
await test();
}
return { pass: true };
} catch (err) {
if (!(err instanceof chai.AssertionError)) {
console.error(err);
}
// to provide useful debugging information when debugging the tests, we
// have to extract the message, stack and, if they exist, expected and
// actual before returning
return {
err: {
message: (err as Error).message,
stack: (err as Error).stack,
expected: (err as { expected?: string }).expected,
actual: (err as { actual?: string }).actual
}
};
}
};
}

10
tools/client-plugins/browser-scripts/package.json
View File

@@ -29,20 +29,11 @@
"@babel/plugin-transform-runtime": "7.23.7",
"@babel/preset-env": "7.23.7",
"@babel/preset-typescript": "7.23.3",
"@freecodecamp/curriculum-helpers": "4.1.0",
"@types/chai": "4.3.12",
"@types/copy-webpack-plugin": "^8.0.1",
"@types/enzyme": "3.10.16",
"@types/enzyme-adapter-react-16": "1.0.9",
"@types/jquery": "3.5.29",
"@types/lodash-es": "4.17.12",
"@typescript/vfs": "^1.6.0",
"babel-loader": "8.3.0",
"chai": "4.4.1",
"copy-webpack-plugin": "9.1.0",
"enzyme": "3.11.0",
"enzyme-adapter-react-16": "1.15.8",
"jquery": "3.7.1",
"lodash-es": "4.17.21",
"process": "0.11.10",
"pyodide": "^0.23.3",
@@ -52,6 +43,7 @@
"webpack-cli": "4.10.0"
},
"dependencies": {
"@freecodecamp/curriculum-helpers": "^4.4.0",
"react": "16",
"react-dom": "16",
"xterm": "^5.2.1"

192
tools/client-plugins/browser-scripts/python-test-evaluator.ts
View File

@@ -1,192 +0,0 @@
// We have to specify pyodide.js because we need to import that file (not .mjs)
// and 'import' defaults to .mjs
import { loadPyodide, type PyodideInterface } from 'pyodide/pyodide.js';
import type { PyProxy, PythonError } from 'pyodide/ffi';
import pkg from 'pyodide/package.json';
import * as helpers from '@freecodecamp/curriculum-helpers';
import chai from 'chai';
const ctx: Worker & typeof globalThis = self as unknown as Worker &
typeof globalThis;
let pyodide: PyodideInterface;
interface PythonRunEvent extends MessageEvent {
data: {
code: {
contents: string;
editableContents: string;
};
firstTest: unknown;
testString: string;
build: string;
sources: {
[fileName: string]: unknown;
};
};
}
type EvaluatedTeststring = {
input: string[];
test: () => Promise<unknown>;
};
async function setupPyodide() {
if (pyodide) return pyodide;
pyodide = await loadPyodide({
// TODO: host this ourselves
indexURL: `https://cdn.jsdelivr.net/pyodide/v${pkg.version}/full/`
});
// We freeze this to prevent learners from getting the worker into a
// weird state. NOTE: this has to come after pyodide is loaded, because
// pyodide modifies self while loading.
Object.freeze(self);
// eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
pyodide.FS.writeFile(
'/home/pyodide/ast_helpers.py',
helpers.python.astHelpers,
{
encoding: 'utf8'
}
);
ctx.postMessage({ type: 'contentLoaded' });
return pyodide;
}
void setupPyodide();
ctx.onmessage = async (e: PythonRunEvent) => {
const pyodide = await setupPyodide();
/* eslint-disable @typescript-eslint/no-unused-vars */
const code = (e.data.code.contents || '').slice();
const editableContents = (e.data.code.editableContents || '').slice();
const testString = e.data.testString;
const assert = chai.assert;
const __helpers = helpers;
// Create fresh globals for each test
// eslint-disable-next-line @typescript-eslint/no-unsafe-call
const __userGlobals = pyodide.globals.get('dict')() as PyProxy;
/* eslint-enable @typescript-eslint/no-unused-vars */
// uncomment the following line to inspect
// the frame-runner as it runs tests
// make sure the dev tools console is open
// debugger;
try {
// eval test string to get the dummy input and actual test
const evaluatedTestString = await new Promise<unknown>(
(resolve, reject) => {
try {
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
const test: { input: string[]; test: () => Promise<unknown> } =
eval(testString);
resolve(test);
} catch (err) {
reject(err as Error);
}
}
);
// If the test string does not evaluate to an object, then we assume that
// it's a standard JS test and any assertions have already passed.
if (typeof evaluatedTestString !== 'object') {
ctx.postMessage({ pass: true });
return;
}
if (!evaluatedTestString || !('test' in evaluatedTestString)) {
throw new Error(
'Test string did not evaluate to an object with the test property'
);
}
const { input, test } = evaluatedTestString as EvaluatedTeststring;
// Some tests rely on __name__ being set to __main__ and we new dicts do not
// have this set by default.
// eslint-disable-next-line @typescript-eslint/no-unsafe-call
__userGlobals.set('__name__', '__main__');
// The runPython helper is a shortcut for running python code with our
// custom globals.
const runPython = (pyCode: string) =>
pyodide.runPython(pyCode, { globals: __userGlobals }) as unknown;
runPython(`
def __inputGen(xs):
def gen():
for x in xs:
yield x
iter = gen()
def input(arg=None):
return next(iter)
return input
input = __inputGen(${JSON.stringify(input ?? [])})
`);
runPython(`from ast_helpers import Node as _Node`);
// The tests need the user's code as a string, so we write it to the virtual
// filesystem...
// eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
pyodide.FS.writeFile('/user_code.py', code, { encoding: 'utf8' });
// ...and then read it back into a variable so that they can evaluate it.
runPython(`
with open("/user_code.py", "r") as f:
_code = f.read()
`);
try {
// Evaluates the learner's code so that any variables they define are
// available to the test.
runPython(code);
} catch (e) {
const err = e as PythonError;
// Quite a lot of lessons can easily lead users to write code that has
// indentation errors. In these cases we want to provide a more helpful
// error message. For other errors, we can just provide the standard
// message.
const errorType =
err.type === 'IndentationError' ? 'indentation' : 'other';
return ctx.postMessage({
err: {
message: err.message,
stack: err.stack,
errorType
}
});
}
await test();
ctx.postMessage({ pass: true });
} catch (err) {
if (!(err instanceof chai.AssertionError)) {
console.error(err);
}
// to provide useful debugging information when debugging the tests, we
// have to extract the message, stack and, if they exist, expected and
// actual before returning
ctx.postMessage({
err: {
message: (err as Error).message,
stack: (err as Error).stack,
expected: (err as { expected?: string }).expected,
actual: (err as { actual?: string }).actual
}
});
} finally {
__userGlobals.destroy();
}
};

163
tools/client-plugins/browser-scripts/test-evaluator.ts
View File

@@ -1,163 +0,0 @@
import chai from 'chai';
import { toString as __toString } from 'lodash-es';
import * as curriculumHelpers from '@freecodecamp/curriculum-helpers';
import { format as __format } from './utils/format';
const ctx: Worker & typeof globalThis = self as unknown as Worker &
typeof globalThis;
const __utils = (() => {
const MAX_LOGS_SIZE = 64 * 1024;
let logs: string[] = [];
function flushLogs() {
if (logs.length) {
ctx.postMessage({
type: 'LOG',
data: logs.join('\n')
});
logs = [];
}
}
function pushLogs(logs: string[], args: string[]) {
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
logs.push(args.map(arg => __format(arg)).join(' '));
if (logs.join('\n').length > MAX_LOGS_SIZE) {
flushLogs();
}
}
const oldLog = ctx.console.log.bind(ctx.console);
function proxyLog(...args: string[]) {
pushLogs(logs, args);
return oldLog(...args);
}
const oldInfo = ctx.console.info.bind(ctx.console);
function proxyInfo(...args: string[]) {
pushLogs(logs, args);
return oldInfo(...args);
}
const oldWarn = ctx.console.warn.bind(ctx.console);
function proxyWarn(...args: string[]) {
pushLogs(logs, args);
return oldWarn(...args);
}
const oldError = ctx.console.error.bind(ctx.console);
function proxyError(...args: string[]) {
pushLogs(logs, args);
return oldError(...args);
}
function log(...msgs: Error[]) {
if (msgs && msgs[0] && !(msgs[0] instanceof chai.AssertionError)) {
// discards the stack trace via toString as it only useful to debug the
// site, not a specific challenge.
console.log(...msgs.map(msg => msg.toString()));
}
}
const toggleProxyLogger = (on: unknown) => {
ctx.console.log = on ? proxyLog : oldLog;
ctx.console.info = on ? proxyInfo : oldInfo;
ctx.console.warn = on ? proxyWarn : oldWarn;
ctx.console.error = on ? proxyError : oldError;
};
return {
log,
toggleProxyLogger,
flushLogs
};
})();
// We can't simply import these because of how webpack names them when building
// the bundle. Since both assert and __helpers have to exist in the global
// scope, we have to declare them.
const assert = chai.assert;
const __helpers = curriculumHelpers;
// We freeze to prevent learners from getting the tester into a weird
// state by modifying these objects.
Object.freeze(self);
Object.freeze(__utils);
Object.freeze(assert);
Object.freeze(__helpers);
interface TestEvaluatorEvent extends MessageEvent {
data: {
code: {
contents: string;
editableContents: string;
};
firstTest: unknown;
testString: string;
build: string;
};
}
/* Run the test if there is one. If not just evaluate the user code */
ctx.onmessage = async (e: TestEvaluatorEvent) => {
/* eslint-disable @typescript-eslint/no-unused-vars */
const code = e.data?.code?.contents || '';
const editableContents = e.data?.code?.editableContents || '';
// Build errors should be reported, but only once:
__utils.toggleProxyLogger(e.data.firstTest);
/* eslint-enable @typescript-eslint/no-unused-vars */
try {
// This can be reassigned by the eval inside the try block, so it should be declared as a let
// eslint-disable-next-line prefer-const
let __userCodeWasExecuted = false;
try {
// Logging is proxyed after the build to catch console.log messages
// generated during testing.
await eval(`${e.data.build}
__utils.flushLogs();
__userCodeWasExecuted = true;
__utils.toggleProxyLogger(true);
(async () => {${e.data.testString}})()`);
} catch (err) {
if (__userCodeWasExecuted) {
// rethrow error, since test failed.
throw err;
}
// log build errors unless they're related to import/export/require (there
// are challenges that use them and they should not trigger warnings)
if (
(err as Error).name !== 'ReferenceError' ||
((err as Error).message !== 'require is not defined' &&
(err as Error).message !== 'exports is not defined')
) {
__utils.log(err as Error);
}
// the tests may not require working code, so they are evaluated even if
// the user code does not get executed.
eval(e.data.testString);
}
__utils.flushLogs();
ctx.postMessage({ pass: true });
} catch (err) {
// Errors from testing go to the browser console only.
__utils.toggleProxyLogger(false);
// Report execution errors in case user code has errors that are only
// uncovered during testing.
__utils.log(err as Error);
// Now that all logs have been created we can flush them.
__utils.flushLogs();
ctx.postMessage({
err: {
message: (err as Error).message,
stack: (err as Error).stack,
expected: (err as { expected?: string }).expected,
actual: (err as { actual?: string }).actual
}
});
}
};
ctx.postMessage({ type: 'contentLoaded' });

3
tools/client-plugins/browser-scripts/test-runner.ts Normal file
View File

@@ -0,0 +1,3 @@
export type { FCCTestRunner } from '@freecodecamp/curriculum-helpers/test-runner.js';
export { version } from '@freecodecamp/curriculum-helpers/package.json';

12
tools/client-plugins/browser-scripts/webpack.config.cjs
View File

@@ -2,6 +2,9 @@ const { writeFileSync } = require('fs');
const path = require('path');
const CopyWebpackPlugin = require('copy-webpack-plugin');
const webpack = require('webpack');
const {
version: helperVersion
} = require('@freecodecamp/curriculum-helpers/package.json');
module.exports = (env = {}) => {
const __DEV__ = env.production !== true;
@@ -14,11 +17,8 @@ module.exports = (env = {}) => {
cache: __DEV__ ? { type: 'filesystem' } : false,
mode: __DEV__ ? 'development' : 'production',
entry: {
'frame-runner': './frame-runner.ts',
'sass-compile': './sass-compile.ts',
'test-evaluator': './test-evaluator.ts',
'python-worker': './python-worker.ts',
'python-test-evaluator': './python-test-evaluator.ts',
'typescript-worker': './typescript-worker.ts'
},
devtool: __DEV__ ? 'inline-source-map' : 'source-map',
@@ -71,7 +71,11 @@ module.exports = (env = {}) => {
patterns: [
'./node_modules/sass.js/dist/sass.sync.js',
// TODO: copy this into the css folder, not the js folder
'./node_modules/xterm/css/xterm.css'
'./node_modules/xterm/css/xterm.css',
{
from: './node_modules/@freecodecamp/curriculum-helpers/dist/test-runner',
to: `test-runner/${helperVersion}/`
}
]
}),
new webpack.ProvidePlugin({