refactor py-config to use json (#754)

* refactor py-config to use toml * switch from toml to json and add unit tests * fix test for py-config * fix integration test * use flat structure for JSON * allow merging configs * replace arrays instead of concatenating them * remove extra keys from inline config of integration test * simplify array replacement logic * allow config from src to be partial as well * add comments to unit tests * add unit test for setting config from both inline and src * handle parse errors + validate config supplied * separate functions for src and inline * suggested improvements * show error message in red on parser error * fix eslint * use resolveJsonModule as true * use default config defined as a variable without import * remove disable eslint comment * remove import for covfefe.json as well * metadata injection * add support for schema + extra keys * use schema_version
2025-12-19 18:27:29 -05:00 · 2022-09-16 02:07:00 +05:30
parent 0b014eea56
commit 4841e29fc6
8 changed files with 330 additions and 46 deletions
--- a/pyscriptjs/src/components/pyconfig.ts
+++ b/pyscriptjs/src/components/pyconfig.ts
@@ -1,9 +1,21 @@
-import * as jsyaml from 'js-yaml';
 import { BaseEvalElement } from './base';
-import { appConfig } from '../stores';
+import { appConfig, addInitializer, runtimeLoaded } from '../stores';
 import type { AppConfig, Runtime } from '../runtime';
-import { PyodideRuntime, DEFAULT_RUNTIME_CONFIG } from '../pyodide';
+import { version } from '../runtime';
+import { PyodideRuntime } from '../pyodide';
 import { getLogger } from '../logger';
+import { readTextFromPath, handleFetchError, mergeConfig, validateConfig, defaultConfig } from '../utils'
+
+// Subscriber used to connect to the first available runtime (can be pyodide or others)
+let runtimeSpec: Runtime;
+runtimeLoaded.subscribe(value => {
+    runtimeSpec = value;
+});
+
+let appConfig_: AppConfig;
+appConfig.subscribe(value => {
+    appConfig_ = value;
+});

 const logger = getLogger('py-config');

@@ -27,25 +39,47 @@ export class PyConfig extends BaseEvalElement {
        super();
    }

-    connectedCallback() {
-        this.code = this.innerHTML;
-        this.innerHTML = '';
-
-        const loadedValues = jsyaml.load(this.code);
-        if (loadedValues === undefined) {
-            this.values = {
-                autoclose_loader: true,
-                runtimes: [DEFAULT_RUNTIME_CONFIG]
-            };
-        } else {
-            // eslint-disable-next-line
-            // @ts-ignore
-            this.values = loadedValues;
+    extractFromSrc() {
+        if (this.hasAttribute('src'))
+        {
+            logger.info('config set from src attribute');
+            return validateConfig(readTextFromPath(this.getAttribute('src')));
        }
+        return {};
+    }
+
+    extractFromInline() {
+        if (this.innerHTML!=='')
+        {
+            this.code = this.innerHTML;
+            this.innerHTML = '';
+            logger.info('config set from inline');
+            return validateConfig(this.code);
+        }
+        return {};
+    }
+
+    injectMetadata() {
+        this.values.pyscript = {
+            "version": version,
+            "time": new Date().toISOString()
+        };
+    }
+
+    connectedCallback() {
+        let srcConfig = this.extractFromSrc();
+        const inlineConfig = this.extractFromInline();
+        // first make config from src whole if it is partial
+        srcConfig = mergeConfig(srcConfig, defaultConfig);
+        // then merge inline config and config from src
+        this.values = mergeConfig(inlineConfig, srcConfig);
+        this.injectMetadata();

        appConfig.set(this.values);
        logger.info('config set:', this.values);

+        addInitializer(this.loadPackages);
+        addInitializer(this.loadPaths);
        this.loadRuntimes();
    }

@@ -59,6 +93,27 @@ export class PyConfig extends BaseEvalElement {
        this.remove();
    }

+    loadPackages = async () => {
+        const env = appConfig_.packages;
+        logger.info("Loading env: ", env);
+        await runtimeSpec.installPackage(env);
+    }
+
+    loadPaths = async () => {
+        const paths = appConfig_.paths;
+        logger.info("Paths to load: ", paths)
+        for (const singleFile of paths) {
+            logger.info(`  loading path: ${singleFile}`);
+            try {
+                await runtimeSpec.loadFromFile(singleFile);
+            } catch (e) {
+                //Should we still export full error contents to console?
+                handleFetchError(<Error>e, singleFile);
+            }
+        }
+        logger.info("All paths loaded");
+    }
+
    loadRuntimes() {
        logger.info('Initializing runtimes');
        for (const runtime of this.values.runtimes) {
--- a/pyscriptjs/src/components/pyenv.ts
+++ b/pyscriptjs/src/components/pyenv.ts
@@ -31,6 +31,7 @@ export class PyEnv extends HTMLElement {
    }

    connectedCallback() {
+        logger.info("The <py-env> tag is deprecated, please use <py-config> instead.")
        this.code = this.innerHTML;
        this.innerHTML = '';

--- a/pyscriptjs/src/pyodide.ts
+++ b/pyscriptjs/src/pyodide.ts
@@ -8,12 +8,6 @@ import pyscript from './python/pyscript.py';

 const logger = getLogger('pyscript/pyodide');

-export const DEFAULT_RUNTIME_CONFIG: RuntimeConfig = {
-    src: 'https://cdn.jsdelivr.net/pyodide/v0.21.2/full/pyodide.js',
-    name: 'pyodide-default',
-    lang: 'python'
-};
-
 export class PyodideRuntime extends Runtime {
    src: string;
    name?: string;
@@ -22,9 +16,9 @@ export class PyodideRuntime extends Runtime {
    globals: any;

    constructor(
-        src = DEFAULT_RUNTIME_CONFIG.src,
-        name = DEFAULT_RUNTIME_CONFIG.name,
-        lang = DEFAULT_RUNTIME_CONFIG.lang,
+        src = 'https://cdn.jsdelivr.net/pyodide/v0.21.2/full/pyodide.js',
+        name = 'pyodide-default',
+        lang = 'python',
    ) {
        logger.info('Runtime config:', { name, lang, src });
        super();
--- a/pyscriptjs/src/runtime.ts
+++ b/pyscriptjs/src/runtime.ts
@@ -16,21 +16,37 @@ import { getLogger } from './logger';

 const logger = getLogger('pyscript/runtime');

+export const version = "<<VERSION>>";
 export type RuntimeInterpreter = PyodideInterface | null;

+export interface AppConfig extends Record<string, any> {
+    name?: string;
+    description?: string;
+    version?: string;
+    schema_version?: number;
+    type?: string;
+    author_name?: string;
+    author_email?: string;
+    license?: string;
+    autoclose_loader?: boolean;
+    runtimes?: Array<RuntimeConfig>;
+    packages?: Array<string>;
+    paths?: Array<string>;
+    plugins?: Array<string>;
+    pyscript?: PyScriptMetadata;
+}
+
+export type PyScriptMetadata = {
+    version?: string;
+    time?: string;
+}
+
 export type RuntimeConfig = {
-    src: string;
+    src?: string;
    name?: string;
    lang?: string;
 };

-export type AppConfig = {
-    autoclose_loader: boolean;
-    name?: string;
-    version?: string;
-    runtimes?: Array<RuntimeConfig>;
-};
-
 let loader: PyLoader | undefined;
 globalLoader.subscribe(value => {
    loader = value;
@@ -52,7 +68,7 @@ scriptsQueue.subscribe((value: PyScript[]) => {
 });

 let appConfig_: AppConfig = {
-    autoclose_loader: true,
+    autoclose_loader: true
 };

 appConfig.subscribe((value: AppConfig) => {
--- a/pyscriptjs/src/stores.ts
+++ b/pyscriptjs/src/stores.ts
@@ -1,7 +1,7 @@
 import { writable } from 'svelte/store';
 import type { PyLoader } from './components/pyloader';
 import type { PyScript } from './components/pyscript';
-import type { Runtime } from './runtime';
+import type { Runtime, AppConfig } from './runtime';
 import { getLogger } from './logger';

 export type Initializer = () => Promise<void>;
@@ -29,7 +29,7 @@ export const scriptsQueue = writable<PyScript[]>([]);
 export const initializers = writable<Initializer[]>([]);
 export const postInitializers = writable<Initializer[]>([]);
 export const globalLoader = writable<PyLoader | undefined>();
-export const appConfig = writable();
+export const appConfig = writable<AppConfig>();

 export const addToScriptsQueue = (script: PyScript) => {
    scriptsQueue.update(scriptsQueue => [...scriptsQueue, script]);
--- a/pyscriptjs/src/utils.ts
+++ b/pyscriptjs/src/utils.ts
@@ -1,3 +1,33 @@
+import type { AppConfig } from "./runtime";
+
+const allKeys = {
+    "string": ["name", "description", "version", "type", "author_name", "author_email", "license"],
+    "number": ["schema_version"],
+    "boolean": ["autoclose_loader"],
+    "array": ["runtimes", "packages", "paths", "plugins"]
+};
+
+const defaultConfig: AppConfig = {
+    "name": "pyscript",
+    "description": "default config",
+    "version": "0.1",
+    "schema_version": 1,
+    "type": "app",
+    "author_name": "anonymous coder",
+    "author_email": "foo@bar.com",
+    "license": "Apache",
+    "autoclose_loader": true,
+    "runtimes": [{
+        "src": "https://cdn.jsdelivr.net/pyodide/v0.21.2/full/pyodide.js",
+        "name": "pyodide-0.21.2",
+        "lang": "python"
+    }],
+    "packages": [],
+    "paths": [],
+    "plugins": []
+}
+
+
 function addClasses(element: HTMLElement, classes: Array<string>) {
    for (const entry of classes) {
        element.classList.add(entry);
@@ -84,4 +114,124 @@ function handleFetchError(e: Error, singleFile: string) {
    showError(errorContent);
 }

-export { addClasses, removeClasses, getLastPath, ltrim, htmlDecode, guidGenerator, showError, handleFetchError };
+function readTextFromPath(path: string) {
+    const request = new XMLHttpRequest();
+    request.open("GET", path, false);
+    request.send();
+    const returnValue = request.responseText;
+
+    return returnValue;
+}
+
+function inJest(): boolean {
+    return typeof process === 'object' && process.env.JEST_WORKER_ID !== undefined;
+}
+
+function fillUserData(inputConfig: AppConfig, resultConfig: AppConfig): AppConfig
+{
+    for (const key in inputConfig)
+    {
+        // fill in all extra keys ignored by the validator
+        if (!(key in defaultConfig))
+        {
+            resultConfig[key] = inputConfig[key];
+        }
+    }
+    return resultConfig;
+}
+
+function mergeConfig(inlineConfig: AppConfig, externalConfig: AppConfig): AppConfig {
+    if (Object.keys(inlineConfig).length === 0 && Object.keys(externalConfig).length === 0)
+    {
+        return defaultConfig;
+    }
+    else if (Object.keys(inlineConfig).length === 0)
+    {
+        return externalConfig;
+    }
+    else if(Object.keys(externalConfig).length === 0)
+    {
+        return inlineConfig;
+    }
+    else
+    {
+        let merged: AppConfig = {};
+
+        for (const keyType in allKeys)
+        {
+            const keys = allKeys[keyType];
+            keys.forEach(function(item: string){
+                if (keyType === "boolean")
+                {
+                    merged[item] = (typeof inlineConfig[item] !== "undefined") ? inlineConfig[item] : externalConfig[item];
+                }
+                else
+                {
+                    merged[item] = inlineConfig[item] || externalConfig[item];
+                }
+            });
+        }
+
+        // fill extra keys from external first
+        // they will be overridden by inline if extra keys also clash
+        merged = fillUserData(externalConfig, merged);
+        merged = fillUserData(inlineConfig, merged);
+
+        return merged;
+    }
+}
+
+function validateConfig(configText: string) {
+    let config: object;
+    try {
+        config = JSON.parse(configText);
+    }
+    catch (err) {
+        const errMessage: string = err.toString();
+        showError(`<p>config supplied: ${configText} is invalid and cannot be parsed: ${errMessage}</p>`);
+    }
+
+    const finalConfig: AppConfig = {}
+
+    for (const keyType in allKeys)
+    {
+        const keys = allKeys[keyType];
+        keys.forEach(function(item: string){
+            if (validateParamInConfig(item, keyType, config))
+            {
+                if (item === "runtimes")
+                {
+                    finalConfig[item] = [];
+                    const runtimes = config[item];
+                    runtimes.forEach(function(eachRuntime: object){
+                        const runtimeConfig: object = {};
+                        for (const eachRuntimeParam in eachRuntime)
+                        {
+                            if (validateParamInConfig(eachRuntimeParam, "string", eachRuntime))
+                            {
+                                runtimeConfig[eachRuntimeParam] = eachRuntime[eachRuntimeParam];
+                            }
+                        }
+                        finalConfig[item].push(runtimeConfig);
+                    });
+                }
+                else
+                {
+                    finalConfig[item] = config[item];
+                }
+            }
+        });
+    }
+
+    return fillUserData(config, finalConfig);
+}
+
+function validateParamInConfig(paramName: string, paramType: string, config: object): boolean {
+    if (paramName in config)
+    {
+        return paramType === "array" ? Array.isArray(config[paramName]) : typeof config[paramName] === paramType;
+    }
+    return false;
+}
+
+export { defaultConfig, addClasses, removeClasses, getLastPath, ltrim, htmlDecode, guidGenerator, showError, handleFetchError, readTextFromPath, inJest, mergeConfig, validateConfig };
--- a/pyscriptjs/tests/integration/test_py_runtime_config.py
+++ b/pyscriptjs/tests/integration/test_py_runtime_config.py
@@ -55,10 +55,13 @@ class TestRuntimeConfig(PyScriptTest):
        """,
            extra_head="""
            <py-config>
-                runtimes:
-                - src: "/pyodide/pyodide.js"
-                  name: pyodide-0.20.0
-                  lang: python
+                {
+                    "runtimes": [{
+                        "src": "/pyodide/pyodide.js",
+                        "name": "pyodide-0.20.0",
+                        "lang": "python"
+                    }]
+                }
            </py-config>
        """,
        )
--- a/pyscriptjs/tests/unit/pyconfig.test.ts
+++ b/pyscriptjs/tests/unit/pyconfig.test.ts
@@ -1,15 +1,32 @@
-import type { AppConfig, RuntimeConfig } from '../../src/runtime';
 import { PyConfig } from '../../src/components/pyconfig';
+// inspired by trump typos
+const covfefeConfig = {
+    "name": "covfefe",
+    "runtimes": [{
+        "src": "/demo/covfefe.js",
+        "name": "covfefe",
+        "lang": "covfefe"
+    }],
+    "wonerful": "discgrace"
+};
+
+import {jest} from '@jest/globals';

 customElements.define('py-config', PyConfig);

 describe('PyConfig', () => {
    let instance: PyConfig;
+
+    const xhrMockClass = () => ({
+        open            : jest.fn(),
+        send            : jest.fn(),
+        responseText    : JSON.stringify(covfefeConfig)
+    });
+    // @ts-ignore
+    window.XMLHttpRequest = jest.fn().mockImplementation(xhrMockClass)
+
    beforeEach(() => {
        instance = new PyConfig();
-        let runtime_config: RuntimeConfig = {src: "/demo/covfefe.js", name: "covfefe", lang: "covfefe"};
-        let app_config: AppConfig = {autoclose_loader: true, runtimes: [runtime_config]};
-        instance.values = app_config;
    });

    it('should get the Config to just instantiate', async () => {
@@ -17,7 +34,55 @@ describe('PyConfig', () => {
    });

    it('should load runtime from config and set as script src', () => {
+        instance.values = covfefeConfig;
        instance.loadRuntimes();
        expect(document.scripts[0].src).toBe("http://localhost/demo/covfefe.js");
    });
+
+    it('should load the default config', ()=> {
+        instance.connectedCallback();
+        expect(instance.values.name).toBe("pyscript");
+        expect(instance.values.author_email).toBe("foo@bar.com");
+        expect(instance.values.pyscript?.time).not.toBeNull();
+        // @ts-ignore
+        expect(instance.values.runtimes[0].lang).toBe("python");
+    });
+
+    it('should load the config from inline', ()=> {
+        instance.innerHTML = JSON.stringify(covfefeConfig);
+        instance.connectedCallback();
+        // @ts-ignore
+        expect(instance.values.runtimes[0].lang).toBe("covfefe");
+        expect(instance.values.pyscript?.time).not.toBeNull();
+        // version wasn't present in `inline config` but is still set due to merging with default
+        expect(instance.values.version).toBe("0.1");
+    });
+
+    it('should load the config from src attribute', ()=> {
+        instance.setAttribute("src", "/covfefe.json");
+        instance.connectedCallback();
+        // @ts-ignore
+        expect(instance.values.runtimes[0].lang).toBe("covfefe");
+        expect(instance.values.pyscript?.time).not.toBeNull();
+        // wonerful is an extra key supplied by the user and is unaffected by merging process
+        expect(instance.values.wonerful).toBe("discgrace");
+        // version wasn't present in `config from src` but is still set due to merging with default
+        expect(instance.values.version).toBe("0.1");
+    });
+
+    it('should load the config from both inline and src', ()=> {
+        instance.innerHTML = JSON.stringify({"version": "0.2a", "wonerful": "highjacked"});
+        instance.setAttribute("src", "/covfefe.json");
+        instance.connectedCallback();
+        // @ts-ignore
+        expect(instance.values.runtimes[0].lang).toBe("covfefe");
+        expect(instance.values.pyscript?.time).not.toBeNull();
+        // config from src had an extra key "wonerful" with value "discgrace"
+        // inline config had the same extra key "wonerful" with value "highjacked"
+        // the merge process works for extra keys that clash as well
+        // so the final value is "highjacked" since inline takes precedence over src
+        expect(instance.values.wonerful).toBe("highjacked");
+        // version wasn't present in `config from src` but is still set due to merging with default and inline
+        expect(instance.values.version).toBe("0.2a");
+    });
 });