refactor py-config to use json (#754)

* refactor py-config to use toml

* switch from toml to json and add unit tests

* fix test for py-config

* fix integration test

* use flat structure for JSON

* allow merging configs

* replace arrays instead of concatenating them

* remove extra keys from inline config of integration test

* simplify array replacement logic

* allow config from src to be partial as well

* add comments to unit tests

* add unit test for setting config from both inline and src

* handle parse errors + validate config supplied

* separate functions for src and inline

* suggested improvements

* show error message in red on parser error

* fix eslint

* use resolveJsonModule as true

* use default config defined as a variable without import

* remove disable eslint comment

* remove import for covfefe.json as well

* metadata injection

* add support for schema + extra keys

* use schema_version

pyscriptjs/src/components/pyconfig.ts

@@ -1,9 +1,21 @@
-import * as jsyaml from 'js-yaml';
 import { BaseEvalElement } from './base';
-import { appConfig } from '../stores';
+import { appConfig, addInitializer, runtimeLoaded } from '../stores';
 import type { AppConfig, Runtime } from '../runtime';
-import { PyodideRuntime, DEFAULT_RUNTIME_CONFIG } from '../pyodide';
+import { version } from '../runtime';
+import { PyodideRuntime } from '../pyodide';
 import { getLogger } from '../logger';
+import { readTextFromPath, handleFetchError, mergeConfig, validateConfig, defaultConfig } from '../utils'
+
+// Subscriber used to connect to the first available runtime (can be pyodide or others)
+let runtimeSpec: Runtime;
+runtimeLoaded.subscribe(value => {
+    runtimeSpec = value;
+});
+
+let appConfig_: AppConfig;
+appConfig.subscribe(value => {
+    appConfig_ = value;
+});
 
 const logger = getLogger('py-config');
 
@@ -27,25 +39,47 @@ export class PyConfig extends BaseEvalElement {
         super();
     }
 
-    connectedCallback() {
-        this.code = this.innerHTML;
-        this.innerHTML = '';
-
-        const loadedValues = jsyaml.load(this.code);
-        if (loadedValues === undefined) {
-            this.values = {
-                autoclose_loader: true,
-                runtimes: [DEFAULT_RUNTIME_CONFIG]
-            };
-        } else {
-            // eslint-disable-next-line
-            // @ts-ignore
-            this.values = loadedValues;
+    extractFromSrc() {
+        if (this.hasAttribute('src'))
+        {
+            logger.info('config set from src attribute');
+            return validateConfig(readTextFromPath(this.getAttribute('src')));
         }
+        return {};
+    }
+
+    extractFromInline() {
+        if (this.innerHTML!=='')
+        {
+            this.code = this.innerHTML;
+            this.innerHTML = '';
+            logger.info('config set from inline');
+            return validateConfig(this.code);
+        }
+        return {};
+    }
+
+    injectMetadata() {
+        this.values.pyscript = {
+            "version": version,
+            "time": new Date().toISOString()
+        };
+    }
+
+    connectedCallback() {
+        let srcConfig = this.extractFromSrc();
+        const inlineConfig = this.extractFromInline();
+        // first make config from src whole if it is partial
+        srcConfig = mergeConfig(srcConfig, defaultConfig);
+        // then merge inline config and config from src
+        this.values = mergeConfig(inlineConfig, srcConfig);
+        this.injectMetadata();
 
         appConfig.set(this.values);
         logger.info('config set:', this.values);
 
+        addInitializer(this.loadPackages);
+        addInitializer(this.loadPaths);
         this.loadRuntimes();
     }
 
@@ -59,6 +93,27 @@ export class PyConfig extends BaseEvalElement {
         this.remove();
     }
 
+    loadPackages = async () => {
+        const env = appConfig_.packages;
+        logger.info("Loading env: ", env);
+        await runtimeSpec.installPackage(env);
+    }
+
+    loadPaths = async () => {
+        const paths = appConfig_.paths;
+        logger.info("Paths to load: ", paths)
+        for (const singleFile of paths) {
+            logger.info(`  loading path: ${singleFile}`);
+            try {
+                await runtimeSpec.loadFromFile(singleFile);
+            } catch (e) {
+                //Should we still export full error contents to console?
+                handleFetchError(<Error>e, singleFile);
+            }
+        }
+        logger.info("All paths loaded");
+    }
+
     loadRuntimes() {
         logger.info('Initializing runtimes');
         for (const runtime of this.values.runtimes) {

pyscriptjs/src/components/pyenv.ts

@@ -31,6 +31,7 @@ export class PyEnv extends HTMLElement {
     }
 
     connectedCallback() {
+        logger.info("The <py-env> tag is deprecated, please use <py-config> instead.")
         this.code = this.innerHTML;
         this.innerHTML = '';