Fix #1993 - Expose a handy fetch API (#1994)

* Keep GitHub Actions up to date with GitHub's Dependabot

Fix the warning like at the bottom right of
https://github.com/pyscript/pyscript/actions/runs/8263920642

# https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot
# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -11,7 +11,9 @@ body:
 
         There will always be more issues than there is time to do them, and so we will need to selectively close issues that don't provide enough information, so we can focus our time on helping people like you who fill out the issue form completely. Thank you for your collaboration!
 
-        There are also already a lot of open issues, so please take 2 minutes and search through existing ones to see if what you are experiencing already exists
+        There are also already a lot of open issues, so please take 2 minutes and search through existing ones to see if what you are experiencing already exists.
+
+        Finally, if you are opening **a bug report related to PyScript.com** please [use this repository instead](https://github.com/anaconda/pyscript-dot-com-issues/issues/new/choose).
 
         Thanks for helping PyScript be amazing. We are nothing without people like you helping build a better community 💐!
   - type: checkboxes

.github/dependabot.yml

@@ -0,0 +1,13 @@
+# Keep GitHub Actions up to date with GitHub's Dependabot...
+# https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot
+# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem
+version: 2
+updates:
+    - package-ecosystem: github-actions
+      directory: /
+      groups:
+          github-actions:
+              patterns:
+                  - "*" # Group all Actions updates into a single larger pull request
+      schedule:
+          interval: weekly

.pre-commit-config.yaml

@@ -7,7 +7,7 @@ ci:
 default_stages: [commit]
 repos:
     - repo: https://github.com/pre-commit/pre-commit-hooks
-      rev: v4.4.0
+      rev: v4.5.0
       hooks:
           - id: check-builtin-literals
           - id: check-case-conflict
@@ -25,13 +25,13 @@ repos:
           - id: trailing-whitespace
 
     - repo: https://github.com/psf/black
-      rev: 23.1.0
+      rev: 24.1.1
       hooks:
           - id: black
             exclude: pyscript\.core/src/stdlib/pyscript/__init__\.py
 
     - repo: https://github.com/codespell-project/codespell
-      rev: v2.2.4
+      rev: v2.2.6
       hooks:
           - id: codespell # See 'pyproject.toml' for args
             exclude: \.js\.map$
@@ -46,7 +46,7 @@ repos:
             args: [--tab-width, "4"]
 
     - repo: https://github.com/pycqa/isort
-      rev: 5.12.0
+      rev: 5.13.2
       hooks:
           - id: isort
             name: isort (python)

CONTRIBUTING.md

@@ -79,3 +79,103 @@ The Project abides by the Organization's [trademark policy](https://github.com/p
 
 Part of MVG-0.1-beta.
 Made with love by GitHub. Licensed under the [CC-BY 4.0 License](https://creativecommons.org/licenses/by-sa/4.0/).
+
+# Quick guide to pytest
+
+We make heavy usage of pytest. Here is a quick guide and collection of useful options:
+
+-   To run all tests in the current directory and subdirectories: pytest
+
+-   To run tests in a specific directory or file: pytest path/to/dir/test_foo.py
+
+-   -s: disables output capturing
+
+-   --pdb: in case of exception, enter a (Pdb) prompt so that you can inspect what went wrong.
+
+-   -v: verbose mode
+
+-   -x: stop the execution as soon as one test fails
+
+-   -k foo: run only the tests whose full name contains foo
+
+-   -k 'foo and bar'
+
+-   -k 'foo and not bar'
+
+## Running integration tests under pytest
+
+make test is useful to run all the tests, but during the development is useful to have more control on how tests are run. The following guide assumes that you are in the directory pyscriptjs/tests/integration/.
+
+### To run all the integration tests, single or multi core
+
+$ pytest -xv
+...
+
+test_00_support.py::TestSupport::test_basic[chromium] PASSED [ 0%]
+test_00_support.py::TestSupport::test_console[chromium] PASSED [ 1%]
+test_00_support.py::TestSupport::test_check_js_errors_simple[chromium] PASSED [ 2%]
+test_00_support.py::TestSupport::test_check_js_errors_expected[chromium] PASSED [ 3%]
+test_00_support.py::TestSupport::test_check_js_errors_expected_but_didnt_raise[chromium] PASSED [ 4%]
+test_00_support.py::TestSupport::test_check_js_errors_multiple[chromium] PASSED [ 5%]
+...
+
+-x means "stop at the first failure". -v means "verbose", so that you can see all the test names one by one. We try to keep tests in a reasonable order, from most basic to most complex. This way, if you introduced some bug in very basic things, you will notice immediately.
+
+If you have the pytest-xdist plugin installed, you can run all the integration tests on 4 cores in parallel:
+
+$ pytest -n 4
+
+### To run a single test, headless
+
+$ pytest test_01_basic.py -k test_pyscript_hello -s
+...
+[ 0.00 page.goto ] pyscript_hello.html
+[ 0.01 request ] 200 - fake_server - http://fake_server/pyscript_hello.html
+...
+[ 0.17 console.info ] [py-loader] Downloading pyodide-x.y.z...
+[ 0.18 request ] 200 - CACHED - https://cdn.jsdelivr.net/pyodide/vx.y.z/full/pyodide.js
+...
+[ 3.59 console.info ] [pyscript/main] PyScript page fully initialized
+[ 3.60 console.log ] hello pyscript
+
+-k selects tests by pattern matching as described above. -s instructs pytest to show the output to the terminal instead of capturing it. In the output you can see various useful things, including network requests and JS console messages.
+
+### To run a single test, headed
+
+$ pytest test_01_basic.py -k test_pyscript_hello -s --headed
+...
+
+Same as above, but with --headed the browser is shown in a window, and you can interact with it. The browser uses a fake server, which means that HTTP requests are cached.
+
+Unfortunately, in this mode source maps does not seem to work, and you cannot debug the original typescript source code. This seems to be a bug in playwright, for which we have a workaround:
+
+$ pytest test_01_basic.py -k test_pyscript_hello -s --headed --no-fake-server
+...
+
+As the name implies, -no-fake-server disables the fake server: HTTP requests are not cached, but source-level debugging works.
+
+Finally:
+
+$ pytest test_01_basic.py -k test_pyscript_hello -s --dev
+...
+
+--dev implies --headed --no-fake-server. In addition, it also automatically open chrome dev tools.
+
+### To run only main thread or worker tests
+
+By default, we run each test twice: one with execution_thread = "main" and one with execution_thread = "worker". If you want to run only half of them, you can use -m:
+
+$ pytest -m main # run only the tests in the main thread
+$ pytest -m worker # ron only the tests in the web worker
+
+## Fake server, HTTP cache
+
+By default, our test machinery uses a playwright router which intercepts and cache HTTP requests, so that for example you don't have to download pyodide again and again. This also enables the possibility of running tests in parallel on multiple cores.
+
+The cache is stored using the pytest-cache plugin, which means that it survives across sessions.
+
+If you want to temporarily disable the cache, the easiest thing is to use --no-fake-server, which bypasses it completely.
+
+If you want to clear the cache, you can use the special option --clear-http-cache:
+
+NOTE: this works only if you are inside tests/integration, or if you explicitly specify tests/integration from the command line. This is due to how pytest decides to search for and load the various conftest.py.

LICENSE

@@ -186,7 +186,11 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+
+   Copyright (c) 2022-present, PyScript Development Team
+
+   Originated at Anaconda, Inc. in 2022
+
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

pyscript.core/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@pyscript/core",
-    "version": "0.3.9",
+    "version": "0.4.6",
     "type": "module",
     "description": "PyScript",
     "module": "./index.js",
@@ -42,31 +42,31 @@
     "dependencies": {
         "@ungap/with-resolvers": "^0.1.0",
         "basic-devtools": "^0.1.6",
-        "polyscript": "^0.6.2",
+        "polyscript": "^0.11.0",
         "sticky-module": "^0.1.1",
         "to-json-callback": "^0.1.1",
         "type-checked-collections": "^0.1.7"
     },
     "devDependencies": {
-        "@codemirror/commands": "^6.3.2",
-        "@codemirror/lang-python": "^6.1.3",
-        "@codemirror/language": "^6.9.3",
-        "@codemirror/state": "^6.3.3",
-        "@codemirror/view": "^6.22.1",
-        "@playwright/test": "^1.40.1",
+        "@codemirror/commands": "^6.3.3",
+        "@codemirror/lang-python": "^6.1.4",
+        "@codemirror/language": "^6.10.1",
+        "@codemirror/state": "^6.4.1",
+        "@codemirror/view": "^6.25.1",
+        "@playwright/test": "^1.42.1",
         "@rollup/plugin-commonjs": "^25.0.7",
         "@rollup/plugin-node-resolve": "^15.2.3",
         "@rollup/plugin-terser": "^0.4.4",
         "@webreflection/toml-j0.4": "^1.1.3",
-        "@xterm/addon-fit": "^0.9.0-beta.1",
-        "chokidar": "^3.5.3",
+        "@xterm/addon-fit": "^0.9.0",
+        "chokidar": "^3.6.0",
         "codemirror": "^6.0.1",
-        "eslint": "^8.55.0",
-        "rollup": "^4.6.1",
+        "eslint": "^8.57.0",
+        "rollup": "^4.12.1",
         "rollup-plugin-postcss": "^4.0.2",
         "rollup-plugin-string": "^3.0.0",
         "static-handler": "^0.4.3",
-        "typescript": "^5.3.3",
+        "typescript": "^5.4.2",
         "xterm": "^5.3.0",
         "xterm-readline": "^1.1.1"
     },

pyscript.core/src/config.js

@@ -63,6 +63,9 @@ for (const [TYPE] of TYPES) {
     /** @type {Error | undefined} The error thrown when parsing the PyScript config, if any.*/
     let error;
 
+    /** @type {string | undefined} The `configURL` field to normalize all config operations as opposite of guessing it once resolved */
+    let configURL;
+
     let config,
         type,
         pyElement,
@@ -105,6 +108,7 @@ for (const [TYPE] of TYPES) {
     if (!error && config) {
         try {
             const { json, toml, text, url } = await configDetails(config, type);
+            if (url) configURL = new URL(url, location.href).href;
             config = text;
             if (json || type === "json") {
                 try {
@@ -146,7 +150,7 @@ for (const [TYPE] of TYPES) {
     // assign plugins as Promise.all only if needed
     plugins = Promise.all(toBeAwaited);
 
-    configs.set(TYPE, { config: parsed, plugins, error });
+    configs.set(TYPE, { config: parsed, configURL, plugins, error });
 }
 
 export default configs;

pyscript.core/src/core.js

@@ -26,37 +26,39 @@ import { ErrorCode } from "./exceptions.js";
 import { robustFetch as fetch, getText } from "./fetch.js";
 import { hooks, main, worker, codeFor, createFunction } from "./hooks.js";
 
-// allows lazy element features on code evaluation
-let currentElement;
-
 // generic helper to disambiguate between custom element and script
 const isScript = ({ tagName }) => tagName === "SCRIPT";
 
-let shouldRegister = true;
-const registerModule = ({ XWorker: $XWorker, interpreter, io }) => {
-    // automatically use the pyscript stderr (when/if defined)
-    // this defaults to console.error
-    function PyWorker(...args) {
-        const worker = $XWorker(...args);
-        worker.onerror = ({ error }) => io.stderr(error);
-        return worker;
-    }
-
-    // enrich the Python env with some JS utility for main
-    interpreter.registerJsModule("_pyscript", {
-        PyWorker,
-        get target() {
-            return isScript(currentElement)
-                ? currentElement.target.id
-                : currentElement.id;
+// Used to create either Pyodide or MicroPython workers
+// with the PyScript module available within the code
+const [PyWorker, MPWorker] = [...TYPES.entries()].map(
+    ([TYPE, interpreter]) =>
+        /**
+         * A `Worker` facade able to bootstrap on the worker thread only a PyScript module.
+         * @param {string} file the python file to run ina worker.
+         * @param {{config?: string | object, async?: boolean}} [options] optional configuration for the worker.
+         * @returns {Promise<Worker & {sync: object}>}
+         */
+        async function PyScriptWorker(file, options) {
+            await configs.get(TYPE).plugins;
+            const xworker = XWorker.call(
+                new Hook(null, hooked.get(TYPE)),
+                file,
+                {
+                    ...options,
+                    type: interpreter,
+                },
+            );
+            assign(xworker.sync, sync);
+            return xworker.ready;
         },
-    });
-};
+);
 
 // avoid multiple initialization of the same library
 const [
     {
         PyWorker: exportedPyWorker,
+        MPWorker: exportedMPWorker,
         hooks: exportedHooks,
         config: exportedConfig,
         whenDefined: exportedWhenDefined,
@@ -64,6 +66,7 @@ const [
     alreadyLive,
 ] = stickyModule("@pyscript/core", {
     PyWorker,
+    MPWorker,
     hooks,
     config: {},
     whenDefined,
@@ -72,6 +75,7 @@ const [
 export {
     TYPES,
     exportedPyWorker as PyWorker,
+    exportedMPWorker as MPWorker,
     exportedHooks as hooks,
     exportedConfig as config,
     exportedWhenDefined as whenDefined,
@@ -88,7 +92,7 @@ for (const [TYPE, interpreter] of TYPES) {
         else dispatch(element, TYPE, "done");
     };
 
-    const { config, plugins, error } = configs.get(TYPE);
+    const { config, configURL, plugins, error } = configs.get(TYPE);
 
     // create a unique identifier when/if needed
     let id = 0;
@@ -118,6 +122,36 @@ for (const [TYPE, interpreter] of TYPES) {
         return code;
     };
 
+    // register once any interpreter
+    let alreadyRegistered = false;
+
+    // allows lazy element features on code evaluation
+    let currentElement;
+
+    const registerModule = ({ XWorker, interpreter, io }) => {
+        // avoid multiple registration of the same interpreter
+        if (alreadyRegistered) return;
+        alreadyRegistered = true;
+
+        // automatically use the pyscript stderr (when/if defined)
+        // this defaults to console.error
+        function PyWorker(...args) {
+            const worker = XWorker(...args);
+            worker.onerror = ({ error }) => io.stderr(error);
+            return worker;
+        }
+
+        // enrich the Python env with some JS utility for main
+        interpreter.registerJsModule("_pyscript", {
+            PyWorker,
+            get target() {
+                return isScript(currentElement)
+                    ? currentElement.target.id
+                    : currentElement.id;
+            },
+        });
+    };
+
     // define the module as both `<script type="py">` and `<py-script>`
     // but only if the config didn't throw an error
     if (!error) {
@@ -133,10 +167,7 @@ for (const [TYPE, interpreter] of TYPES) {
                 main: {
                     ...codeFor(main),
                     async onReady(wrap, element) {
-                        if (shouldRegister) {
-                            shouldRegister = false;
-                            registerModule(wrap);
-                        }
+                        registerModule(wrap);
 
                         // allows plugins to do whatever they want with the element
                         // before regular stuff happens in here
@@ -256,6 +287,7 @@ for (const [TYPE, interpreter] of TYPES) {
 
             define(TYPE, {
                 config,
+                configURL,
                 interpreter,
                 hooks,
                 env: `${TYPE}-script`,
@@ -310,24 +342,3 @@ for (const [TYPE, interpreter] of TYPES) {
     // export the used config without allowing leaks through it
     exportedConfig[TYPE] = structuredClone(config);
 }
-
-/**
- * A `Worker` facade able to bootstrap on the worker thread only a PyScript module.
- * @param {string} file the python file to run ina worker.
- * @param {{config?: string | object, async?: boolean}} [options] optional configuration for the worker.
- * @returns {Worker & {sync: ProxyHandler<object>}}
- */
-function PyWorker(file, options) {
-    const hooks = hooked.get("py");
-    // this propagates pyscript worker hooks without needing a pyscript
-    // bootstrap + it passes arguments and enforces `pyodide`
-    // as the interpreter to use in the worker, as all hooks assume that
-    // and as `pyodide` is the only default interpreter that can deal with
-    // all the features we need to deliver pyscript out there.
-    const xworker = XWorker.call(new Hook(null, hooks), file, {
-        type: "pyodide",
-        ...options,
-    });
-    assign(xworker.sync, sync);
-    return xworker;
-}

pyscript.core/src/fetch.js

@@ -1,7 +1,10 @@
 import { FetchError, ErrorCode } from "./exceptions.js";
-import { getText } from "polyscript/exports";
 
-export { getText };
+/**
+ * @param {Response} response
+ * @returns
+ */
+export const getText = (response) => response.text();
 
 /**
  * This is a fetch wrapper that handles any non 200 responses and throws a

pyscript.core/src/plugins/py-editor.js

@@ -23,9 +23,12 @@ const hooks = {
 
 async function execute({ currentTarget }) {
     const { env, pySrc, outDiv } = this;
+    const hasRunButton = !!currentTarget;
 
-    currentTarget.disabled = true;
-    outDiv.innerHTML = "";
+    if (hasRunButton) {
+        currentTarget.disabled = true;
+        outDiv.innerHTML = "";
+    }
 
     if (!envs.has(env)) {
         const srcLink = URL.createObjectURL(new Blob([""]));
@@ -46,21 +49,25 @@ async function execute({ currentTarget }) {
     // before executing the current code
     envs.get(env).then((xworker) => {
         xworker.onerror = ({ error }) => {
-            outDiv.innerHTML += `<span style='color:red'>${
-                error.message || error
-            }</span>\n`;
+            if (hasRunButton) {
+                outDiv.innerHTML += `<span style='color:red'>${
+                    error.message || error
+                }</span>\n`;
+            }
             console.error(error);
         };
 
         const enable = () => {
-            currentTarget.disabled = false;
+            if (hasRunButton) currentTarget.disabled = false;
         };
         const { sync } = xworker;
         sync.write = (str) => {
-            outDiv.innerText += `${str}\n`;
+            if (hasRunButton) outDiv.innerText += `${str}\n`;
         };
         sync.writeErr = (str) => {
-            outDiv.innerHTML += `<span style='color:red'>${str}</span>\n`;
+            if (hasRunButton) {
+                outDiv.innerHTML += `<span style='color:red'>${str}</span>\n`;
+            }
         };
         sync.runAsync(pySrc).then(enable, enable);
     });
@@ -120,7 +127,6 @@ const init = async (script, type, interpreter) => {
         { keymap },
         { defaultKeymap },
     ] = await Promise.all([
-        // TODO: find a way to actually produce these bundles locally
         import(/* webpackIgnore: true */ "../3rd-party/codemirror.js"),
         import(/* webpackIgnore: true */ "../3rd-party/codemirror_state.js"),
         import(
@@ -131,6 +137,27 @@ const init = async (script, type, interpreter) => {
         import(/* webpackIgnore: true */ "../3rd-party/codemirror_commands.js"),
     ]);
 
+    const isSetup = script.hasAttribute("setup");
+    const env = `${interpreter}-${script.getAttribute("env") || getID(type)}`;
+    const source = script.src
+        ? await fetch(script.src).then((b) => b.text())
+        : script.textContent;
+    const context = {
+        interpreter,
+        env,
+        get pySrc() {
+            return isSetup ? source : editor.state.doc.toString();
+        },
+        get outDiv() {
+            return isSetup ? null : outDiv;
+        },
+    };
+
+    if (isSetup) {
+        execute.call(context, { currentTarget: null });
+        return;
+    }
+
     const selector = script.getAttribute("target");
 
     let target;
@@ -149,18 +176,6 @@ const init = async (script, type, interpreter) => {
     if (!target.hasAttribute("exec-id")) target.setAttribute("exec-id", 0);
     if (!target.hasAttribute("root")) target.setAttribute("root", target.id);
 
-    const env = `${interpreter}-${script.getAttribute("env") || getID(type)}`;
-    const context = {
-        interpreter,
-        env,
-        get pySrc() {
-            return editor.state.doc.toString();
-        },
-        get outDiv() {
-            return outDiv;
-        },
-    };
-
     // @see https://github.com/JeffersGlass/mkdocs-pyscript/blob/main/mkdocs_pyscript/js/makeblocks.js
     const listener = execute.bind(context);
     const [boxDiv, outDiv] = makeBoxDiv(listener, type);
@@ -200,6 +215,9 @@ const init = async (script, type, interpreter) => {
 // avoid too greedy MutationObserver operations at distance
 let timeout = 0;
 
+// avoid delayed initialization
+let queue = Promise.resolve();
+
 // reset interval value then check for new scripts
 const resetTimeout = () => {
     timeout = 0;
@@ -213,11 +231,14 @@ const pyEditor = async () => {
     for (const [type, interpreter] of TYPES) {
         const selector = `script[type="${type}-editor"]`;
         for (const script of document.querySelectorAll(selector)) {
-            // avoid any further bootstrap
+            // avoid any further bootstrap by changing the type as active
             script.type += "-active";
-            await init(script, type, interpreter);
+            // don't await in here or multiple calls might happen
+            // while the first script is being initialized
+            queue = queue.then(() => init(script, type, interpreter));
         }
     }
+    return queue;
 };
 
 new MutationObserver(pyEditor).observe(document, {

pyscript.core/src/plugins/py-terminal.js

@@ -1,6 +1,7 @@
 // PyScript py-terminal plugin
 import { TYPES, hooks } from "../core.js";
 import { notify } from "./error.js";
+import { defineProperty } from "polyscript/exports";
 
 const SELECTOR = [...TYPES.keys()]
     .map((type) => `script[type="${type}"][terminal],${type}-script[terminal]`)
@@ -13,31 +14,73 @@ const notifyAndThrow = (message) => {
     throw new Error(message);
 };
 
+const notParsedYet = (script) => !bootstrapped.has(script);
+
+const onceOnMain = ({ attributes: { worker } }) => !worker;
+
+const bootstrapped = new WeakSet();
+
+let addStyle = true;
+
+// this callback will be serialized as string and it never needs
+// to be invoked multiple times. Each xworker here is bootstrapped
+// only once thanks to the `sync.is_pyterminal()` check.
+const workerReady = ({ interpreter, io, run }, { sync }) => {
+    if (!sync.is_pyterminal()) return;
+
+    // in workers it's always safe to grab the polyscript currentScript
+    run("from polyscript.currentScript import terminal as __terminal__");
+
+    // This part is inevitably duplicated as external scope
+    // can't be reached by workers out of the box.
+    // The detail is that here we use sync though, not readline.
+    const decoder = new TextDecoder();
+    let data = "";
+    const generic = {
+        isatty: true,
+        write(buffer) {
+            data = decoder.decode(buffer);
+            sync.pyterminal_write(data);
+            return buffer.length;
+        },
+    };
+    interpreter.setStdout(generic);
+    interpreter.setStderr(generic);
+    interpreter.setStdin({
+        isatty: true,
+        stdin: () => sync.pyterminal_read(data),
+    });
+
+    io.stderr = (error) => {
+        sync.pyterminal_write(`${error.message || error}\n`);
+    };
+};
+
 const pyTerminal = async () => {
     const terminals = document.querySelectorAll(SELECTOR);
 
-    // no results will look further for runtime nodes
-    if (!terminals.length) return;
+    const unknown = [].filter.call(terminals, notParsedYet);
 
-    // if we arrived this far, let's drop the MutationObserver
-    // as we only support one terminal per page (right now).
-    mo.disconnect();
+    // no results will look further for runtime nodes
+    if (!unknown.length) return;
+    // early flag elements as known to avoid concurrent
+    // MutationObserver invokes of this async handler
+    else unknown.forEach(bootstrapped.add, bootstrapped);
 
     // we currently support only one terminal as in "classic"
-    if (terminals.length > 1) notifyAndThrow("You can use at most 1 terminal.");
-
-    const [element] = terminals;
-    // hopefully to be removed in the near future!
-    if (element.matches('script[type="mpy"],mpy-script'))
-        notifyAndThrow("Unsupported terminal.");
+    if ([].filter.call(terminals, onceOnMain).length > 1)
+        notifyAndThrow("You can use at most 1 main terminal");
 
     // import styles lazily
-    document.head.append(
-        Object.assign(document.createElement("link"), {
-            rel: "stylesheet",
-            href: new URL("./xterm.css", import.meta.url),
-        }),
-    );
+    if (addStyle) {
+        addStyle = false;
+        document.head.append(
+            Object.assign(document.createElement("link"), {
+                rel: "stylesheet",
+                href: new URL("./xterm.css", import.meta.url),
+            }),
+        );
+    }
 
     // lazy load these only when a valid terminal is found
     const [{ Terminal }, { Readline }, { FitAddon }] = await Promise.all([
@@ -46,100 +89,113 @@ const pyTerminal = async () => {
         import(/* webpackIgnore: true */ "../3rd-party/xterm_addon-fit.js"),
     ]);
 
-    const readline = new Readline();
+    for (const element of unknown) {
+        // hopefully to be removed in the near future!
+        if (element.matches('script[type="mpy"],mpy-script'))
+            notifyAndThrow("Unsupported terminal.");
 
-    // common main thread initialization for both worker
-    // or main case, bootstrapping the terminal on its target
-    const init = (options) => {
-        let target = element;
-        const selector = element.getAttribute("target");
-        if (selector) {
-            target =
-                document.getElementById(selector) ||
-                document.querySelector(selector);
-            if (!target) throw new Error(`Unknown target ${selector}`);
-        } else {
-            target = document.createElement("py-terminal");
-            target.style.display = "block";
-            element.after(target);
-        }
-        const terminal = new Terminal({
-            theme: {
-                background: "#191A19",
-                foreground: "#F5F2E7",
-            },
-            ...options,
-        });
-        const fitAddon = new FitAddon();
-        terminal.loadAddon(fitAddon);
-        terminal.loadAddon(readline);
-        terminal.open(target);
-        fitAddon.fit();
-        terminal.focus();
-    };
+        const readline = new Readline();
 
-    // branch logic for the worker
-    if (element.hasAttribute("worker")) {
-        // when the remote thread onReady triggers:
-        // setup the interpreter stdout and stderr
-        const workerReady = ({ interpreter }, { sync }) => {
-            sync.pyterminal_drop_hooks();
-            const decoder = new TextDecoder();
-            let data = "";
-            const generic = {
-                isatty: true,
-                write(buffer) {
-                    data = decoder.decode(buffer);
-                    sync.pyterminal_write(data);
-                    return buffer.length;
+        // common main thread initialization for both worker
+        // or main case, bootstrapping the terminal on its target
+        const init = (options) => {
+            let target = element;
+            const selector = element.getAttribute("target");
+            if (selector) {
+                target =
+                    document.getElementById(selector) ||
+                    document.querySelector(selector);
+                if (!target) throw new Error(`Unknown target ${selector}`);
+            } else {
+                target = document.createElement("py-terminal");
+                target.style.display = "block";
+                element.after(target);
+            }
+            const terminal = new Terminal({
+                theme: {
+                    background: "#191A19",
+                    foreground: "#F5F2E7",
                 },
-            };
-            interpreter.setStdout(generic);
-            interpreter.setStderr(generic);
-            interpreter.setStdin({
-                isatty: true,
-                stdin: () => sync.pyterminal_read(data),
+                ...options,
             });
+            const fitAddon = new FitAddon();
+            terminal.loadAddon(fitAddon);
+            terminal.loadAddon(readline);
+            terminal.open(target);
+            fitAddon.fit();
+            terminal.focus();
+            defineProperty(element, "terminal", { value: terminal });
+            return terminal;
         };
 
-        // add a hook on the main thread to setup all sync helpers
-        // also bootstrapping the XTerm target on main
-        hooks.main.onWorker.add(function worker(_, xworker) {
-            hooks.main.onWorker.delete(worker);
-            init({
-                disableStdin: false,
-                cursorBlink: true,
-                cursorStyle: "block",
-            });
-            xworker.sync.pyterminal_read = readline.read.bind(readline);
-            xworker.sync.pyterminal_write = readline.write.bind(readline);
-            // allow a worker to drop main thread hooks ASAP
-            xworker.sync.pyterminal_drop_hooks = () => {
-                hooks.worker.onReady.delete(workerReady);
-            };
-        });
+        // branch logic for the worker
+        if (element.hasAttribute("worker")) {
+            // add a hook on the main thread to setup all sync helpers
+            // also bootstrapping the XTerm target on main *BUT* ...
+            hooks.main.onWorker.add(function worker(_, xworker) {
+                // ... as multiple workers will add multiple callbacks
+                // be sure no xworker is ever initialized twice!
+                if (bootstrapped.has(xworker)) return;
+                bootstrapped.add(xworker);
 
-        // setup remote thread JS/Python code for whenever the
-        // worker is ready to become a terminal
-        hooks.worker.onReady.add(workerReady);
-    } else {
-        // in the main case, just bootstrap XTerm without
-        // allowing any input as that's not possible / awkward
-        hooks.main.onReady.add(function main({ io }) {
-            console.warn("py-terminal is read only on main thread");
-            hooks.main.onReady.delete(main);
-            init({
-                disableStdin: true,
-                cursorBlink: false,
-                cursorStyle: "underline",
+                // still cleanup this callback for future scripts/workers
+                hooks.main.onWorker.delete(worker);
+
+                init({
+                    disableStdin: false,
+                    cursorBlink: true,
+                    cursorStyle: "block",
+                });
+
+                xworker.sync.is_pyterminal = () => true;
+                xworker.sync.pyterminal_read = readline.read.bind(readline);
+                xworker.sync.pyterminal_write = readline.write.bind(readline);
             });
-            io.stdout = (value) => {
-                readline.write(`${value}\n`);
-            };
-            io.stderr = (error) => {
-                readline.write(`${error.message || error}\n`);
-            };
-        });
+
+            // setup remote thread JS/Python code for whenever the
+            // worker is ready to become a terminal
+            hooks.worker.onReady.add(workerReady);
+        } else {
+            // in the main case, just bootstrap XTerm without
+            // allowing any input as that's not possible / awkward
+            hooks.main.onReady.add(function main({ interpreter, io, run }) {
+                console.warn("py-terminal is read only on main thread");
+                hooks.main.onReady.delete(main);
+
+                // on main, it's easy to trash and clean the current terminal
+                globalThis.__py_terminal__ = init({
+                    disableStdin: true,
+                    cursorBlink: false,
+                    cursorStyle: "underline",
+                });
+                run("from js import __py_terminal__ as __terminal__");
+                delete globalThis.__py_terminal__;
+
+                // This part is inevitably duplicated as external scope
+                // can't be reached by workers out of the box.
+                // The detail is that here we use readline here, not sync.
+                const decoder = new TextDecoder();
+                let data = "";
+                const generic = {
+                    isatty: true,
+                    write(buffer) {
+                        data = decoder.decode(buffer);
+                        readline.write(data);
+                        return buffer.length;
+                    },
+                };
+                interpreter.setStdout(generic);
+                interpreter.setStderr(generic);
+                interpreter.setStdin({
+                    isatty: true,
+                    stdin: () => readline.read(data),
+                });
+
+                io.stderr = (error) => {
+                    readline.write(`${error.message || error}\n`);
+                };
+            });
+        }
     }
 };
 

pyscript.core/src/stdlib.js

@@ -21,7 +21,7 @@ const write = (base, literal) => {
         python.push(`_path = _Path("${base}/${key}")`);
         if (typeof value === "string") {
             const code = JSON.stringify(value);
-            python.push(`_path.write_text(${code})`);
+            python.push(`_path.write_text(${code},encoding="utf-8")`);
         } else {
             // @see https://github.com/pyscript/pyscript/pull/1813#issuecomment-1781502909
             python.push(`if not _os.path.exists("${base}/${key}"):`);

pyscript.core/src/stdlib/pyscript/__init__.py

@@ -30,6 +30,7 @@
 #     as it works transparently in both the main thread and worker cases.
 
 from pyscript.display import HTML, display
+from pyscript.fetch import fetch
 from pyscript.magic_js import (
     RUNNING_IN_WORKER,
     PyWorker,
@@ -43,6 +44,8 @@ from pyscript.magic_js import (
 try:
     from pyscript.event_handling import when
 except:
+    # TODO: should we remove this? Or at the very least, we should capture
+    # the traceback otherwise it's very hard to debug
     from pyscript.util import NotSupported
 
     when = NotSupported(

pyscript.core/src/stdlib/pyscript/event_handling.py

@@ -1,6 +1,14 @@
 import inspect
 
-from pyodide.ffi.wrappers import add_event_listener
+try:
+    from pyodide.ffi.wrappers import add_event_listener
+
+except ImportError:
+
+    def add_event_listener(el, event_type, func):
+        el.addEventListener(event_type, func)
+
+
 from pyscript.magic_js import document
 
 
@@ -27,19 +35,32 @@ def when(event_type=None, selector=None):
                     f"Invalid selector: {selector}. Selector must"
                     " be a string, a pydom.Element or a pydom.ElementCollection."
                 )
+        try:
+            sig = inspect.signature(func)
+            # Function doesn't receive events
+            if not sig.parameters:
 
-        sig = inspect.signature(func)
-        # Function doesn't receive events
-        if not sig.parameters:
+                def wrapper(*args, **kwargs):
+                    func()
 
+            else:
+                wrapper = func
+
+        except AttributeError:
+            # TODO: this is currently an quick hack to get micropython working but we need
+            #       to actually properly replace inspect.signature with something else
             def wrapper(*args, **kwargs):
-                func()
+                try:
+                    return func(*args, **kwargs)
+                except TypeError as e:
+                    if "takes 0 positional arguments" in str(e):
+                        return func()
+
+                    raise
+
+        for el in elements:
+            add_event_listener(el, event_type, wrapper)
 
-            for el in elements:
-                add_event_listener(el, event_type, wrapper)
-        else:
-            for el in elements:
-                add_event_listener(el, event_type, func)
         return func
 
     return decorator

pyscript.core/src/stdlib/pyscript/fetch.py

@@ -0,0 +1,64 @@
+import json
+
+import js
+
+
+def _as_bytearray(buffer):
+    ui8a = js.Uint8Array.new(buffer)
+    size = ui8a.length
+    ba = bytearray(size)
+    for i in range(0, size):
+        ba[i] = ui8a[i]
+    return ba
+
+
+class _Fetch:
+    def __init__(self, url, **kw):
+        # avoid both Pyodide and MicroPython FFI
+        options = js.JSON.parse(json.dumps(kw))
+        self._url = url
+        self._fetch = js.fetch(url, options)
+
+    async def _arrayBuffer(self):
+        response = await self._response()
+        return await response.arrayBuffer()
+
+    async def _response(self):
+        response = await self._fetch
+        if not response.ok:
+            msg = f"URL {self._url} failed with status {response.status}"
+            raise Exception(msg)
+        return response
+
+    # https://developer.mozilla.org/en-US/docs/Web/API/Response/arrayBuffer
+    # returns a memoryview of the buffer
+    async def arrayBuffer(self):
+        buffer = await self._arrayBuffer()
+        # works in Pyodide
+        if hasattr(buffer, "to_py"):
+            return buffer.to_py()
+        # shims in MicroPython
+        return memoryview(_as_bytearray(buffer))
+
+    # https://developer.mozilla.org/en-US/docs/Web/API/Response/blob
+    async def blob(self):
+        response = await self._response()
+        return await response.blob()
+
+    # return a bytearray from the uint8 view of the buffer
+    async def bytearray(self):
+        buffer = await self._arrayBuffer()
+        return _as_bytearray(buffer)
+
+    # https://developer.mozilla.org/en-US/docs/Web/API/Response/json
+    async def json(self):
+        return json.loads(await self.text())
+
+    # https://developer.mozilla.org/en-US/docs/Web/API/Response/text
+    async def text(self):
+        response = await self._response()
+        return await response.text()
+
+
+def fetch(url, **kw):
+    return _Fetch(url, **kw)

pyscript.core/src/stdlib/pyscript/magic_js.py

@@ -1,20 +1,56 @@
+import sys
+
 import js as globalThis
 from polyscript import js_modules
 from pyscript.util import NotSupported
 
 RUNNING_IN_WORKER = not hasattr(globalThis, "document")
 
+
+# allow `from pyscript.js_modules.xxx import yyy`
+class JSModule:
+    def __init__(self, name):
+        self.name = name
+
+    def __getattr__(self, field):
+        # avoid pyodide looking for non existent fields
+        if not field.startswith("_"):
+            return getattr(getattr(js_modules, self.name), field)
+
+
+# generate N modules in the system that will proxy the real value
+for name in globalThis.Reflect.ownKeys(js_modules):
+    sys.modules[f"pyscript.js_modules.{name}"] = JSModule(name)
+sys.modules["pyscript.js_modules"] = js_modules
+
 if RUNNING_IN_WORKER:
-    import js
     import polyscript
 
     PyWorker = NotSupported(
         "pyscript.PyWorker",
         "pyscript.PyWorker works only when running in the main thread",
     )
-    window = polyscript.xworker.window
-    document = window.document
-    js.document = document
+
+    try:
+        globalThis.SharedArrayBuffer.new(4)
+        import js
+
+        window = polyscript.xworker.window
+        document = window.document
+        js.document = document
+    except:
+        globalThis.console.debug("SharedArrayBuffer is not available")
+        # in this scenario none of the utilities would work
+        # as expected so we better export these as NotSupported
+        window = NotSupported(
+            "pyscript.window",
+            "pyscript.window in workers works only via SharedArrayBuffer",
+        )
+        document = NotSupported(
+            "pyscript.document",
+            "pyscript.document in workers works only via SharedArrayBuffer",
+        )
+
     sync = polyscript.xworker.sync
 
     # in workers the display does not have a default ID

pyscript.core/src/stdlib/pyweb/__init__.py

@@ -0,0 +1 @@
+from .pydom import dom as pydom

pyscript.core/src/stdlib/pyweb/media.py

@@ -0,0 +1,95 @@
+from pyodide.ffi import to_js
+from pyscript import window
+
+
+class Device:
+    """Device represents a media input or output device, such as a microphone,
+    camera, or headset.
+    """
+
+    def __init__(self, device):
+        self._js = device
+
+    @property
+    def id(self):
+        return self._js.deviceId
+
+    @property
+    def group(self):
+        return self._js.groupId
+
+    @property
+    def kind(self):
+        return self._js.kind
+
+    @property
+    def label(self):
+        return self._js.label
+
+    def __getitem__(self, key):
+        return getattr(self, key)
+
+    @classmethod
+    async def load(cls, audio=False, video=True):
+        """Load the device stream."""
+        options = window.Object.new()
+        options.audio = audio
+        if isinstance(video, bool):
+            options.video = video
+        else:
+            # TODO: Think this can be simplified but need to check it on the pyodide side
+
+            # TODO: this is pyodide specific. shouldn't be!
+            options.video = window.Object.new()
+            for k in video:
+                setattr(
+                    options.video,
+                    k,
+                    to_js(video[k], dict_converter=window.Object.fromEntries),
+                )
+
+        stream = await window.navigator.mediaDevices.getUserMedia(options)
+        return stream
+
+    async def get_stream(self):
+        key = self.kind.replace("input", "").replace("output", "")
+        options = {key: {"deviceId": {"exact": self.id}}}
+
+        return await self.load(**options)
+
+
+async def list_devices() -> list[dict]:
+    """
+    Return the list of the currently available media input and output devices,
+    such as microphones, cameras, headsets, and so forth.
+
+    Output:
+
+        list(dict) - list of dictionaries representing the available media devices.
+            Each dictionary has the following keys:
+            * deviceId: a string that is an identifier for the represented device
+                that is persisted across sessions. It is un-guessable by other
+                applications and unique to the origin of the calling application.
+                It is reset when the user clears cookies (for Private Browsing, a
+                different identifier is used that is not persisted across sessions).
+
+            * groupId: a string that is a group identifier. Two devices have the same
+                group identifier if they belong to the same physical device — for
+                example a monitor with both a built-in camera and a microphone.
+
+            * kind: an enumerated value that is either "videoinput", "audioinput"
+                or "audiooutput".
+
+            * label: a string describing this device (for example "External USB
+                Webcam").
+
+    Note: the returned list will omit any devices that are blocked by the document
+    Permission Policy: microphone, camera, speaker-selection (for output devices),
+    and so on. Access to particular non-default devices is also gated by the
+    Permissions API, and the list will omit devices for which the user has not
+    granted explicit permission.
+    """
+    # https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/enumerateDevices
+    return [
+        Device(obj) for obj in await window.navigator.mediaDevices.enumerateDevices()
+    ]

pyscript.core/src/stdlib/pyweb/pydom.py

@@ -1,9 +1,34 @@
-import sys
-import warnings
-from functools import cached_property
-from typing import Any
+try:
+    from typing import Any
+except ImportError:
+    Any = "Any"
+
+try:
+    import warnings
+except ImportError:
+    # TODO: For now it probably means we are in MicroPython. We should figure
+    # out the "right" way to handle this. For now we just ignore the warning
+    # and logging to console
+    class warnings:
+        @staticmethod
+        def warn(*args, **kwargs):
+            print("WARNING: ", *args, **kwargs)
+
+
+try:
+    from functools import cached_property
+except ImportError:
+    # TODO: same comment about micropython as above
+    cached_property = property
+
+try:
+    from pyodide.ffi import JsProxy
+except ImportError:
+    # TODO: same comment about micropython as above
+    def JsProxy(obj):
+        return obj
+
 
-from pyodide.ffi import JsProxy
 from pyscript import display, document, window
 
 alert = window.alert
@@ -100,6 +125,14 @@ class Element(BaseElement):
     def html(self, value):
         self._js.innerHTML = value
 
+    @property
+    def text(self):
+        return self._js.textContent
+
+    @text.setter
+    def text(self, value):
+        self._js.textContent = value
+
     @property
     def content(self):
         # TODO: This breaks with with standard template elements. Define how to best
@@ -204,6 +237,91 @@ class Element(BaseElement):
     def show_me(self):
         self._js.scrollIntoView()
 
+    def snap(
+        self,
+        to: BaseElement | str = None,
+        width: int | None = None,
+        height: int | None = None,
+    ):
+        """
+        Captures a snapshot of a video element. (Only available for video elements)
+
+        Inputs:
+
+            * to: element where to save the snapshot of the video frame to
+            * width: width of the image
+            * height: height of the image
+
+        Output:
+            (Element) canvas element where the video frame snapshot was drawn into
+        """
+        if self._js.tagName != "VIDEO":
+            raise AttributeError("Snap method is only available for video Elements")
+
+        if to is None:
+            canvas = self.create("canvas")
+            if width is None:
+                width = self._js.width
+            if height is None:
+                height = self._js.height
+            canvas._js.width = width
+            canvas._js.height = height
+
+        elif isistance(to, Element):
+            if to._js.tagName != "CANVAS":
+                raise TypeError("Element to snap to must a canvas.")
+            canvas = to
+        elif getattr(to, "tagName", "") == "CANVAS":
+            canvas = Element(to)
+        elif isinstance(to, str):
+            canvas = pydom[to][0]
+            if canvas._js.tagName != "CANVAS":
+                raise TypeError("Element to snap to must a be canvas.")
+
+        canvas.draw(self, width, height)
+
+        return canvas
+
+    def download(self, filename: str = "snapped.png") -> None:
+        """Download the current element (only available for canvas elements) with the filename
+        provided in input.
+
+        Inputs:
+            * filename (str): name of the file being downloaded
+
+        Output:
+            None
+        """
+        if self._js.tagName != "CANVAS":
+            raise AttributeError(
+                "The download method is only available for canvas Elements"
+            )
+
+        link = self.create("a")
+        link._js.download = filename
+        link._js.href = self._js.toDataURL()
+        link._js.click()
+
+    def draw(self, what, width, height):
+        """Draw `what` on the current element  (only available for canvas elements).
+
+        Inputs:
+
+            * what (canvas image source): An element to draw into the context. The specification permits any canvas
+                image source, specifically, an HTMLImageElement, an SVGImageElement, an HTMLVideoElement,
+                an HTMLCanvasElement, an ImageBitmap, an OffscreenCanvas, or a VideoFrame.
+        """
+        if self._js.tagName != "CANVAS":
+            raise AttributeError(
+                "The draw method is only available for canvas Elements"
+            )
+
+        if isinstance(what, Element):
+            what = what._js
+
+        # https://developer.mozilla.org/en-US/docs/Web/API/CanvasRenderingContext2D/drawImage
+        self._js.getContext("2d").drawImage(what, 0, 0, width, height)
+
 
 class OptionsProxy:
     """This class represents the options of a select element. It
@@ -276,7 +394,7 @@ class OptionsProxy:
         return self.options[key]
 
 
-class StyleProxy(dict):
+class StyleProxy:  # (dict):
     def __init__(self, element: Element) -> None:
         self._element = element
 
@@ -395,7 +513,7 @@ class ElementCollection:
 
 
 class DomScope:
-    def __getattr__(self, __name: str) -> Any:
+    def __getattr__(self, __name: str):
         element = document[f"#{__name}"]
         if element:
             return element[0]
@@ -409,7 +527,12 @@ class PyDom(BaseElement):
     ElementCollection = ElementCollection
 
     def __init__(self):
-        super().__init__(document)
+        # PyDom is a special case of BaseElement where we don't want to create a new JS element
+        # and it really doesn't have a need for styleproxy or parent to to call to __init__
+        # (which actually fails in MP for some reason)
+        self._js = document
+        self._parent = None
+        self._proxies = {}
         self.ids = DomScope()
         self.body = Element(document.body)
         self.head = Element(document.head)
@@ -418,10 +541,6 @@ class PyDom(BaseElement):
         return super().create(type_, is_child=False, classes=classes, html=html)
 
     def __getitem__(self, key):
-        if isinstance(key, int):
-            indices = range(*key.indices(len(self.list)))
-            return [self.list[i] for i in indices]
-
         elements = self._js.querySelectorAll(key)
         if not elements:
             return None
@@ -429,5 +548,3 @@ class PyDom(BaseElement):
 
 
 dom = PyDom()
-
-sys.modules[__name__] = dom

pyscript.core/src/sync.js

@@ -1,4 +1,7 @@
 export default {
+    // allow pyterminal checks to bootstrap
+    is_pyterminal: () => false,
+
     /**
      * 'Sleep' for the given number of seconds. Used to implement Python's time.sleep in Worker threads.
      * @param {number} seconds The number of seconds to sleep.

pyscript.core/test/camera.html

@@ -0,0 +1,24 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>PyScript Media Example</title>
+  <link rel="stylesheet" href="../dist/core.css">
+  <script type="module" src="../dist/core.js"></script>
+</head>
+<body>
+  <script type="py" src="camera.py" async></script>
+
+  <label for="cars">Choose a device:</label>
+
+  <select name="devices" id="devices"></select>
+
+  <button id="pick-device">Select the device</button>
+  <button id="snap">Snap</button>
+
+  <div id="result"></div>
+
+  <video id="video" width="600" height="400" autoplay></video>
+</body>
+</html>

pyscript.core/test/camera.py

@@ -0,0 +1,32 @@
+from pyodide.ffi import create_proxy
+from pyscript import display, document, when, window
+from pyweb import media, pydom
+
+devicesSelect = pydom["#devices"][0]
+video = pydom["video"][0]
+devices = {}
+
+
+async def list_media_devices(event=None):
+    """List the available media devices."""
+    global devices
+    for i, device in enumerate(await media.list_devices()):
+        devices[device.id] = device
+        label = f"{i} - ({device.kind}) {device.label} [{device.id}]"
+        devicesSelect.options.add(value=device.id, html=label)
+
+
+@when("click", "#pick-device")
+async def connect_to_device(e):
+    """Connect to the selected device."""
+    device = devices[devicesSelect.value]
+    video._js.srcObject = await device.get_stream()
+
+
+@when("click", "#snap")
+async def camera_click(e):
+    """Take a picture and download it."""
+    video.snap().download()
+
+
+await list_media_devices()

pyscript.core/test/code-a-part.html

@@ -0,0 +1,19 @@
+<!doctype html>
+<html lang="en">
+    <head>
+        <meta charset="UTF-8" />
+        <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+        <link rel="stylesheet" href="../dist/core.css">
+        <script type="module">
+            import { hooks } from "../dist/core.js";
+            hooks.main.codeBeforeRun.add('print(0)');
+            hooks.main.codeAfterRun.add('print(2)');
+        </script>
+    </head>
+    <body>
+        <script type="py">
+            # raise an error instead to see it on line 1
+            print(1)
+        </script>
+    </body>
+</html>

pyscript.core/test/config-url.html

@@ -6,6 +6,12 @@
   <title>PyScript Next Plugin</title>
   <link rel="stylesheet" href="../dist/core.css">
   <script type="module" src="../dist/core.js"></script>
-  <py-config src="bad.toml" type="toml"></py-config>
+  <mpy-config src="config-url/config.json"></mpy-config>
+  <script type="mpy">
+    from runtime import test
+  </script>
+  <script type="mpy" worker>
+    from runtime import test
+  </script>
 </head>
 </html>

pyscript.core/test/config-url/config.json

@@ -0,0 +1,7 @@
+{
+    "files":{
+        "{FROM}": "./src",
+        "{TO}": "./runtime",
+        "{FROM}/test.py": "{TO}/test.py"
+    }
+}

pyscript.core/test/config-url/src/test.py

@@ -0,0 +1,8 @@
+from pyscript import RUNNING_IN_WORKER, document
+
+classList = document.documentElement.classList
+
+if RUNNING_IN_WORKER:
+    classList.add("worker")
+else:
+    classList.add("main")

pyscript.core/test/fetch.html

@@ -0,0 +1,75 @@
+<!doctype html>
+<html lang="en">
+    <head>
+        <meta charset="UTF-8" />
+        <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+        <link rel="stylesheet" href="../dist/core.css">
+    </head>
+    <body>
+      <script type="module">
+        import fetch from 'https://esm.run/@webreflection/fetch';
+
+        globalThis.fetch_text = await fetch("config.json").text();
+        globalThis.fetch_json = JSON.stringify(await fetch("config.json").json());
+        globalThis.fetch_buffer = new Uint8Array((await fetch("config.json").arrayBuffer())).length;
+
+        document.head.appendChild(
+          Object.assign(
+            document.createElement('script'),
+            {
+              type: 'module',
+              src: '../dist/core.js'
+            }
+          )
+        );
+      </script>
+      <script type="mpy" async>
+          import js, json
+          from pyscript import document, fetch
+
+          fetch_text = await fetch("config.json").text()
+          if (fetch_text != js.fetch_text):
+            raise Exception("fetch_text")
+
+          fetch_json = await fetch("config.json").json()
+          if (json.dumps(fetch_json).replace(" ", "") != js.fetch_json):
+            raise Exception("fetch_json")
+
+          fetch_buffer = await fetch("config.json").bytearray()
+          if (len(fetch_buffer) != js.fetch_buffer):
+            raise Exception("fetch_buffer")
+
+          print(await fetch("config.json").bytearray())
+          print(await fetch("config.json").blob())
+
+          try:
+            await fetch("shenanigans.nope").text()
+          except:
+            document.documentElement.classList.add('mpy')
+      </script>
+      <script type="py" async>
+        import js, json
+        from pyscript import document, fetch
+
+        fetch_text = await fetch("config.json").text()
+        if (fetch_text != js.fetch_text):
+          raise Exception("fetch_text")
+
+        fetch_json = await fetch("config.json").json()
+        if (json.dumps(fetch_json).replace(" ", "") != js.fetch_json):
+          raise Exception("fetch_json")
+
+        fetch_buffer = await fetch("config.json").bytearray()
+        if (len(fetch_buffer) != js.fetch_buffer):
+          raise Exception("fetch_buffer")
+
+        print(await fetch("config.json").bytearray())
+        print(await fetch("config.json").blob())
+
+        try:
+          await fetch("shenanigans.nope").text()
+        except:
+          document.documentElement.classList.add('py')
+      </script>
+    </body>
+</html>

pyscript.core/test/js_modules.html

@@ -0,0 +1,39 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <link rel="stylesheet" href="../dist/core.css">
+    <script type="module" src="../dist/core.js"></script>
+</head>
+<body>
+    <mpy-config>
+        [js_modules.main]
+        "./js_modules.js" = "random_js"
+    </mpy-config>
+    <mpy-script>
+        from pyscript.js_modules.random_js import default as value
+        from pyscript.js_modules import random_js
+        from pyscript import js_modules
+
+        print("mpy", value)
+        print("mpy", random_js.default)
+        print("mpy", js_modules.random_js.default)
+    </mpy-script>
+    <py-config>
+        [js_modules.main]
+        "./js_modules.js" = "random_js"
+    </py-config>
+    <py-script>
+        from pyscript.js_modules.random_js import default as value
+        from pyscript.js_modules import random_js
+        from pyscript import js_modules, document
+
+        print("py", value)
+        print("py", random_js.default)
+        print("py", js_modules.random_js.default)
+
+        document.documentElement.classList.add('done')
+    </py-script>
+</body>
+</html>

pyscript.core/test/js_modules.js

@@ -0,0 +1 @@
+export default Math.random();

pyscript.core/test/mpy.spec.js

@@ -35,3 +35,51 @@ test('MicroPython hooks', async ({ page }) => {
     'worker onAfterRun',
   ].join('\n'));
 });
+
+test('MicroPython + Pyodide js_modules', async ({ page }) => {
+  const logs = [];
+  page.on('console', msg => {
+    const text = msg.text();
+    if (!text.startsWith('['))
+      logs.push(text);
+  });
+  await page.goto('http://localhost:8080/test/js_modules.html');
+  await page.waitForSelector('html.done');
+  await expect(logs.length).toBe(6);
+  await expect(logs[0]).toBe(logs[1]);
+  await expect(logs[1]).toBe(logs[2]);
+  await expect(logs[3]).toBe(logs[4]);
+  await expect(logs[4]).toBe(logs[5]);
+});
+
+test('MicroPython + configURL', async ({ page }) => {
+  const logs = [];
+  page.on('console', msg => {
+    const text = msg.text();
+    if (!text.startsWith('['))
+      logs.push(text);
+  });
+  await page.goto('http://localhost:8080/test/config-url.html');
+  await page.waitForSelector('html.main.worker');
+});
+
+test('Pyodide + terminal on Main', async ({ page }) => {
+  await page.goto('http://localhost:8080/test/py-terminal-main.html');
+  await page.waitForSelector('html.ok');
+});
+
+
+test('Pyodide + terminal on Worker', async ({ page }) => {
+  await page.goto('http://localhost:8080/test/py-terminal-worker.html');
+  await page.waitForSelector('html.ok');
+});
+
+test('Pyodide + multiple terminals via Worker', async ({ page }) => {
+  await page.goto('http://localhost:8080/test/py-terminals.html');
+  await page.waitForSelector('html.first.second');
+});
+
+test('MicroPython + Pyodide fetch', async ({ page }) => {
+  await page.goto('http://localhost:8080/test/fetch.html');
+  await page.waitForSelector('html.mpy.py');
+});

pyscript.core/test/multi.html

@@ -0,0 +1,20 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <script type="module" src="../dist/core.js"></script>
+  </head>
+  <body>
+    <script type="mpy">
+      from pyscript import document
+      import sys
+      document.body.append(sys.version)
+    </script>
+    <script type="py">
+      from pyscript import document
+      import sys
+      document.body.append(sys.version)
+    </script>
+  </body>
+</html>

pyscript.core/test/multiple-editors.html

@@ -0,0 +1,31 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width,initial-scale=1">
+  <title>PyScript Test</title>
+  <link rel="stylesheet" href="../dist/core.css">
+  <script type="module" src="../dist/core.js"></script>
+</head>
+<body>
+  <script type="py-editor">
+    0
+  </script>
+  <script type="py-editor">
+    1
+  </script>
+  <script type="py-editor">
+    2
+  </script>
+  <script type="py-editor">
+    3
+  </script>
+  <script type="py-editor">
+    4
+  </script>
+  <script type="py-editor">
+    5
+  </script>
+  <!-- more... -->
+</body>
+</html>

pyscript.core/test/no_sab/index.html

@@ -0,0 +1,23 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <link rel="stylesheet" href="../../dist/core.css">
+  <script type="module">
+    import { PyWorker } from '../../dist/core.js';
+    const { sync } = await PyWorker(
+      './worker.py',
+      {
+        config: {
+          sync_main_only: true
+        }
+      }
+    );
+
+    document.documentElement.classList.add(
+      await sync.get_class()
+    );
+  </script>
+</head>
+</html>

pyscript.core/test/no_sab/worker.py

@@ -0,0 +1,3 @@
+from pyscript import sync
+
+sync.get_class = lambda: "ok"

pyscript.core/test/py-editor/index.html

@@ -0,0 +1,21 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <link rel="stylesheet" href="../../dist/core.css">
+  <script type="module" src="../../dist/core.js"></script>
+</head>
+<body>
+  <script type="mpy-editor" src="task1.py" env="task1" setup></script>
+  <script type="mpy-editor" env="task1">
+    print(a)
+  </script>
+  <script type="mpy-editor" env="task2" setup>
+    b = 2
+  </script>
+  <script type="mpy-editor" env="task2">
+    print(b)
+  </script>
+</body>
+</html>

pyscript.core/test/py-editor/task1.py

@@ -0,0 +1 @@
+a = 1

pyscript.core/test/py-terminal-main.html

@@ -0,0 +1,14 @@
+<!doctype html>
+<html lang="en">
+    <head>
+        <meta charset="UTF-8" />
+        <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+        <title>PyTerminal Main</title>
+        <link rel="stylesheet" href="../dist/core.css">
+        <script type="module" src="../dist/core.js"></script>
+        <style>.xterm { padding: .5rem; }</style>
+    </head>
+    <body>
+        <py-script src="terminal.py" terminal></py-script>
+    </body>
+</html>

pyscript.core/test/py-terminal-worker.html

@@ -0,0 +1,15 @@
+<!doctype html>
+<html lang="en">
+    <head>
+        <meta charset="UTF-8" />
+        <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+        <title>PyTerminal Main</title>
+        <link rel="stylesheet" href="../dist/core.css">
+        <script type="module" src="../dist/core.js"></script>
+        <style>.xterm { padding: .5rem; }</style>
+    </head>
+    <body>
+        <script type="py" src="terminal.py" worker terminal></script>
+        <script type="py" src="terminal.py" worker terminal></script>
+    </body>
+</html>

pyscript.core/test/py-terminal.html

@@ -14,6 +14,9 @@
                 print('hello world')
         </script>
         <py-script worker terminal>
+            # works on both worker and main scripts
+            print("__terminal__", __terminal__)
+
             import sys
             from pyscript import display, document
             display("Hello", "PyScript Next - PyTerminal", append=False)

pyscript.core/test/py-terminals.html

@@ -0,0 +1,27 @@
+<!doctype html>
+<html lang="en">
+    <head>
+        <meta charset="UTF-8" />
+        <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+        <title>PyTerminal Main</title>
+        <link rel="stylesheet" href="../dist/core.css">
+        <script type="module" src="../dist/core.js"></script>
+        <style>.xterm { padding: .5rem; }</style>
+    </head>
+    <body>
+        <script type="py" worker terminal>
+            from pyscript import document
+            document.documentElement.classList.add("first")
+
+            import code
+            code.interact()
+        </script>
+        <script type="py" worker terminal>
+            from pyscript import document
+            document.documentElement.classList.add("second")
+
+            import code
+            code.interact()
+        </script>
+    </body>
+</html>

pyscript.core/test/pydom.html

@@ -3,7 +3,7 @@
 <head>
   <meta charset="UTF-8">
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
-  <title>PyScript Next Plugin</title>
+  <title>PyDom Example</title>
   <link rel="stylesheet" href="../dist/core.css">
   <script type="module" src="../dist/core.js"></script>
 </head>

pyscript.core/test/pydom.py

@@ -1,26 +1,32 @@
 import random
+import time
 from datetime import datetime as dt
 
-from pyscript import display
+from pyscript import display, when
 from pyweb import pydom
-from pyweb.base import when
 
 
 @when("click", "#just-a-button")
-def on_click(event):
-    print(f"Hello from Python! {dt.now()}")
-    display(f"Hello from Python! {dt.now()}", append=False, target="result")
+def on_click():
+    try:
+        timenow = dt.now()
+    except NotImplementedError:
+        # In this case we assume it's not implemented because we are using MycroPython
+        tnow = time.localtime()
+        tstr = "{:02d}/{:02d}/{:04d} {:02d}:{:02d}:{:02d}"
+        timenow = tstr.format(tnow[2], tnow[1], tnow[0], *tnow[2:])
+
+    display(f"Hello from PyScript, time is: {timenow}", append=False, target="result")
 
 
 @when("click", "#color-button")
 def on_color_click(event):
-    print("1")
     btn = pydom["#result"]
-    print("2")
     btn.style["background-color"] = f"#{random.randrange(0x1000000):06x}"
 
 
-def reset_color():
+@when("click", "#color-reset-button")
+def reset_color(*args, **kwargs):
     pydom["#result"].style["background-color"] = "white"
 
 

pyscript.core/test/pydom_mp.html

@@ -0,0 +1,19 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>PyDom Example (MicroPython)</title>
+  <link rel="stylesheet" href="../dist/core.css">
+  <script type="module" src="../dist/core.js"></script>
+</head>
+<body>
+  <script type="mpy" src="pydom.py"></script>
+
+  <button id="just-a-button">Click For Time</button>
+  <button id="color-button">Click For Color</button>
+  <button id="color-reset-button">Reset Color</button>
+
+  <div id="result"></div>
+</body>
+</html>

pyscript.core/test/pyscript_dom/index.html

@@ -1,6 +1,6 @@
 <html lang="en">
   <head>
-    <title>PyperCard PyTest Suite</title>
+    <title>PyDom Test Suite</title>
     <meta charset="UTF-8">
     <meta name="viewport" content="width=device-width,initial-scale=1">
     <link rel="stylesheet" href="../../dist/core.css">
@@ -32,7 +32,7 @@
      </style>
   </head>
   <body>
-    <script type="py" src="run_tests.py" config="tests.toml"></script>
+    <script type="py" src="./run_tests.py" config="./tests.toml"></script>
 
     <h1>pyscript.dom Tests</h1>
     <p>You can pass test parameters to this test suite by passing them as query params on the url.
@@ -98,6 +98,8 @@
       <p class="collection"></p>
       <div class="collection"></div>
       <h3 class="collection"></h3>
+
+      <div id="element_attribute_tests"></div>
     </div>
 
 

pyscript.core/test/pyscript_dom/tests/test_dom.py

@@ -163,6 +163,30 @@ class TestElement:
 
         assert called
 
+    def test_html_attribute(self):
+        # GIVEN an existing element on the page with a known empty text content
+        div = pydom["#element_attribute_tests"][0]
+
+        # WHEN we set the html attribute
+        div.html = "<b>New Content</b>"
+
+        # EXPECT the element html and underlying JS Element innerHTML property
+        # to match what we expect and what
+        assert div.html == div._js.innerHTML == "<b>New Content</b>"
+        assert div.text == div._js.textContent == "New Content"
+
+    def test_text_attribute(self):
+        # GIVEN an existing element on the page with a known empty text content
+        div = pydom["#element_attribute_tests"][0]
+
+        # WHEN we set the html attribute
+        div.text = "<b>New Content</b>"
+
+        # EXPECT the element html and underlying JS Element innerHTML property
+        # to match what we expect and what
+        assert div.html == div._js.innerHTML == "&lt;b&gt;New Content&lt;/b&gt;"
+        assert div.text == div._js.textContent == "<b>New Content</b>"
+
 
 class TestCollection:
     def test_iter_eq_children(self):
@@ -336,7 +360,7 @@ class TestSelect:
         assert select.options[0].html == "Option 1"
 
         # WHEN we add another option (blank this time)
-        select.options.add()
+        select.options.add("")
 
         # EXPECT the select element to have 2 options
         assert len(select.options) == 2

pyscript.core/test/terminal.py

@@ -0,0 +1,8 @@
+from pyscript import document
+
+classList = document.documentElement.classList
+
+if not __terminal__:
+    classList.add("error")
+else:
+    classList.add("ok")

pyscript.core/tests/integration/support.py

@@ -17,6 +17,7 @@ from playwright.sync_api import Error as PlaywrightError
 
 ROOT = py.path.local(__file__).dirpath("..", "..", "..")
 BUILD = ROOT.join("pyscript.core").join("dist")
+TEST = ROOT.join("pyscript.core").join("test")
 
 
 def params_with_marks(params):
@@ -206,6 +207,14 @@ class PyScriptTest:
         self.tmpdir = tmpdir
         # create a symlink to BUILD inside tmpdir
         tmpdir.join("build").mksymlinkto(BUILD)
+        # create a symlink ALSO to dist folder so we can run the tests in
+        # the test folder
+        tmpdir.join("dist").mksymlinkto(BUILD)
+        # create a symlink to TEST inside tmpdir so we can run tests in that
+        # manual test folder
+        tmpdir.join("test").mksymlinkto(TEST)
+
+        # create a symlink to the favicon, so that we can use it in the HTML
         self.tmpdir.chdir()
         self.tmpdir.join("favicon.ico").write("")
         self.logger = logger

pyscript.core/tests/integration/test_integration.py

@@ -0,0 +1,30 @@
+from .support import PyScriptTest, with_execution_thread
+
+
+@with_execution_thread(None)
+class TestSmokeTests(PyScriptTest):
+    """
+    Each example requires the same three tests:
+
+        - Test that the initial markup loads properly (currently done by
+          testing the <title> tag's content)
+        - Testing that pyscript is loading properly
+        - Testing that the page contains appropriate content after rendering
+    """
+
+    def test_pydom(self):
+        # Test the full pydom test suite by running it in the browser
+        self.goto("test/pyscript_dom/index.html?-v&-s")
+        assert self.page.title() == "PyDom Test Suite"
+
+        # wait for the test suite to finish
+        self.wait_for_console(
+            "============================= test session starts =============================="
+        )
+
+        self.assert_no_banners()
+
+        results = self.page.inner_html("#tests-terminal")
+        assert results
+        assert "PASSED" in results
+        assert "FAILED" not in results

pyscript.core/tests/integration/test_py_terminal.py

@@ -7,6 +7,7 @@ from .support import PageErrors, PyScriptTest, only_worker, skip_worker
 
 
 class TestPyTerminal(PyScriptTest):
+    @skip_worker("We do support multiple worker terminal now")
     def test_multiple_terminals(self):
         """
         Multiple terminals are not currently supported
@@ -19,9 +20,9 @@ class TestPyTerminal(PyScriptTest):
             wait_for_pyscript=False,
             check_js_errors=False,
         )
-        assert self.assert_banner_message("You can use at most 1 terminal")
+        assert self.assert_banner_message("You can use at most 1 main terminal")
 
-        with pytest.raises(PageErrors, match="You can use at most 1 terminal"):
+        with pytest.raises(PageErrors, match="You can use at most 1 main terminal"):
             self.check_js_errors()
 
     # TODO: interactive shell still unclear
@@ -112,7 +113,7 @@ class TestPyTerminal(PyScriptTest):
         This test isn't meant to capture all of the behaviors of an xtermjs terminal;
         rather, it confirms with a few basic formatting sequences that (1) the xtermjs
         terminal is functioning/loaded correctly and (2) that output toward that terminal
-        isn't being escaped in a way that prevents it reacting to escape seqeunces. The
+        isn't being escaped in a way that prevents it reacting to escape sequences. The
         main goal is preventing regressions.
         """
         self.pyscript_run(

pyscript.core/types/core.d.ts

@@ -3,14 +3,26 @@ import TYPES from "./types.js";
  * A `Worker` facade able to bootstrap on the worker thread only a PyScript module.
  * @param {string} file the python file to run ina worker.
  * @param {{config?: string | object, async?: boolean}} [options] optional configuration for the worker.
- * @returns {Worker & {sync: ProxyHandler<object>}}
+ * @returns {Promise<Worker & {sync: object}>}
  */
 declare function exportedPyWorker(file: string, options?: {
     config?: string | object;
     async?: boolean;
-}): Worker & {
-    sync: ProxyHandler<object>;
-};
+}): Promise<Worker & {
+    sync: object;
+}>;
+/**
+ * A `Worker` facade able to bootstrap on the worker thread only a PyScript module.
+ * @param {string} file the python file to run ina worker.
+ * @param {{config?: string | object, async?: boolean}} [options] optional configuration for the worker.
+ * @returns {Promise<Worker & {sync: object}>}
+ */
+declare function exportedMPWorker(file: string, options?: {
+    config?: string | object;
+    async?: boolean;
+}): Promise<Worker & {
+    sync: object;
+}>;
 declare const exportedHooks: {
     main: {
         onWorker: Set<Function>;
@@ -26,7 +38,11 @@ declare const exportedHooks: {
     };
     worker: {
         onReady: Set<Function>;
-        onBeforeRun: Set<Function>;
+        onBeforeRun: Set<Function>; /**
+         * Given a generic DOM Element, tries to fetch the 'src' attribute, if present.
+         * It either throws an error if the 'src' can't be fetched or it returns a fallback
+         * content as source.
+         */
         onBeforeRunAsync: Set<Function>;
         onAfterRun: Set<Function>;
         onAfterRunAsync: Set<Function>;
@@ -39,4 +55,4 @@ declare const exportedHooks: {
 declare const exportedConfig: {};
 declare const exportedWhenDefined: (type: string) => Promise<any>;
 import sync from "./sync.js";
-export { TYPES, exportedPyWorker as PyWorker, exportedHooks as hooks, exportedConfig as config, exportedWhenDefined as whenDefined };
+export { TYPES, exportedPyWorker as PyWorker, exportedMPWorker as MPWorker, exportedHooks as hooks, exportedConfig as config, exportedWhenDefined as whenDefined };

pyscript.core/types/fetch.d.ts

@@ -8,5 +8,4 @@
  * @returns {Promise<Response>}
  */
 export function robustFetch(url: string, options?: Request): Promise<Response>;
-export { getText };
-import { getText } from "polyscript/exports";
+export function getText(response: Response): Promise<string>;

pyscript.core/types/stdlib/pyscript.d.ts

@@ -3,10 +3,13 @@ declare namespace _default {
         "__init__.py": string;
         "display.py": string;
         "event_handling.py": string;
+        "fetch.py": string;
         "magic_js.py": string;
         "util.py": string;
     };
     let pyweb: {
+        "__init__.py": string;
+        "media.py": string;
         "pydom.py": string;
     };
 }

pyscript.core/types/sync.d.ts

@@ -1,4 +1,5 @@
 declare namespace _default {
+    function is_pyterminal(): boolean;
     /**
      * 'Sleep' for the given number of seconds. Used to implement Python's time.sleep in Worker threads.
      * @param {number} seconds The number of seconds to sleep.

requirements.txt

@@ -1,13 +1,13 @@
-black
-isort
+black==23.11.0
+isort==5.12.0
 pytest==7.1.2
-pre-commit
+pre-commit==3.5.0
 playwright==1.33.0
 pytest-playwright==0.3.3
 pytest-xdist==3.3.0
-pexpect
+pexpect==4.9.0
 pyodide_py==0.24.1
-micropip
-toml
-numpy
-pillow
+micropip==0.5.0
+toml==0.10.2
+numpy==1.26.2
+pillow==10.1.0