PyDom compatibility with MicroPython (#1954)

* fix pydom example

* fix the pydom test example to use a python syntax that works with MicroPython by replacing datetime

* add note about capturing errors importing when

* patch event_handler to handle compat with micropython

* turn pyweb into a package and remove hack to make pydom a sort of module with an ugly hack

* add pydom example using micropython

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* fix select element test

* change pydom test page to let pytest tests load it properly

* add missing folders to test dev server so it can run examples in the manual tests folder

* add pydom tests to the test suite as integration tests

* lint

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* improve fixes in event_handling

* change when decorator to actually dynamically fail in micropython and support handlers with or without arguments

* simplify when decorator code

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* add type declaration back for the MP use case

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* removed code to access pydom get index as I can't think of any proper use case

* remove old commented hack to replace pydom module with class

* fix examples title

* precommit checks

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.pre-commit-config.yaml

@@ -7,7 +7,7 @@ ci:
 default_stages: [commit]
 repos:
     - repo: https://github.com/pre-commit/pre-commit-hooks
-      rev: v4.4.0
+      rev: v4.5.0
       hooks:
           - id: check-builtin-literals
           - id: check-case-conflict
@@ -25,13 +25,13 @@ repos:
           - id: trailing-whitespace
 
     - repo: https://github.com/psf/black
-      rev: 23.1.0
+      rev: 23.11.0
       hooks:
           - id: black
             exclude: pyscript\.core/src/stdlib/pyscript/__init__\.py
 
     - repo: https://github.com/codespell-project/codespell
-      rev: v2.2.4
+      rev: v2.2.6
       hooks:
           - id: codespell # See 'pyproject.toml' for args
             exclude: \.js\.map$

LICENSE

@@ -186,7 +186,11 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+
+   Copyright (c) 2022-present, PyScript Development Team
+
+   Originated at Anaconda, Inc. in 2022
+
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

pyscript.core/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@pyscript/core",
-    "version": "0.3.9",
+    "version": "0.3.23",
     "type": "module",
     "description": "PyScript",
     "module": "./index.js",
@@ -42,18 +42,18 @@
     "dependencies": {
         "@ungap/with-resolvers": "^0.1.0",
         "basic-devtools": "^0.1.6",
-        "polyscript": "^0.6.2",
+        "polyscript": "^0.6.18",
         "sticky-module": "^0.1.1",
         "to-json-callback": "^0.1.1",
         "type-checked-collections": "^0.1.7"
     },
     "devDependencies": {
-        "@codemirror/commands": "^6.3.2",
+        "@codemirror/commands": "^6.3.3",
         "@codemirror/lang-python": "^6.1.3",
-        "@codemirror/language": "^6.9.3",
+        "@codemirror/language": "^6.10.0",
-        "@codemirror/state": "^6.3.3",
+        "@codemirror/state": "^6.4.0",
-        "@codemirror/view": "^6.22.1",
+        "@codemirror/view": "^6.23.1",
-        "@playwright/test": "^1.40.1",
+        "@playwright/test": "^1.41.1",
         "@rollup/plugin-commonjs": "^25.0.7",
         "@rollup/plugin-node-resolve": "^15.2.3",
         "@rollup/plugin-terser": "^0.4.4",
@@ -61,8 +61,8 @@
         "@xterm/addon-fit": "^0.9.0-beta.1",
         "chokidar": "^3.5.3",
         "codemirror": "^6.0.1",
-        "eslint": "^8.55.0",
+        "eslint": "^8.56.0",
-        "rollup": "^4.6.1",
+        "rollup": "^4.9.6",
         "rollup-plugin-postcss": "^4.0.2",
         "rollup-plugin-string": "^3.0.0",
         "static-handler": "^0.4.3",

pyscript.core/src/config.js

@@ -63,6 +63,9 @@ for (const [TYPE] of TYPES) {
     /** @type {Error | undefined} The error thrown when parsing the PyScript config, if any.*/
     let error;
 
+    /** @type {string | undefined} The `configURL` field to normalize all config operations as opposite of guessing it once resolved */
+    let configURL;
+
     let config,
         type,
         pyElement,
@@ -105,6 +108,7 @@ for (const [TYPE] of TYPES) {
     if (!error && config) {
         try {
             const { json, toml, text, url } = await configDetails(config, type);
+            if (url) configURL = new URL(url, location.href).href;
             config = text;
             if (json || type === "json") {
                 try {
@@ -146,7 +150,7 @@ for (const [TYPE] of TYPES) {
     // assign plugins as Promise.all only if needed
     plugins = Promise.all(toBeAwaited);
 
-    configs.set(TYPE, { config: parsed, plugins, error });
+    configs.set(TYPE, { config: parsed, configURL, plugins, error });
 }
 
 export default configs;

pyscript.core/src/core.js

@@ -26,33 +26,9 @@ import { ErrorCode } from "./exceptions.js";
 import { robustFetch as fetch, getText } from "./fetch.js";
 import { hooks, main, worker, codeFor, createFunction } from "./hooks.js";
 
-// allows lazy element features on code evaluation
-let currentElement;
-
 // generic helper to disambiguate between custom element and script
 const isScript = ({ tagName }) => tagName === "SCRIPT";
 
-let shouldRegister = true;
-const registerModule = ({ XWorker: $XWorker, interpreter, io }) => {
-    // automatically use the pyscript stderr (when/if defined)
-    // this defaults to console.error
-    function PyWorker(...args) {
-        const worker = $XWorker(...args);
-        worker.onerror = ({ error }) => io.stderr(error);
-        return worker;
-    }
-
-    // enrich the Python env with some JS utility for main
-    interpreter.registerJsModule("_pyscript", {
-        PyWorker,
-        get target() {
-            return isScript(currentElement)
-                ? currentElement.target.id
-                : currentElement.id;
-        },
-    });
-};
-
 // avoid multiple initialization of the same library
 const [
     {
@@ -88,7 +64,7 @@ for (const [TYPE, interpreter] of TYPES) {
         else dispatch(element, TYPE, "done");
     };
 
-    const { config, plugins, error } = configs.get(TYPE);
+    const { config, configURL, plugins, error } = configs.get(TYPE);
 
     // create a unique identifier when/if needed
     let id = 0;
@@ -118,6 +94,36 @@ for (const [TYPE, interpreter] of TYPES) {
         return code;
     };
 
+    // register once any interpreter
+    let alreadyRegistered = false;
+
+    // allows lazy element features on code evaluation
+    let currentElement;
+
+    const registerModule = ({ XWorker, interpreter, io }) => {
+        // avoid multiple registration of the same interpreter
+        if (alreadyRegistered) return;
+        alreadyRegistered = true;
+
+        // automatically use the pyscript stderr (when/if defined)
+        // this defaults to console.error
+        function PyWorker(...args) {
+            const worker = XWorker(...args);
+            worker.onerror = ({ error }) => io.stderr(error);
+            return worker;
+        }
+
+        // enrich the Python env with some JS utility for main
+        interpreter.registerJsModule("_pyscript", {
+            PyWorker,
+            get target() {
+                return isScript(currentElement)
+                    ? currentElement.target.id
+                    : currentElement.id;
+            },
+        });
+    };
+
     // define the module as both `<script type="py">` and `<py-script>`
     // but only if the config didn't throw an error
     if (!error) {
@@ -133,10 +139,7 @@ for (const [TYPE, interpreter] of TYPES) {
                 main: {
                     ...codeFor(main),
                     async onReady(wrap, element) {
-                        if (shouldRegister) {
+                        registerModule(wrap);
-                            shouldRegister = false;
-                            registerModule(wrap);
-                        }
 
                         // allows plugins to do whatever they want with the element
                         // before regular stuff happens in here
@@ -256,6 +259,7 @@ for (const [TYPE, interpreter] of TYPES) {
 
             define(TYPE, {
                 config,
+                configURL,
                 interpreter,
                 hooks,
                 env: `${TYPE}-script`,
@@ -320,7 +324,7 @@ for (const [TYPE, interpreter] of TYPES) {
 function PyWorker(file, options) {
     const hooks = hooked.get("py");
     // this propagates pyscript worker hooks without needing a pyscript
-    // bootstrap + it passes arguments and enforces `pyodide`
+    // bootstrap + it passes arguments and it defaults to `pyodide`
     // as the interpreter to use in the worker, as all hooks assume that
     // and as `pyodide` is the only default interpreter that can deal with
     // all the features we need to deliver pyscript out there.

pyscript.core/src/plugins/py-editor.js

@@ -200,6 +200,9 @@ const init = async (script, type, interpreter) => {
 // avoid too greedy MutationObserver operations at distance
 let timeout = 0;
 
+// avoid delayed initialization
+let queue = Promise.resolve();
+
 // reset interval value then check for new scripts
 const resetTimeout = () => {
     timeout = 0;
@@ -213,11 +216,14 @@ const pyEditor = async () => {
     for (const [type, interpreter] of TYPES) {
         const selector = `script[type="${type}-editor"]`;
         for (const script of document.querySelectorAll(selector)) {
-            // avoid any further bootstrap
+            // avoid any further bootstrap by changing the type as active
             script.type += "-active";
-            await init(script, type, interpreter);
+            // don't await in here or multiple calls might happen
+            // while the first script is being initialized
+            queue = queue.then(() => init(script, type, interpreter));
         }
     }
+    return queue;
 };
 
 new MutationObserver(pyEditor).observe(document, {

pyscript.core/src/plugins/py-terminal.js

@@ -1,6 +1,7 @@
 // PyScript py-terminal plugin
 import { TYPES, hooks } from "../core.js";
 import { notify } from "./error.js";
+import { defineProperty } from "polyscript/exports";
 
 const SELECTOR = [...TYPES.keys()]
     .map((type) => `script[type="${type}"][terminal],${type}-script[terminal]`)
@@ -13,31 +14,73 @@ const notifyAndThrow = (message) => {
     throw new Error(message);
 };
 
+const notParsedYet = (script) => !bootstrapped.has(script);
+
+const onceOnMain = ({ attributes: { worker } }) => !worker;
+
+const bootstrapped = new WeakSet();
+
+let addStyle = true;
+
+// this callback will be serialized as string and it never needs
+// to be invoked multiple times. Each xworker here is bootstrapped
+// only once thanks to the `sync.is_pyterminal()` check.
+const workerReady = ({ interpreter, io, run }, { sync }) => {
+    if (!sync.is_pyterminal()) return;
+
+    // in workers it's always safe to grab the polyscript currentScript
+    run("from polyscript.currentScript import terminal as __terminal__");
+
+    // This part is inevitably duplicated as external scope
+    // can't be reached by workers out of the box.
+    // The detail is that here we use sync though, not readline.
+    const decoder = new TextDecoder();
+    let data = "";
+    const generic = {
+        isatty: true,
+        write(buffer) {
+            data = decoder.decode(buffer);
+            sync.pyterminal_write(data);
+            return buffer.length;
+        },
+    };
+    interpreter.setStdout(generic);
+    interpreter.setStderr(generic);
+    interpreter.setStdin({
+        isatty: true,
+        stdin: () => sync.pyterminal_read(data),
+    });
+
+    io.stderr = (error) => {
+        sync.pyterminal_write(`${error.message || error}\n`);
+    };
+};
+
 const pyTerminal = async () => {
     const terminals = document.querySelectorAll(SELECTOR);
 
-    // no results will look further for runtime nodes
+    const unknown = [].filter.call(terminals, notParsedYet);
-    if (!terminals.length) return;
 
-    // if we arrived this far, let's drop the MutationObserver
+    // no results will look further for runtime nodes
-    // as we only support one terminal per page (right now).
+    if (!unknown.length) return;
-    mo.disconnect();
+    // early flag elements as known to avoid concurrent
+    // MutationObserver invokes of this async handler
+    else unknown.forEach(bootstrapped.add, bootstrapped);
 
     // we currently support only one terminal as in "classic"
-    if (terminals.length > 1) notifyAndThrow("You can use at most 1 terminal.");
+    if ([].filter.call(terminals, onceOnMain).length > 1)
-
+        notifyAndThrow("You can use at most 1 main terminal");
-    const [element] = terminals;
-    // hopefully to be removed in the near future!
-    if (element.matches('script[type="mpy"],mpy-script'))
-        notifyAndThrow("Unsupported terminal.");
 
     // import styles lazily
-    document.head.append(
+    if (addStyle) {
-        Object.assign(document.createElement("link"), {
+        addStyle = false;
-            rel: "stylesheet",
+        document.head.append(
-            href: new URL("./xterm.css", import.meta.url),
+            Object.assign(document.createElement("link"), {
-        }),
+                rel: "stylesheet",
-    );
+                href: new URL("./xterm.css", import.meta.url),
+            }),
+        );
+    }
 
     // lazy load these only when a valid terminal is found
     const [{ Terminal }, { Readline }, { FitAddon }] = await Promise.all([
@@ -46,100 +89,113 @@ const pyTerminal = async () => {
         import(/* webpackIgnore: true */ "../3rd-party/xterm_addon-fit.js"),
     ]);
 
-    const readline = new Readline();
+    for (const element of unknown) {
+        // hopefully to be removed in the near future!
+        if (element.matches('script[type="mpy"],mpy-script'))
+            notifyAndThrow("Unsupported terminal.");
 
-    // common main thread initialization for both worker
+        const readline = new Readline();
-    // or main case, bootstrapping the terminal on its target
-    const init = (options) => {
-        let target = element;
-        const selector = element.getAttribute("target");
-        if (selector) {
-            target =
-                document.getElementById(selector) ||
-                document.querySelector(selector);
-            if (!target) throw new Error(`Unknown target ${selector}`);
-        } else {
-            target = document.createElement("py-terminal");
-            target.style.display = "block";
-            element.after(target);
-        }
-        const terminal = new Terminal({
-            theme: {
-                background: "#191A19",
-                foreground: "#F5F2E7",
-            },
-            ...options,
-        });
-        const fitAddon = new FitAddon();
-        terminal.loadAddon(fitAddon);
-        terminal.loadAddon(readline);
-        terminal.open(target);
-        fitAddon.fit();
-        terminal.focus();
-    };
 
-    // branch logic for the worker
+        // common main thread initialization for both worker
-    if (element.hasAttribute("worker")) {
+        // or main case, bootstrapping the terminal on its target
-        // when the remote thread onReady triggers:
+        const init = (options) => {
-        // setup the interpreter stdout and stderr
+            let target = element;
-        const workerReady = ({ interpreter }, { sync }) => {
+            const selector = element.getAttribute("target");
-            sync.pyterminal_drop_hooks();
+            if (selector) {
-            const decoder = new TextDecoder();
+                target =
-            let data = "";
+                    document.getElementById(selector) ||
-            const generic = {
+                    document.querySelector(selector);
-                isatty: true,
+                if (!target) throw new Error(`Unknown target ${selector}`);
-                write(buffer) {
+            } else {
-                    data = decoder.decode(buffer);
+                target = document.createElement("py-terminal");
-                    sync.pyterminal_write(data);
+                target.style.display = "block";
-                    return buffer.length;
+                element.after(target);
+            }
+            const terminal = new Terminal({
+                theme: {
+                    background: "#191A19",
+                    foreground: "#F5F2E7",
                 },
-            };
+                ...options,
-            interpreter.setStdout(generic);
-            interpreter.setStderr(generic);
-            interpreter.setStdin({
-                isatty: true,
-                stdin: () => sync.pyterminal_read(data),
             });
+            const fitAddon = new FitAddon();
+            terminal.loadAddon(fitAddon);
+            terminal.loadAddon(readline);
+            terminal.open(target);
+            fitAddon.fit();
+            terminal.focus();
+            defineProperty(element, "terminal", { value: terminal });
+            return terminal;
         };
 
-        // add a hook on the main thread to setup all sync helpers
+        // branch logic for the worker
-        // also bootstrapping the XTerm target on main
+        if (element.hasAttribute("worker")) {
-        hooks.main.onWorker.add(function worker(_, xworker) {
+            // add a hook on the main thread to setup all sync helpers
-            hooks.main.onWorker.delete(worker);
+            // also bootstrapping the XTerm target on main *BUT* ...
-            init({
+            hooks.main.onWorker.add(function worker(_, xworker) {
-                disableStdin: false,
+                // ... as multiple workers will add multiple callbacks
-                cursorBlink: true,
+                // be sure no xworker is ever initialized twice!
-                cursorStyle: "block",
+                if (bootstrapped.has(xworker)) return;
-            });
+                bootstrapped.add(xworker);
-            xworker.sync.pyterminal_read = readline.read.bind(readline);
-            xworker.sync.pyterminal_write = readline.write.bind(readline);
-            // allow a worker to drop main thread hooks ASAP
-            xworker.sync.pyterminal_drop_hooks = () => {
-                hooks.worker.onReady.delete(workerReady);
-            };
-        });
 
-        // setup remote thread JS/Python code for whenever the
+                // still cleanup this callback for future scripts/workers
-        // worker is ready to become a terminal
+                hooks.main.onWorker.delete(worker);
-        hooks.worker.onReady.add(workerReady);
+
-    } else {
+                init({
-        // in the main case, just bootstrap XTerm without
+                    disableStdin: false,
-        // allowing any input as that's not possible / awkward
+                    cursorBlink: true,
-        hooks.main.onReady.add(function main({ io }) {
+                    cursorStyle: "block",
-            console.warn("py-terminal is read only on main thread");
+                });
-            hooks.main.onReady.delete(main);
+
-            init({
+                xworker.sync.is_pyterminal = () => true;
-                disableStdin: true,
+                xworker.sync.pyterminal_read = readline.read.bind(readline);
-                cursorBlink: false,
+                xworker.sync.pyterminal_write = readline.write.bind(readline);
-                cursorStyle: "underline",
             });
-            io.stdout = (value) => {
+
-                readline.write(`${value}\n`);
+            // setup remote thread JS/Python code for whenever the
-            };
+            // worker is ready to become a terminal
-            io.stderr = (error) => {
+            hooks.worker.onReady.add(workerReady);
-                readline.write(`${error.message || error}\n`);
+        } else {
-            };
+            // in the main case, just bootstrap XTerm without
-        });
+            // allowing any input as that's not possible / awkward
+            hooks.main.onReady.add(function main({ interpreter, io, run }) {
+                console.warn("py-terminal is read only on main thread");
+                hooks.main.onReady.delete(main);
+
+                // on main, it's easy to trash and clean the current terminal
+                globalThis.__py_terminal__ = init({
+                    disableStdin: true,
+                    cursorBlink: false,
+                    cursorStyle: "underline",
+                });
+                run("from js import __py_terminal__ as __terminal__");
+                delete globalThis.__py_terminal__;
+
+                // This part is inevitably duplicated as external scope
+                // can't be reached by workers out of the box.
+                // The detail is that here we use readline here, not sync.
+                const decoder = new TextDecoder();
+                let data = "";
+                const generic = {
+                    isatty: true,
+                    write(buffer) {
+                        data = decoder.decode(buffer);
+                        readline.write(data);
+                        return buffer.length;
+                    },
+                };
+                interpreter.setStdout(generic);
+                interpreter.setStderr(generic);
+                interpreter.setStdin({
+                    isatty: true,
+                    stdin: () => readline.read(data),
+                });
+
+                io.stderr = (error) => {
+                    readline.write(`${error.message || error}\n`);
+                };
+            });
+        }
     }
 };
 

pyscript.core/src/stdlib/pyscript/__init__.py

@@ -43,6 +43,8 @@ from pyscript.magic_js import (
 try:
     from pyscript.event_handling import when
 except:
+    # TODO: should we remove this? Or at the very least, we should capture
+    # the traceback otherwise it's very hard to debug
     from pyscript.util import NotSupported
 
     when = NotSupported(

pyscript.core/src/stdlib/pyscript/event_handling.py

@@ -1,6 +1,14 @@
 import inspect
 
-from pyodide.ffi.wrappers import add_event_listener
+try:
+    from pyodide.ffi.wrappers import add_event_listener
+
+except ImportError:
+
+    def add_event_listener(el, event_type, func):
+        el.addEventListener(event_type, func)
+
+
 from pyscript.magic_js import document
 
 
@@ -27,19 +35,32 @@ def when(event_type=None, selector=None):
                     f"Invalid selector: {selector}. Selector must"
                     " be a string, a pydom.Element or a pydom.ElementCollection."
                 )
+        try:
+            sig = inspect.signature(func)
+            # Function doesn't receive events
+            if not sig.parameters:
 
-        sig = inspect.signature(func)
+                def wrapper(*args, **kwargs):
-        # Function doesn't receive events
+                    func()
-        if not sig.parameters:
 
+            else:
+                wrapper = func
+
+        except AttributeError:
+            # TODO: this is currently an quick hack to get micropython working but we need
+            #       to actually properly replace inspect.signature with something else
             def wrapper(*args, **kwargs):
-                func()
+                try:
+                    return func(*args, **kwargs)
+                except TypeError as e:
+                    if "takes 0 positional arguments" in str(e):
+                        return func()
+
+                    raise
+
+        for el in elements:
+            add_event_listener(el, event_type, wrapper)
 
-            for el in elements:
-                add_event_listener(el, event_type, wrapper)
-        else:
-            for el in elements:
-                add_event_listener(el, event_type, func)
         return func
 
     return decorator

pyscript.core/src/stdlib/pyscript/magic_js.py

@@ -1,9 +1,28 @@
+import sys
+
 import js as globalThis
 from polyscript import js_modules
 from pyscript.util import NotSupported
 
 RUNNING_IN_WORKER = not hasattr(globalThis, "document")
 
+
+# allow `from pyscript.js_modules.xxx import yyy`
+class JSModule:
+    def __init__(self, name):
+        self.name = name
+
+    def __getattr__(self, field):
+        # avoid pyodide looking for non existent fields
+        if not field.startswith("_"):
+            return getattr(getattr(js_modules, self.name), field)
+
+
+# generate N modules in the system that will proxy the real value
+for name in globalThis.Reflect.ownKeys(js_modules):
+    sys.modules[f"pyscript.js_modules.{name}"] = JSModule(name)
+sys.modules["pyscript.js_modules"] = js_modules
+
 if RUNNING_IN_WORKER:
     import js
     import polyscript

pyscript.core/src/stdlib/pyweb/__init__.py

@@ -0,0 +1 @@
+from .pydom import dom as pydom

pyscript.core/src/stdlib/pyweb/media.py

@@ -0,0 +1,95 @@
+from pyodide.ffi import to_js
+from pyscript import window
+
+
+class Device:
+    """Device represents a media input or output device, such as a microphone,
+    camera, or headset.
+    """
+
+    def __init__(self, device):
+        self._js = device
+
+    @property
+    def id(self):
+        return self._js.deviceId
+
+    @property
+    def group(self):
+        return self._js.groupId
+
+    @property
+    def kind(self):
+        return self._js.kind
+
+    @property
+    def label(self):
+        return self._js.label
+
+    def __getitem__(self, key):
+        return getattr(self, key)
+
+    @classmethod
+    async def load(cls, audio=False, video=True):
+        """Load the device stream."""
+        options = window.Object.new()
+        options.audio = audio
+        if isinstance(video, bool):
+            options.video = video
+        else:
+            # TODO: Think this can be simplified but need to check it on the pyodide side
+
+            # TODO: this is pyodide specific. shouldn't be!
+            options.video = window.Object.new()
+            for k in video:
+                setattr(
+                    options.video,
+                    k,
+                    to_js(video[k], dict_converter=window.Object.fromEntries),
+                )
+
+        stream = await window.navigator.mediaDevices.getUserMedia(options)
+        return stream
+
+    async def get_stream(self):
+        key = self.kind.replace("input", "").replace("output", "")
+        options = {key: {"deviceId": {"exact": self.id}}}
+
+        return await self.load(**options)
+
+
+async def list_devices() -> list[dict]:
+    """
+    Return the list of the currently available media input and output devices,
+    such as microphones, cameras, headsets, and so forth.
+
+    Output:
+
+        list(dict) - list of dictionaries representing the available media devices.
+            Each dictionary has the following keys:
+            * deviceId: a string that is an identifier for the represented device
+                that is persisted across sessions. It is un-guessable by other
+                applications and unique to the origin of the calling application.
+                It is reset when the user clears cookies (for Private Browsing, a
+                different identifier is used that is not persisted across sessions).
+
+            * groupId: a string that is a group identifier. Two devices have the same
+                group identifier if they belong to the same physical device — for
+                example a monitor with both a built-in camera and a microphone.
+
+            * kind: an enumerated value that is either "videoinput", "audioinput"
+                or "audiooutput".
+
+            * label: a string describing this device (for example "External USB
+                Webcam").
+
+    Note: the returned list will omit any devices that are blocked by the document
+    Permission Policy: microphone, camera, speaker-selection (for output devices),
+    and so on. Access to particular non-default devices is also gated by the
+    Permissions API, and the list will omit devices for which the user has not
+    granted explicit permission.
+    """
+    # https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/enumerateDevices
+    return [
+        Device(obj) for obj in await window.navigator.mediaDevices.enumerateDevices()
+    ]

pyscript.core/src/stdlib/pyweb/pydom.py

@@ -1,9 +1,34 @@
-import sys
+try:
-import warnings
+    from typing import Any
-from functools import cached_property
+except ImportError:
-from typing import Any
+    Any = "Any"
+
+try:
+    import warnings
+except ImportError:
+    # TODO: For now it probably means we are in MicroPython. We should figure
+    # out the "right" way to handle this. For now we just ignore the warning
+    # and logging to console
+    class warnings:
+        @staticmethod
+        def warn(*args, **kwargs):
+            print("WARNING: ", *args, **kwargs)
+
+
+try:
+    from functools import cached_property
+except ImportError:
+    # TODO: same comment about micropython as above
+    cached_property = property
+
+try:
+    from pyodide.ffi import JsProxy
+except ImportError:
+    # TODO: same comment about micropython as above
+    def JsProxy(obj):
+        return obj
+
 
-from pyodide.ffi import JsProxy
 from pyscript import display, document, window
 
 alert = window.alert
@@ -204,6 +229,91 @@ class Element(BaseElement):
     def show_me(self):
         self._js.scrollIntoView()
 
+    def snap(
+        self,
+        to: BaseElement | str = None,
+        width: int | None = None,
+        height: int | None = None,
+    ):
+        """
+        Captures a snapshot of a video element. (Only available for video elements)
+
+        Inputs:
+
+            * to: element where to save the snapshot of the video frame to
+            * width: width of the image
+            * height: height of the image
+
+        Output:
+            (Element) canvas element where the video frame snapshot was drawn into
+        """
+        if self._js.tagName != "VIDEO":
+            raise AttributeError("Snap method is only available for video Elements")
+
+        if to is None:
+            canvas = self.create("canvas")
+            if width is None:
+                width = self._js.width
+            if height is None:
+                height = self._js.height
+            canvas._js.width = width
+            canvas._js.height = height
+
+        elif isistance(to, Element):
+            if to._js.tagName != "CANVAS":
+                raise TypeError("Element to snap to must a canvas.")
+            canvas = to
+        elif getattr(to, "tagName", "") == "CANVAS":
+            canvas = Element(to)
+        elif isinstance(to, str):
+            canvas = pydom[to][0]
+            if canvas._js.tagName != "CANVAS":
+                raise TypeError("Element to snap to must a be canvas.")
+
+        canvas.draw(self, width, height)
+
+        return canvas
+
+    def download(self, filename: str = "snapped.png") -> None:
+        """Download the current element (only available for canvas elements) with the filename
+        provided in input.
+
+        Inputs:
+            * filename (str): name of the file being downloaded
+
+        Output:
+            None
+        """
+        if self._js.tagName != "CANVAS":
+            raise AttributeError(
+                "The download method is only available for canvas Elements"
+            )
+
+        link = self.create("a")
+        link._js.download = filename
+        link._js.href = self._js.toDataURL()
+        link._js.click()
+
+    def draw(self, what, width, height):
+        """Draw `what` on the current element  (only available for canvas elements).
+
+        Inputs:
+
+            * what (canvas image source): An element to draw into the context. The specification permits any canvas
+                image source, specifically, an HTMLImageElement, an SVGImageElement, an HTMLVideoElement,
+                an HTMLCanvasElement, an ImageBitmap, an OffscreenCanvas, or a VideoFrame.
+        """
+        if self._js.tagName != "CANVAS":
+            raise AttributeError(
+                "The draw method is only available for canvas Elements"
+            )
+
+        if isinstance(what, Element):
+            what = what._js
+
+        # https://developer.mozilla.org/en-US/docs/Web/API/CanvasRenderingContext2D/drawImage
+        self._js.getContext("2d").drawImage(what, 0, 0, width, height)
+
 
 class OptionsProxy:
     """This class represents the options of a select element. It
@@ -276,7 +386,7 @@ class OptionsProxy:
         return self.options[key]
 
 
-class StyleProxy(dict):
+class StyleProxy:  # (dict):
     def __init__(self, element: Element) -> None:
         self._element = element
 
@@ -395,7 +505,7 @@ class ElementCollection:
 
 
 class DomScope:
-    def __getattr__(self, __name: str) -> Any:
+    def __getattr__(self, __name: str):
         element = document[f"#{__name}"]
         if element:
             return element[0]
@@ -409,7 +519,12 @@ class PyDom(BaseElement):
     ElementCollection = ElementCollection
 
     def __init__(self):
-        super().__init__(document)
+        # PyDom is a special case of BaseElement where we don't want to create a new JS element
+        # and it really doesn't have a need for styleproxy or parent to to call to __init__
+        # (which actually fails in MP for some reason)
+        self._js = document
+        self._parent = None
+        self._proxies = {}
         self.ids = DomScope()
         self.body = Element(document.body)
         self.head = Element(document.head)
@@ -418,10 +533,6 @@ class PyDom(BaseElement):
         return super().create(type_, is_child=False, classes=classes, html=html)
 
     def __getitem__(self, key):
-        if isinstance(key, int):
-            indices = range(*key.indices(len(self.list)))
-            return [self.list[i] for i in indices]
-
         elements = self._js.querySelectorAll(key)
         if not elements:
             return None
@@ -429,5 +540,3 @@ class PyDom(BaseElement):
 
 
 dom = PyDom()
-
-sys.modules[__name__] = dom

pyscript.core/src/sync.js

@@ -1,4 +1,7 @@
 export default {
+    // allow pyterminal checks to bootstrap
+    is_pyterminal: () => false,
+
     /**
      * 'Sleep' for the given number of seconds. Used to implement Python's time.sleep in Worker threads.
      * @param {number} seconds The number of seconds to sleep.

pyscript.core/test/camera.html

@@ -0,0 +1,24 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>PyScript Media Example</title>
+  <link rel="stylesheet" href="../dist/core.css">
+  <script type="module" src="../dist/core.js"></script>
+</head>
+<body>
+  <script type="py" src="camera.py" async></script>
+
+  <label for="cars">Choose a device:</label>
+
+  <select name="devices" id="devices"></select>
+
+  <button id="pick-device">Select the device</button>
+  <button id="snap">Snap</button>
+
+  <div id="result"></div>
+
+  <video id="video" width="600" height="400" autoplay></video>
+</body>
+</html>

pyscript.core/test/camera.py

@@ -0,0 +1,32 @@
+from pyodide.ffi import create_proxy
+from pyscript import display, document, when, window
+from pyweb import media, pydom
+
+devicesSelect = pydom["#devices"][0]
+video = pydom["video"][0]
+devices = {}
+
+
+async def list_media_devices(event=None):
+    """List the available media devices."""
+    global devices
+    for i, device in enumerate(await media.list_devices()):
+        devices[device.id] = device
+        label = f"{i} - ({device.kind}) {device.label} [{device.id}]"
+        devicesSelect.options.add(value=device.id, html=label)
+
+
+@when("click", "#pick-device")
+async def connect_to_device(e):
+    """Connect to the selected device."""
+    device = devices[devicesSelect.value]
+    video._js.srcObject = await device.get_stream()
+
+
+@when("click", "#snap")
+async def camera_click(e):
+    """Take a picture and download it."""
+    video.snap().download()
+
+
+await list_media_devices()

pyscript.core/test/config-url.html

@@ -6,6 +6,12 @@
   <title>PyScript Next Plugin</title>
   <link rel="stylesheet" href="../dist/core.css">
   <script type="module" src="../dist/core.js"></script>
-  <py-config src="bad.toml" type="toml"></py-config>
+  <mpy-config src="config-url/config.json"></mpy-config>
+  <script type="mpy">
+    from runtime import test
+  </script>
+  <script type="mpy" worker>
+    from runtime import test
+  </script>
 </head>
 </html>

pyscript.core/test/config-url/config.json

@@ -0,0 +1,7 @@
+{
+    "files":{
+        "{FROM}": "./src",
+        "{TO}": "./runtime",
+        "{FROM}/test.py": "{TO}/test.py"
+    }
+}

pyscript.core/test/config-url/src/test.py

@@ -0,0 +1,8 @@
+from pyscript import RUNNING_IN_WORKER, document
+
+classList = document.documentElement.classList
+
+if RUNNING_IN_WORKER:
+    classList.add("worker")
+else:
+    classList.add("main")

pyscript.core/test/js_modules.html

@@ -0,0 +1,39 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <link rel="stylesheet" href="../dist/core.css">
+    <script type="module" src="../dist/core.js"></script>
+</head>
+<body>
+    <mpy-config>
+        [js_modules.main]
+        "./js_modules.js" = "random_js"
+    </mpy-config>
+    <mpy-script>
+        from pyscript.js_modules.random_js import default as value
+        from pyscript.js_modules import random_js
+        from pyscript import js_modules
+
+        print("mpy", value)
+        print("mpy", random_js.default)
+        print("mpy", js_modules.random_js.default)
+    </mpy-script>
+    <py-config>
+        [js_modules.main]
+        "./js_modules.js" = "random_js"
+    </py-config>
+    <py-script>
+        from pyscript.js_modules.random_js import default as value
+        from pyscript.js_modules import random_js
+        from pyscript import js_modules, document
+
+        print("py", value)
+        print("py", random_js.default)
+        print("py", js_modules.random_js.default)
+
+        document.documentElement.classList.add('done')
+    </py-script>
+</body>
+</html>

pyscript.core/test/js_modules.js

@@ -0,0 +1 @@
+export default Math.random();

pyscript.core/test/mpy.spec.js

@@ -35,3 +35,46 @@ test('MicroPython hooks', async ({ page }) => {
     'worker onAfterRun',
   ].join('\n'));
 });
+
+test('MicroPython + Pyodide js_modules', async ({ page }) => {
+  const logs = [];
+  page.on('console', msg => {
+    const text = msg.text();
+    if (!text.startsWith('['))
+      logs.push(text);
+  });
+  await page.goto('http://localhost:8080/test/js_modules.html');
+  await page.waitForSelector('html.done');
+  await expect(logs.length).toBe(6);
+  await expect(logs[0]).toBe(logs[1]);
+  await expect(logs[1]).toBe(logs[2]);
+  await expect(logs[3]).toBe(logs[4]);
+  await expect(logs[4]).toBe(logs[5]);
+});
+
+test('MicroPython + configURL', async ({ page }) => {
+  const logs = [];
+  page.on('console', msg => {
+    const text = msg.text();
+    if (!text.startsWith('['))
+      logs.push(text);
+  });
+  await page.goto('http://localhost:8080/test/config-url.html');
+  await page.waitForSelector('html.main.worker');
+});
+
+test('Pyodide + terminal on Main', async ({ page }) => {
+  await page.goto('http://localhost:8080/test/py-terminal-main.html');
+  await page.waitForSelector('html.ok');
+});
+
+
+test('Pyodide + terminal on Worker', async ({ page }) => {
+  await page.goto('http://localhost:8080/test/py-terminal-worker.html');
+  await page.waitForSelector('html.ok');
+});
+
+test('Pyodide + multiple terminals via Worker', async ({ page }) => {
+  await page.goto('http://localhost:8080/test/py-terminals.html');
+  await page.waitForSelector('html.first.second');
+});

pyscript.core/test/multi.html

@@ -0,0 +1,20 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <script type="module" src="../dist/core.js"></script>
+  </head>
+  <body>
+    <script type="mpy">
+      from pyscript import document
+      import sys
+      document.body.append(sys.version)
+    </script>
+    <script type="py">
+      from pyscript import document
+      import sys
+      document.body.append(sys.version)
+    </script>
+  </body>
+</html>

pyscript.core/test/multiple-editors.html

@@ -0,0 +1,31 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width,initial-scale=1">
+  <title>PyScript Test</title>
+  <link rel="stylesheet" href="../dist/core.css">
+  <script type="module" src="../dist/core.js"></script>
+</head>
+<body>
+  <script type="py-editor">
+    0
+  </script>
+  <script type="py-editor">
+    1
+  </script>
+  <script type="py-editor">
+    2
+  </script>
+  <script type="py-editor">
+    3
+  </script>
+  <script type="py-editor">
+    4
+  </script>
+  <script type="py-editor">
+    5
+  </script>
+  <!-- more... -->
+</body>
+</html>

pyscript.core/test/py-terminal-main.html

@@ -0,0 +1,14 @@
+<!doctype html>
+<html lang="en">
+    <head>
+        <meta charset="UTF-8" />
+        <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+        <title>PyTerminal Main</title>
+        <link rel="stylesheet" href="../dist/core.css">
+        <script type="module" src="../dist/core.js"></script>
+        <style>.xterm { padding: .5rem; }</style>
+    </head>
+    <body>
+        <py-script src="terminal.py" terminal></py-script>
+    </body>
+</html>

pyscript.core/test/py-terminal-worker.html

@@ -0,0 +1,15 @@
+<!doctype html>
+<html lang="en">
+    <head>
+        <meta charset="UTF-8" />
+        <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+        <title>PyTerminal Main</title>
+        <link rel="stylesheet" href="../dist/core.css">
+        <script type="module" src="../dist/core.js"></script>
+        <style>.xterm { padding: .5rem; }</style>
+    </head>
+    <body>
+        <script type="py" src="terminal.py" worker terminal></script>
+        <script type="py" src="terminal.py" worker terminal></script>
+    </body>
+</html>

pyscript.core/test/py-terminal.html

@@ -14,6 +14,9 @@
                 print('hello world')
         </script>
         <py-script worker terminal>
+            # works on both worker and main scripts
+            print("__terminal__", __terminal__)
+
             import sys
             from pyscript import display, document
             display("Hello", "PyScript Next - PyTerminal", append=False)

pyscript.core/test/py-terminals.html

@@ -0,0 +1,27 @@
+<!doctype html>
+<html lang="en">
+    <head>
+        <meta charset="UTF-8" />
+        <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+        <title>PyTerminal Main</title>
+        <link rel="stylesheet" href="../dist/core.css">
+        <script type="module" src="../dist/core.js"></script>
+        <style>.xterm { padding: .5rem; }</style>
+    </head>
+    <body>
+        <script type="py" worker terminal>
+            from pyscript import document
+            document.documentElement.classList.add("first")
+
+            import code
+            code.interact()
+        </script>
+        <script type="py" worker terminal>
+            from pyscript import document
+            document.documentElement.classList.add("second")
+
+            import code
+            code.interact()
+        </script>
+    </body>
+</html>

pyscript.core/test/pydom.html

@@ -3,7 +3,7 @@
 <head>
   <meta charset="UTF-8">
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
-  <title>PyScript Next Plugin</title>
+  <title>PyDom Example</title>
   <link rel="stylesheet" href="../dist/core.css">
   <script type="module" src="../dist/core.js"></script>
 </head>

pyscript.core/test/pydom.py

@@ -1,26 +1,32 @@
 import random
+import time
 from datetime import datetime as dt
 
-from pyscript import display
+from pyscript import display, when
 from pyweb import pydom
-from pyweb.base import when
 
 
 @when("click", "#just-a-button")
-def on_click(event):
+def on_click():
-    print(f"Hello from Python! {dt.now()}")
+    try:
-    display(f"Hello from Python! {dt.now()}", append=False, target="result")
+        timenow = dt.now()
+    except NotImplementedError:
+        # In this case we assume it's not implemented because we are using MycroPython
+        tnow = time.localtime()
+        tstr = "{:02d}/{:02d}/{:04d} {:02d}:{:02d}:{:02d}"
+        timenow = tstr.format(tnow[2], tnow[1], tnow[0], *tnow[2:])
+
+    display(f"Hello from PyScript, time is: {timenow}", append=False, target="result")
 
 
 @when("click", "#color-button")
 def on_color_click(event):
-    print("1")
     btn = pydom["#result"]
-    print("2")
     btn.style["background-color"] = f"#{random.randrange(0x1000000):06x}"
 
 
-def reset_color():
+@when("click", "#color-reset-button")
+def reset_color(*args, **kwargs):
     pydom["#result"].style["background-color"] = "white"
 
 

pyscript.core/test/pydom_mp.html

@@ -0,0 +1,19 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>PyDom Example (MicroPython)</title>
+  <link rel="stylesheet" href="../dist/core.css">
+  <script type="module" src="../dist/core.js"></script>
+</head>
+<body>
+  <script type="mpy" src="pydom.py"></script>
+
+  <button id="just-a-button">Click For Time</button>
+  <button id="color-button">Click For Color</button>
+  <button id="color-reset-button">Reset Color</button>
+
+  <div id="result"></div>
+</body>
+</html>

pyscript.core/test/pyscript_dom/index.html

@@ -1,6 +1,6 @@
 <html lang="en">
   <head>
-    <title>PyperCard PyTest Suite</title>
+    <title>PyDom Test Suite</title>
     <meta charset="UTF-8">
     <meta name="viewport" content="width=device-width,initial-scale=1">
     <link rel="stylesheet" href="../../dist/core.css">
@@ -32,7 +32,7 @@
      </style>
   </head>
   <body>
-    <script type="py" src="run_tests.py" config="tests.toml"></script>
+    <script type="py" src="./run_tests.py" config="./tests.toml"></script>
 
     <h1>pyscript.dom Tests</h1>
     <p>You can pass test parameters to this test suite by passing them as query params on the url.

pyscript.core/test/pyscript_dom/tests/test_dom.py

@@ -336,7 +336,7 @@ class TestSelect:
         assert select.options[0].html == "Option 1"
 
         # WHEN we add another option (blank this time)
-        select.options.add()
+        select.options.add("")
 
         # EXPECT the select element to have 2 options
         assert len(select.options) == 2

pyscript.core/test/terminal.py

@@ -0,0 +1,8 @@
+from pyscript import document
+
+classList = document.documentElement.classList
+
+if not __terminal__:
+    classList.add("error")
+else:
+    classList.add("ok")

pyscript.core/tests/integration/support.py

@@ -17,6 +17,7 @@ from playwright.sync_api import Error as PlaywrightError
 
 ROOT = py.path.local(__file__).dirpath("..", "..", "..")
 BUILD = ROOT.join("pyscript.core").join("dist")
+TEST = ROOT.join("pyscript.core").join("test")
 
 
 def params_with_marks(params):
@@ -206,6 +207,14 @@ class PyScriptTest:
         self.tmpdir = tmpdir
         # create a symlink to BUILD inside tmpdir
         tmpdir.join("build").mksymlinkto(BUILD)
+        # create a symlink ALSO to dist folder so we can run the tests in
+        # the test folder
+        tmpdir.join("dist").mksymlinkto(BUILD)
+        # create a symlink to TEST inside tmpdir so we can run tests in that
+        # manual test folder
+        tmpdir.join("test").mksymlinkto(TEST)
+
+        # create a symlink to the favicon, so that we can use it in the HTML
         self.tmpdir.chdir()
         self.tmpdir.join("favicon.ico").write("")
         self.logger = logger

pyscript.core/tests/integration/test_integration.py

@@ -0,0 +1,30 @@
+from .support import PyScriptTest, with_execution_thread
+
+
+@with_execution_thread(None)
+class TestSmokeTests(PyScriptTest):
+    """
+    Each example requires the same three tests:
+
+        - Test that the initial markup loads properly (currently done by
+          testing the <title> tag's content)
+        - Testing that pyscript is loading properly
+        - Testing that the page contains appropriate content after rendering
+    """
+
+    def test_pydom(self):
+        # Test the full pydom test suite by running it in the browser
+        self.goto("test/pyscript_dom/index.html?-v&-s")
+        assert self.page.title() == "PyDom Test Suite"
+
+        # wait for the test suite to finish
+        self.wait_for_console(
+            "============================= test session starts =============================="
+        )
+
+        self.assert_no_banners()
+
+        results = self.page.inner_html("#tests-terminal")
+        assert results
+        assert "PASSED" in results
+        assert "FAILED" not in results

pyscript.core/tests/integration/test_py_terminal.py

@@ -7,6 +7,7 @@ from .support import PageErrors, PyScriptTest, only_worker, skip_worker
 
 
 class TestPyTerminal(PyScriptTest):
+    @skip_worker("We do support multiple worker terminal now")
     def test_multiple_terminals(self):
         """
         Multiple terminals are not currently supported
@@ -19,9 +20,9 @@ class TestPyTerminal(PyScriptTest):
             wait_for_pyscript=False,
             check_js_errors=False,
         )
-        assert self.assert_banner_message("You can use at most 1 terminal")
+        assert self.assert_banner_message("You can use at most 1 main terminal")
 
-        with pytest.raises(PageErrors, match="You can use at most 1 terminal"):
+        with pytest.raises(PageErrors, match="You can use at most 1 main terminal"):
             self.check_js_errors()
 
     # TODO: interactive shell still unclear
@@ -112,7 +113,7 @@ class TestPyTerminal(PyScriptTest):
         This test isn't meant to capture all of the behaviors of an xtermjs terminal;
         rather, it confirms with a few basic formatting sequences that (1) the xtermjs
         terminal is functioning/loaded correctly and (2) that output toward that terminal
-        isn't being escaped in a way that prevents it reacting to escape seqeunces. The
+        isn't being escaped in a way that prevents it reacting to escape sequences. The
         main goal is preventing regressions.
         """
         self.pyscript_run(

pyscript.core/types/stdlib/pyscript.d.ts

@@ -7,6 +7,8 @@ declare namespace _default {
         "util.py": string;
     };
     let pyweb: {
+        "__init__.py": string;
+        "media.py": string;
         "pydom.py": string;
     };
 }

pyscript.core/types/sync.d.ts

@@ -1,4 +1,5 @@
 declare namespace _default {
+    function is_pyterminal(): boolean;
     /**
      * 'Sleep' for the given number of seconds. Used to implement Python's time.sleep in Worker threads.
      * @param {number} seconds The number of seconds to sleep.

requirements.txt

@@ -1,13 +1,13 @@
-black
+black==23.11.0
-isort
+isort==5.12.0
 pytest==7.1.2
-pre-commit
+pre-commit==3.5.0
 playwright==1.33.0
 pytest-playwright==0.3.3
 pytest-xdist==3.3.0
-pexpect
+pexpect==4.9.0
 pyodide_py==0.24.1
-micropip
+micropip==0.5.0
-toml
+toml==0.10.2
-numpy
+numpy==1.26.2
-pillow
+pillow==10.1.0