Multiple Worker based Terminals (#1948)

Multiple Worker based Terminals

pyscript.core/package-lock.json

@@ -1,12 +1,12 @@
 {
     "name": "@pyscript/core",
-    "version": "0.3.19",
+    "version": "0.3.20",
     "lockfileVersion": 3,
     "requires": true,
     "packages": {
         "": {
             "name": "@pyscript/core",
-            "version": "0.3.19",
+            "version": "0.3.20",
             "license": "APACHE-2.0",
             "dependencies": {
                 "@ungap/with-resolvers": "^0.1.0",

pyscript.core/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@pyscript/core",
-    "version": "0.3.19",
+    "version": "0.3.20",
     "type": "module",
     "description": "PyScript",
     "module": "./index.js",

pyscript.core/src/plugins/py-terminal.js

@@ -14,31 +14,73 @@ const notifyAndThrow = (message) => {
     throw new Error(message);
 };
 
+const notParsedYet = (script) => !bootstrapped.has(script);
+
+const onceOnMain = ({ attributes: { worker } }) => !worker;
+
+const bootstrapped = new WeakSet();
+
+let addStyle = true;
+
+// this callback will be serialized as string and it never needs
+// to be invoked multiple times. Each xworker here is bootstrapped
+// only once thanks to the `sync.is_pyterminal()` check.
+const workerReady = ({ interpreter, io, run }, { sync }) => {
+    if (!sync.is_pyterminal()) return;
+
+    // in workers it's always safe to grab the polyscript currentScript
+    run("from polyscript.currentScript import terminal as __terminal__");
+
+    // This part is inevitably duplicated as external scope
+    // can't be reached by workers out of the box.
+    // The detail is that here we use sync though, not readline.
+    const decoder = new TextDecoder();
+    let data = "";
+    const generic = {
+        isatty: true,
+        write(buffer) {
+            data = decoder.decode(buffer);
+            sync.pyterminal_write(data);
+            return buffer.length;
+        },
+    };
+    interpreter.setStdout(generic);
+    interpreter.setStderr(generic);
+    interpreter.setStdin({
+        isatty: true,
+        stdin: () => sync.pyterminal_read(data),
+    });
+
+    io.stderr = (error) => {
+        sync.pyterminal_write(`${error.message || error}\n`);
+    };
+};
+
 const pyTerminal = async () => {
     const terminals = document.querySelectorAll(SELECTOR);
 
-    // no results will look further for runtime nodes
+    const unknown = [].filter.call(terminals, notParsedYet);
-    if (!terminals.length) return;
 
-    // if we arrived this far, let's drop the MutationObserver
+    // no results will look further for runtime nodes
-    // as we only support one terminal per page (right now).
+    if (!unknown.length) return;
-    mo.disconnect();
+    // early flag elements as known to avoid concurrent
+    // MutationObserver invokes of this async handler
+    else unknown.forEach(bootstrapped.add, bootstrapped);
 
     // we currently support only one terminal as in "classic"
-    if (terminals.length > 1) notifyAndThrow("You can use at most 1 terminal.");
+    if ([].filter.call(terminals, onceOnMain).length > 1)
-
+        notifyAndThrow("You can use at most 1 main terminal");
-    const [element] = terminals;
-    // hopefully to be removed in the near future!
-    if (element.matches('script[type="mpy"],mpy-script'))
-        notifyAndThrow("Unsupported terminal.");
 
     // import styles lazily
-    document.head.append(
+    if (addStyle) {
-        Object.assign(document.createElement("link"), {
+        addStyle = false;
-            rel: "stylesheet",
+        document.head.append(
-            href: new URL("./xterm.css", import.meta.url),
+            Object.assign(document.createElement("link"), {
-        }),
+                rel: "stylesheet",
-    );
+                href: new URL("./xterm.css", import.meta.url),
+            }),
+        );
+    }
 
     // lazy load these only when a valid terminal is found
     const [{ Terminal }, { Readline }, { FitAddon }] = await Promise.all([
@@ -47,136 +89,113 @@ const pyTerminal = async () => {
         import(/* webpackIgnore: true */ "../3rd-party/xterm_addon-fit.js"),
     ]);
 
-    const readline = new Readline();
+    for (const element of unknown) {
+        // hopefully to be removed in the near future!
+        if (element.matches('script[type="mpy"],mpy-script'))
+            notifyAndThrow("Unsupported terminal.");
 
-    // common main thread initialization for both worker
+        const readline = new Readline();
-    // or main case, bootstrapping the terminal on its target
-    const init = (options) => {
-        let target = element;
-        const selector = element.getAttribute("target");
-        if (selector) {
-            target =
-                document.getElementById(selector) ||
-                document.querySelector(selector);
-            if (!target) throw new Error(`Unknown target ${selector}`);
-        } else {
-            target = document.createElement("py-terminal");
-            target.style.display = "block";
-            element.after(target);
-        }
-        const terminal = new Terminal({
-            theme: {
-                background: "#191A19",
-                foreground: "#F5F2E7",
-            },
-            ...options,
-        });
-        const fitAddon = new FitAddon();
-        terminal.loadAddon(fitAddon);
-        terminal.loadAddon(readline);
-        terminal.open(target);
-        fitAddon.fit();
-        terminal.focus();
-        defineProperty(element, "terminal", { value: terminal });
-        return terminal;
-    };
 
-    // branch logic for the worker
+        // common main thread initialization for both worker
-    if (element.hasAttribute("worker")) {
+        // or main case, bootstrapping the terminal on its target
-        // when the remote thread onReady triggers:
+        const init = (options) => {
-        // setup the interpreter stdout and stderr
+            let target = element;
-        const workerReady = ({ interpreter, io, run }, { sync }) => {
+            const selector = element.getAttribute("target");
-            // in workers it's always safe to grab the polyscript currentScript
+            if (selector) {
-            run(
+                target =
-                "from polyscript.currentScript import terminal as __terminal__",
+                    document.getElementById(selector) ||
-            );
+                    document.querySelector(selector);
-            sync.pyterminal_drop_hooks();
+                if (!target) throw new Error(`Unknown target ${selector}`);
-
+            } else {
-            // This part is inevitably duplicated as external scope
+                target = document.createElement("py-terminal");
-            // can't be reached by workers out of the box.
+                target.style.display = "block";
-            // The detail is that here we use sync though, not readline.
+                element.after(target);
-            const decoder = new TextDecoder();
+            }
-            let data = "";
+            const terminal = new Terminal({
-            const generic = {
+                theme: {
-                isatty: true,
+                    background: "#191A19",
-                write(buffer) {
+                    foreground: "#F5F2E7",
-                    data = decoder.decode(buffer);
-                    sync.pyterminal_write(data);
-                    return buffer.length;
                 },
-            };
+                ...options,
-            interpreter.setStdout(generic);
-            interpreter.setStderr(generic);
-            interpreter.setStdin({
-                isatty: true,
-                stdin: () => sync.pyterminal_read(data),
             });
-
+            const fitAddon = new FitAddon();
-            io.stderr = (error) => {
+            terminal.loadAddon(fitAddon);
-                sync.pyterminal_write(`${error.message || error}\n`);
+            terminal.loadAddon(readline);
-            };
+            terminal.open(target);
+            fitAddon.fit();
+            terminal.focus();
+            defineProperty(element, "terminal", { value: terminal });
+            return terminal;
         };
 
-        // add a hook on the main thread to setup all sync helpers
+        // branch logic for the worker
-        // also bootstrapping the XTerm target on main
+        if (element.hasAttribute("worker")) {
-        hooks.main.onWorker.add(function worker(_, xworker) {
+            // add a hook on the main thread to setup all sync helpers
-            hooks.main.onWorker.delete(worker);
+            // also bootstrapping the XTerm target on main *BUT* ...
-            init({
+            hooks.main.onWorker.add(function worker(_, xworker) {
-                disableStdin: false,
+                // ... as multiple workers will add multiple callbacks
-                cursorBlink: true,
+                // be sure no xworker is ever initialized twice!
-                cursorStyle: "block",
+                if (bootstrapped.has(xworker)) return;
-            });
+                bootstrapped.add(xworker);
-            xworker.sync.pyterminal_read = readline.read.bind(readline);
-            xworker.sync.pyterminal_write = readline.write.bind(readline);
-            // allow a worker to drop main thread hooks ASAP
-            xworker.sync.pyterminal_drop_hooks = () => {
-                hooks.worker.onReady.delete(workerReady);
-            };
-        });
 
-        // setup remote thread JS/Python code for whenever the
+                // still cleanup this callback for future scripts/workers
-        // worker is ready to become a terminal
+                hooks.main.onWorker.delete(worker);
-        hooks.worker.onReady.add(workerReady);
-    } else {
-        // in the main case, just bootstrap XTerm without
-        // allowing any input as that's not possible / awkward
-        hooks.main.onReady.add(function main({ interpreter, io, run }) {
-            console.warn("py-terminal is read only on main thread");
-            hooks.main.onReady.delete(main);
 
-            // on main, it's easy to trash and clean the current terminal
+                init({
-            globalThis.__py_terminal__ = init({
+                    disableStdin: false,
-                disableStdin: true,
+                    cursorBlink: true,
-                cursorBlink: false,
+                    cursorStyle: "block",
-                cursorStyle: "underline",
+                });
-            });
-            run("from js import __py_terminal__ as __terminal__");
-            delete globalThis.__py_terminal__;
 
-            // This part is inevitably duplicated as external scope
+                xworker.sync.is_pyterminal = () => true;
-            // can't be reached by workers out of the box.
+                xworker.sync.pyterminal_read = readline.read.bind(readline);
-            // The detail is that here we use readline here, not sync.
+                xworker.sync.pyterminal_write = readline.write.bind(readline);
-            const decoder = new TextDecoder();
-            let data = "";
-            const generic = {
-                isatty: true,
-                write(buffer) {
-                    data = decoder.decode(buffer);
-                    readline.write(data);
-                    return buffer.length;
-                },
-            };
-            interpreter.setStdout(generic);
-            interpreter.setStderr(generic);
-            interpreter.setStdin({
-                isatty: true,
-                stdin: () => readline.read(data),
             });
 
-            io.stderr = (error) => {
+            // setup remote thread JS/Python code for whenever the
-                readline.write(`${error.message || error}\n`);
+            // worker is ready to become a terminal
-            };
+            hooks.worker.onReady.add(workerReady);
-        });
+        } else {
+            // in the main case, just bootstrap XTerm without
+            // allowing any input as that's not possible / awkward
+            hooks.main.onReady.add(function main({ interpreter, io, run }) {
+                console.warn("py-terminal is read only on main thread");
+                hooks.main.onReady.delete(main);
+
+                // on main, it's easy to trash and clean the current terminal
+                globalThis.__py_terminal__ = init({
+                    disableStdin: true,
+                    cursorBlink: false,
+                    cursorStyle: "underline",
+                });
+                run("from js import __py_terminal__ as __terminal__");
+                delete globalThis.__py_terminal__;
+
+                // This part is inevitably duplicated as external scope
+                // can't be reached by workers out of the box.
+                // The detail is that here we use readline here, not sync.
+                const decoder = new TextDecoder();
+                let data = "";
+                const generic = {
+                    isatty: true,
+                    write(buffer) {
+                        data = decoder.decode(buffer);
+                        readline.write(data);
+                        return buffer.length;
+                    },
+                };
+                interpreter.setStdout(generic);
+                interpreter.setStderr(generic);
+                interpreter.setStdin({
+                    isatty: true,
+                    stdin: () => readline.read(data),
+                });
+
+                io.stderr = (error) => {
+                    readline.write(`${error.message || error}\n`);
+                };
+            });
+        }
     }
 };
 

pyscript.core/src/sync.js

@@ -1,4 +1,7 @@
 export default {
+    // allow pyterminal checks to bootstrap
+    is_pyterminal: () => false,
+
     /**
      * 'Sleep' for the given number of seconds. Used to implement Python's time.sleep in Worker threads.
      * @param {number} seconds The number of seconds to sleep.

pyscript.core/test/mpy.spec.js

@@ -73,3 +73,8 @@ test('Pyodide + terminal on Worker', async ({ page }) => {
   await page.goto('http://localhost:8080/test/py-terminal-worker.html');
   await page.waitForSelector('html.ok');
 });
+
+test('Pyodide + multiple terminals via Worker', async ({ page }) => {
+  await page.goto('http://localhost:8080/test/py-terminals.html');
+  await page.waitForSelector('html.first.second');
+});

pyscript.core/test/py-terminal-worker.html

@@ -9,6 +9,7 @@
         <style>.xterm { padding: .5rem; }</style>
     </head>
     <body>
-        <py-script src="terminal.py" worker terminal></py-script>
+        <script type="py" src="terminal.py" worker terminal></script>
+        <script type="py" src="terminal.py" worker terminal></script>
     </body>
 </html>

pyscript.core/test/py-terminals.html

@@ -0,0 +1,27 @@
+<!doctype html>
+<html lang="en">
+    <head>
+        <meta charset="UTF-8" />
+        <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+        <title>PyTerminal Main</title>
+        <link rel="stylesheet" href="../dist/core.css">
+        <script type="module" src="../dist/core.js"></script>
+        <style>.xterm { padding: .5rem; }</style>
+    </head>
+    <body>
+        <script type="py" worker terminal>
+            from pyscript import document
+            document.documentElement.classList.add("first")
+
+            import code
+            code.interact()
+        </script>
+        <script type="py" worker terminal>
+            from pyscript import document
+            document.documentElement.classList.add("second")
+
+            import code
+            code.interact()
+        </script>
+    </body>
+</html>

pyscript.core/tests/integration/test_py_terminal.py

@@ -7,6 +7,7 @@ from .support import PageErrors, PyScriptTest, only_worker, skip_worker
 
 
 class TestPyTerminal(PyScriptTest):
+    @skip_worker("We do support multiple worker terminal now")
     def test_multiple_terminals(self):
         """
         Multiple terminals are not currently supported
@@ -19,9 +20,9 @@ class TestPyTerminal(PyScriptTest):
             wait_for_pyscript=False,
             check_js_errors=False,
         )
-        assert self.assert_banner_message("You can use at most 1 terminal")
+        assert self.assert_banner_message("You can use at most 1 main terminal")
 
-        with pytest.raises(PageErrors, match="You can use at most 1 terminal"):
+        with pytest.raises(PageErrors, match="You can use at most 1 main terminal"):
             self.check_js_errors()
 
     # TODO: interactive shell still unclear

pyscript.core/types/sync.d.ts

@@ -1,4 +1,5 @@
 declare namespace _default {
+    function is_pyterminal(): boolean;
     /**
      * 'Sleep' for the given number of seconds. Used to implement Python's time.sleep in Worker threads.
      * @param {number} seconds The number of seconds to sleep.