[next] Bring in stdlib as artifact (#1666)

.pre-commit-config.yaml

@@ -27,7 +27,7 @@ repos:
       rev: v0.0.257
       hooks:
           - id: ruff
-            exclude: pyscript\.core/test|pyscript\.core/src/display.py
+            exclude: pyscript\.core/test|pyscript.core/src/stdlib/pyscript.py
             args: [--fix]
 
     - repo: https://github.com/psf/black
@@ -47,7 +47,7 @@ repos:
       rev: "v3.0.0-alpha.6"
       hooks:
           - id: prettier
-            exclude: pyscript\.core/test|pyscript\.core/core.*|pyscript\.core/types/|pyscript\.sw/
+            exclude: pyscript\.core/test|pyscript\.core/core.*|pyscript\.core/types/|pyscript.core/src/stdlib/pyscript.js|pyscript\.sw/
             args: [--tab-width, "4"]
 
     - repo: https://github.com/pre-commit/mirrors-eslint

pyscript.core/.npmignore

@@ -1,5 +1,7 @@
 node_modules/
 rollup/
 test/
+src/stdlib/_pyscript
+src/stdlib/pyscript.py
 package-lock.json
 tsconfig.json

pyscript.core/package.json

@@ -21,7 +21,7 @@
     },
     "scripts": {
         "server": "npx static-handler --cors --coep --coop --corp .",
-        "build": "rollup --config rollup/core.config.js && rollup --config rollup/core-css.config.js && npm run ts",
+        "build": "node rollup/stdlib.cjs && rollup --config rollup/core.config.js && rollup --config rollup/core-css.config.js && npm run ts",
         "ts": "tsc -p ."
     },
     "keywords": [

pyscript.core/rollup/core.config.js

@@ -3,14 +3,8 @@
 
 import { nodeResolve } from "@rollup/plugin-node-resolve";
 import terser from "@rollup/plugin-terser";
-import { string } from "rollup-plugin-string";
 
-const plugins = [
+const plugins = [];
-    string({
-        // Required to be specified
-        include: "**/*.py",
-    }),
-];
 
 export default {
     input: "./src/core.js",

pyscript.core/rollup/stdlib.cjs

@@ -0,0 +1,29 @@
+const {
+    readdirSync,
+    readFileSync,
+    statSync,
+    writeFileSync,
+} = require("node:fs");
+const { join } = require("node:path");
+
+const crawl = (path, json) => {
+    for (const file of readdirSync(path)) {
+        const full = join(path, file);
+        if (/\.py$/.test(file)) json[file] = readFileSync(full).toString();
+        else if (statSync(full).isDirectory() && !file.endsWith("_"))
+            crawl(full, (json[file] = {}));
+    }
+};
+
+const json = {};
+
+crawl(join(__dirname, "..", "src", "stdlib"), json);
+
+writeFileSync(
+    join(__dirname, "..", "src", "stdlib", "pyscript.js"),
+    `// ⚠️ This file is an artifact: DO NOT MODIFY\nexport default ${JSON.stringify(
+        json,
+        null,
+        "  ",
+    )};\n`,
+);

pyscript.core/src/core.js

@@ -4,8 +4,7 @@ import { define, XWorker } from "polyscript";
 import { htmlDecode } from "./utils.js";
 import sync from "./sync.js";
 
-// this is imported as string (via rollup)
+import stdlib from "./stdlib.js";
-import display from "./display.py";
 
 // TODO: this is not strictly polyscript related but handy ... not sure
 //       we should factor this utility out a part but this works anyway.
@@ -81,6 +80,7 @@ const bootstrapNodeAndPlugins = (pyodide, element, callback, hook) => {
     for (const fn of hooks[hook]) fn(pyodide, element);
 };
 
+let shouldRegister = true;
 const registerModule = ({ XWorker: $XWorker, interpreter, io }) => {
     // automatically use the pyscript stderr (when/if defined)
     // this defaults to console.error
@@ -89,43 +89,20 @@ const registerModule = ({ XWorker: $XWorker, interpreter, io }) => {
         worker.onerror = ({ error }) => io.stderr(error);
         return worker;
     }
-    // trap once the python `display` utility (borrowed from "classic PyScript")
+
-    // provide the regular Pyodide globals instead of those from xworker
+    // enrich the Python env with some JS utility for main
-    const pyDisplay = interpreter.runPython(
+    defineProperty(globalThis, "_pyscript", {
-        [
+        value: {
-            "import js as window",
-            "document=window.document",
-            display,
-            "display",
-        ].join("\n"),
-        // avoid leaking on global
-        { globals: interpreter.runPython("{}") },
-    );
-    interpreter.registerJsModule("pyscript", {
             PyWorker,
-        document,
+            get id() {
-        window,
+                return isScript(currentElement)
-        // a getter to ensure if multiple scripts with same
-        // env (py) runs, their execution code will have the correct
-        // display reference with automatic target
-        get display() {
-            const id = isScript(currentElement)
                     ? currentElement.target.id
                     : currentElement.id;
-
+            },
-            return (...args) => {
-                const last = args.at(-1);
-                let kw = { target: id, append: false };
-                if (
-                    typeof last === "object" &&
-                    last &&
-                    ("target" in last || "append" in last)
-                )
-                    kw = { ...kw, ...args.pop() };
-                pyDisplay.callKwargs(...args, kw);
-            };
         },
     });
+
+    interpreter.runPython(stdlib, { globals: interpreter.runPython("{}") });
 };
 
 export const hooks = {
@@ -150,25 +127,11 @@ export const hooks = {
     codeAfterRunWorkerAsync: new Set(),
 };
 
-const workerPyScriptModule = [
-    "from pathlib import Path as _Path",
-    `_Path("./pyscript.py").write_text(${JSON.stringify(
-        [
-            "from polyscript import xworker as _xworker",
-            "window=_xworker.window",
-            "document=window.document",
-            "sync=_xworker.sync",
-            display,
-        ].join("\n"),
-    )})`,
-    "del _Path",
-].join("\n");
-
 const workerHooks = {
     codeBeforeRunWorker: () =>
-        [workerPyScriptModule, ...hooks.codeBeforeRunWorker].join("\n"),
+        [stdlib, ...hooks.codeBeforeRunWorker].join("\n"),
     codeBeforeRunWorkerAsync: () =>
-        [workerPyScriptModule, ...hooks.codeBeforeRunWorkerAsync].join("\n"),
+        [stdlib, ...hooks.codeBeforeRunWorkerAsync].join("\n"),
     codeAfterRunWorker: () => [...hooks.codeAfterRunWorker].join("\n"),
     codeAfterRunWorkerAsync: () =>
         [...hooks.codeAfterRunWorkerAsync].join("\n"),
@@ -198,7 +161,10 @@ define("py", {
         bootstrapNodeAndPlugins(pyodide, element, after, "onAfterRunAsync");
     },
     async onInterpreterReady(pyodide, element) {
-        registerModule(pyodide, element);
+        if (shouldRegister) {
+            shouldRegister = false;
+            registerModule(pyodide);
+        }
         // allows plugins to do whatever they want with the element
         // before regular stuff happens in here
         for (const callback of hooks.onInterpreterReady)

pyscript.core/src/stdlib.js

@@ -0,0 +1,33 @@
+/**
+ * Create through Python the pyscript module through
+ * the artifact generated at build time.
+ * This the returned value is a string that must be used
+ * either before a worker execute code or when the module
+ * is registered on the main thread.
+ */
+
+import pyscript from "./stdlib/pyscript.js";
+
+const { entries } = Object;
+
+const python = ["from pathlib import Path as _Path"];
+
+const write = (base, literal) => {
+    for (const [key, value] of entries(literal)) {
+        const path = `_Path("${base}/${key}")`;
+        if (typeof value === "string") {
+            const code = JSON.stringify(value);
+            python.push(`${path}.write_text(${code})`);
+        } else {
+            python.push(`${path}.mkdir(parents=True, exist_ok=True)`);
+            write(`${base}/${key}`, value);
+        }
+    }
+};
+
+write(".", pyscript);
+
+python.push("del _Path");
+python.push("\n");
+
+export default python.join("\n");

pyscript.core/src/stdlib/_pyscript/__init__.py

@@ -0,0 +1,12 @@
+import js as window
+
+IS_WORKER = not hasattr(window, "document")
+
+if IS_WORKER:
+    from polyscript import xworker as _xworker
+
+    window = _xworker.window
+    document = window.document
+    sync = _xworker.sync
+else:
+    document = window.document

pyscript.core/src/stdlib/_pyscript/display.py

@@ -1,10 +1,9 @@
-# ⚠️ WARNING - both `document` and `window` are added at runtime
-
 import base64
 import html
 import io
 import re
 
+from . import document, window
 
 _MIME_METHODS = {
     "__repr__": "text/plain",

pyscript.core/src/stdlib/pyscript.js

@@ -0,0 +1,8 @@
+// ⚠️ This file is an artifact: DO NOT MODIFY
+export default {
+  "_pyscript": {
+    "__init__.py": "import js as window\n\nIS_WORKER = not hasattr(window, \"document\")\n\nif IS_WORKER:\n    from polyscript import xworker as _xworker\n\n    window = _xworker.window\n    document = window.document\n    sync = _xworker.sync\nelse:\n    document = window.document\n",
+    "display.py": "import base64\nimport html\nimport io\nimport re\n\nfrom . import document, window\n\n_MIME_METHODS = {\n    \"__repr__\": \"text/plain\",\n    \"_repr_html_\": \"text/html\",\n    \"_repr_markdown_\": \"text/markdown\",\n    \"_repr_svg_\": \"image/svg+xml\",\n    \"_repr_png_\": \"image/png\",\n    \"_repr_pdf_\": \"application/pdf\",\n    \"_repr_jpeg_\": \"image/jpeg\",\n    \"_repr_latex\": \"text/latex\",\n    \"_repr_json_\": \"application/json\",\n    \"_repr_javascript_\": \"application/javascript\",\n    \"savefig\": \"image/png\",\n}\n\n\ndef _render_image(mime, value, meta):\n    # If the image value is using bytes we should convert it to base64\n    # otherwise it will return raw bytes and the browser will not be able to\n    # render it.\n    if isinstance(value, bytes):\n        value = base64.b64encode(value).decode(\"utf-8\")\n\n    # This is the pattern of base64 strings\n    base64_pattern = re.compile(\n        r\"^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{2}==)?$\"\n    )\n    # If value doesn't match the base64 pattern we should encode it to base64\n    if len(value) > 0 and not base64_pattern.match(value):\n        value = base64.b64encode(value.encode(\"utf-8\")).decode(\"utf-8\")\n\n    data = f\"data:{mime};charset=utf-8;base64,{value}\"\n    attrs = \" \".join(['{k}=\"{v}\"' for k, v in meta.items()])\n    return f'<img src=\"{data}\" {attrs}></img>'\n\n\ndef _identity(value, meta):\n    return value\n\n\n_MIME_RENDERERS = {\n    \"text/plain\": html.escape,\n    \"text/html\": _identity,\n    \"image/png\": lambda value, meta: _render_image(\"image/png\", value, meta),\n    \"image/jpeg\": lambda value, meta: _render_image(\"image/jpeg\", value, meta),\n    \"image/svg+xml\": _identity,\n    \"application/json\": _identity,\n    \"application/javascript\": lambda value, meta: f\"<script>{value}<\\\\/script>\",\n}\n\n\ndef _eval_formatter(obj, print_method):\n    \"\"\"\n    Evaluates a formatter method.\n    \"\"\"\n    if print_method == \"__repr__\":\n        return repr(obj)\n    elif hasattr(obj, print_method):\n        if print_method == \"savefig\":\n            buf = io.BytesIO()\n            obj.savefig(buf, format=\"png\")\n            buf.seek(0)\n            return base64.b64encode(buf.read()).decode(\"utf-8\")\n        return getattr(obj, print_method)()\n    elif print_method == \"_repr_mimebundle_\":\n        return {}, {}\n    return None\n\n\ndef _format_mime(obj):\n    \"\"\"\n    Formats object using _repr_x_ methods.\n    \"\"\"\n    if isinstance(obj, str):\n        return html.escape(obj), \"text/plain\"\n\n    mimebundle = _eval_formatter(obj, \"_repr_mimebundle_\")\n    if isinstance(mimebundle, tuple):\n        format_dict, _ = mimebundle\n    else:\n        format_dict = mimebundle\n\n    output, not_available = None, []\n    for method, mime_type in reversed(_MIME_METHODS.items()):\n        if mime_type in format_dict:\n            output = format_dict[mime_type]\n        else:\n            output = _eval_formatter(obj, method)\n\n        if output is None:\n            continue\n        elif mime_type not in _MIME_RENDERERS:\n            not_available.append(mime_type)\n            continue\n        break\n    if output is None:\n        if not_available:\n            window.console.warn(\n                f\"Rendered object requested unavailable MIME renderers: {not_available}\"\n            )\n        output = repr(output)\n        mime_type = \"text/plain\"\n    elif isinstance(output, tuple):\n        output, meta = output\n    else:\n        meta = {}\n    return _MIME_RENDERERS[mime_type](output, meta), mime_type\n\n\ndef _write(element, value, append=False):\n    html, mime_type = _format_mime(value)\n    if html == \"\\\\n\":\n        return\n\n    if append:\n        out_element = document.createElement(\"div\")\n        element.append(out_element)\n    else:\n        out_element = element.lastElementChild\n        if out_element is None:\n            out_element = element\n\n    if mime_type in (\"application/javascript\", \"text/html\"):\n        script_element = document.createRange().createContextualFragment(html)\n        out_element.append(script_element)\n    else:\n        out_element.innerHTML = html\n\n\ndef display(*values, target=None, append=True):\n    element = document.getElementById(target)\n    for v in values:\n        _write(element, v, append=append)\n"
+  },
+  "pyscript.py": "# export only what we want to expose as `pyscript` module\n# but not what is WORKER/MAIN dependent\nfrom _pyscript import window, document, IS_WORKER\n\n# this part is needed to disambiguate between MAIN and WORKER\nif IS_WORKER:\n    # in workers the display does not have a default ID\n    # but there is a sync utility from xworker\n    from _pyscript.display import display\n    from _pyscript import sync\nelse:\n    # in MAIN both PyWorker and a runtime currentScript.id exist\n    # so these are both exposed and the display, if imported,\n    # will point at the right script as default target\n    PyWorker = window._pyscript.PyWorker\n\n    def __getattr__(attribute_name):\n        if attribute_name == \"display\":\n            from _pyscript.display import display\n\n            id = window._pyscript.id\n            return lambda *values, target=id, append=True: display(\n                *values, target=target, append=append\n            )\n        raise AttributeError(f\"'{__name__}' has no attribute '{attribute_name}'\")\n"
+};

pyscript.core/src/stdlib/pyscript.py

@@ -0,0 +1,25 @@
+# export only what we want to expose as `pyscript` module
+# but not what is WORKER/MAIN dependent
+from _pyscript import window, document, IS_WORKER
+
+# this part is needed to disambiguate between MAIN and WORKER
+if IS_WORKER:
+    # in workers the display does not have a default ID
+    # but there is a sync utility from xworker
+    from _pyscript.display import display
+    from _pyscript import sync
+else:
+    # in MAIN both PyWorker and a runtime currentScript.id exist
+    # so these are both exposed and the display, if imported,
+    # will point at the right script as default target
+    PyWorker = window._pyscript.PyWorker
+
+    def __getattr__(attribute_name):
+        if attribute_name == "display":
+            from _pyscript.display import display
+
+            id = window._pyscript.id
+            return lambda *values, target=id, append=True: display(
+                *values, target=target, append=append
+            )
+        raise AttributeError(f"'{__name__}' has no attribute '{attribute_name}'")

pyscript.core/types/stdlib.d.ts

@@ -0,0 +1,2 @@
+declare const _default: string;
+export default _default;

pyscript.core/types/stdlib/pyscript.d.ts

@@ -0,0 +1,8 @@
+declare const _default: {
+    _pyscript: {
+        "__init__.py": string;
+        "display.py": string;
+    };
+    "pyscript.py": string;
+};
+export default _default;