Allow pyscript package to contain multiple files (#1309)

Followup to pyscript#1232. Closes pyscript#1226.

Use node to make a manifest of the src/python dir and then use an esbuild
plugin to resolve an import called `pyscript_python_package.esbuild_injected.json`
to an object indicating the directories and files in the package folder.
This object is then used to govern runtime installation of the package.

pyscriptjs/esbuild.js

@@ -2,7 +2,7 @@ const { build } = require('esbuild');
 const { spawn } = require('child_process');
 const { join } = require('path');
 const { watchFile } = require('fs');
-const { cp, lstat, readdir } = require('fs/promises');
+const { cp, lstat, readdir, opendir, readFile } = require('fs/promises');
 
 const production = !process.env.NODE_WATCH || process.env.NODE_ENV === 'production';
 
@@ -15,12 +15,84 @@ if (!production) {
     copy_targets.push({ src: 'build/*', dest: 'examples/build' });
 }
 
+/**
+ * List out everything in a directory, but skip __pycache__ directory. Used to
+ * list out the directory paths and the [file path, file contents] pairs in the
+ * Python package. All paths are relative to the directory we are listing. The
+ * directories are sorted topologically so that a parent directory always
+ * appears before its children.
+ *
+ * This is consumed in main.ts which calls mkdir for each directory and then
+ * writeFile to create each file.
+ *
+ * @param {string} dir The path to the directory we want to list out
+ * @returns {dirs: string[], files: [string, string][]}
+ */
+async function directoryManifest(dir) {
+    const result = { dirs: [], files: [] };
+    await _directoryManifestHelper(dir, '.', result);
+    return result;
+}
+
+/**
+ * Recursive helper function for directoryManifest
+ */
+async function _directoryManifestHelper(root, dir, result) {
+    const dirObj = await opendir(join(root, dir));
+    for await (const d of dirObj) {
+        const entry = join(dir, d.name);
+        if (d.isDirectory()) {
+            if (d.name === '__pycache__') {
+                continue;
+            }
+            result.dirs.push(entry);
+            await _directoryManifestHelper(root, entry, result);
+        } else if (d.isFile()) {
+            result.files.push([entry, await readFile(join(root, entry), { encoding: 'utf-8' })]);
+        }
+    }
+}
+
+/**
+ * An esbuild plugin that injects the Pyscript Python package.
+ *
+ * It uses onResolve to attach our custom namespace to the import and then uses
+ * onLoad to inject the file contents.
+ */
+function bundlePyscriptPythonPlugin() {
+    const namespace = 'bundlePyscriptPythonPlugin';
+    return {
+        name: namespace,
+        setup(build) {
+            // Resolve the pyscript_package to our custom namespace
+            // The path doesn't really matter, but we need a separate namespace
+            // or else the file system resolver will raise an error.
+            build.onResolve({ filter: /^pyscript_python_package.esbuild_injected.json$/ }, args => {
+                return { path: 'dummy', namespace };
+            });
+            // Inject our manifest as JSON contents, and use the JSON loader.
+            // Also tell esbuild to watch the files & directories we've listed
+            // for updates.
+            build.onLoad({ filter: /^dummy$/, namespace }, async args => {
+                const manifest = await directoryManifest('./src/python');
+                return {
+                    contents: JSON.stringify(manifest),
+                    loader: 'json',
+                    watchFiles: manifest.files.map(([k, v]) => k),
+                    watchDirs: manifest.dirs,
+                };
+            });
+        },
+    };
+}
+
 const pyScriptConfig = {
     entryPoints: ['src/main.ts'],
     loader: { '.py': 'text' },
     bundle: true,
     format: 'iife',
     globalName: 'pyscript',
+    plugins: [bundlePyscriptPythonPlugin()],
 };
 
 const copyPath = (source, dest, ...rest) => cp(join(__dirname, source), join(__dirname, dest), ...rest);

pyscriptjs/src/interpreter_client.ts

@@ -75,11 +75,11 @@ export class InterpreterClient extends Object {
         return this._remote.pyimport(mod_name);
     }
 
-    async mkdirTree(path: string) {
+    async mkdir(path: string) {
-        await this._remote.mkdirTree(path);
+        await this._remote.FS.mkdir(path);
     }
 
     async writeFile(path: string, content: string) {
-        await this._remote.writeFile(path, content);
+        await this._remote.FS.writeFile(path, content, { encoding: 'utf8' });
     }
 }

pyscriptjs/src/main.ts

@@ -17,12 +17,17 @@ import { SplashscreenPlugin } from './plugins/splashscreen';
 import { ImportmapPlugin } from './plugins/importmap';
 import { StdioDirector as StdioDirector } from './plugins/stdiodirector';
 import type { PyProxy } from 'pyodide';
-import * as Synclink from 'synclink';
-// eslint-disable-next-line
-// @ts-ignore
-import pyscript from './python/pyscript/__init__.py';
-import { robustFetch } from './fetch';
 import { RemoteInterpreter } from './remote_interpreter';
+import { robustFetch } from './fetch';
+import * as Synclink from 'synclink';
+
+// pyscript_package is injected from src/python by bundlePyscriptPythonPlugin in
+// esbuild.js
+
+// @ts-ignore
+import python_package from 'pyscript_python_package.esbuild_injected.json';
+
+declare const python_package: { dirs: string[]; files: [string, string] };
 
 const logger = getLogger('pyscript/main');
 
@@ -268,9 +273,13 @@ export class PyScriptApp {
         // compatible with the old behavior.
         logger.info('importing pyscript');
 
-        // Save and load pyscript.py from FS
+        // Write pyscript package into file system
-        await interpreter.mkdirTree('/home/pyodide/pyscript');
+        for (const dir of python_package.dirs) {
-        await interpreter.writeFile('pyscript/__init__.py', pyscript as string);
+            await interpreter._remote.FS.mkdir('/home/pyodide/' + dir);
+        }
+        for (const [path, value] of python_package.files) {
+            await interpreter._remote.FS.writeFile('/home/pyodide/' + path, value);
+        }
         //Refresh the module cache so Python consistently finds pyscript module
         await interpreter._remote.invalidate_module_path_cache();
 

pyscriptjs/src/remote_interpreter.ts

@@ -272,14 +272,6 @@ export class RemoteInterpreter extends Object {
         return Synclink.proxy(this.interface.pyimport(mod_name));
     }
 
-    mkdirTree(path: string) {
-        this.FS.mkdirTree(path);
-    }
-
-    writeFile(path: string, content: string) {
-        this.FS.writeFile(path, content, { encoding: 'utf8' });
-    }
-
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     setHandler(func_name: string, handler: any): void {
         const pyscript_module = this.interface.pyimport('pyscript');