From 26f07246e17a37457de1a8e336c06cabc848e77f Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Wed, 29 Mar 2023 07:31:14 -0700 Subject: [PATCH] Allow pyscript package to contain multiple files (#1309) Followup to pyscript#1232. Closes pyscript#1226. Use node to make a manifest of the src/python dir and then use an esbuild plugin to resolve an import called `pyscript_python_package.esbuild_injected.json` to an object indicating the directories and files in the package folder. This object is then used to govern runtime installation of the package. --- pyscriptjs/esbuild.js | 74 +++++++++++++++++++++++++++- pyscriptjs/src/interpreter_client.ts | 6 +-- pyscriptjs/src/main.ts | 25 +++++++--- pyscriptjs/src/remote_interpreter.ts | 8 --- 4 files changed, 93 insertions(+), 20 deletions(-) diff --git a/pyscriptjs/esbuild.js b/pyscriptjs/esbuild.js index e637cda6..18037e31 100644 --- a/pyscriptjs/esbuild.js +++ b/pyscriptjs/esbuild.js @@ -2,7 +2,7 @@ const { build } = require('esbuild'); const { spawn } = require('child_process'); const { join } = require('path'); const { watchFile } = require('fs'); -const { cp, lstat, readdir } = require('fs/promises'); +const { cp, lstat, readdir, opendir, readFile } = require('fs/promises'); const production = !process.env.NODE_WATCH || process.env.NODE_ENV === 'production'; @@ -15,12 +15,84 @@ if (!production) { copy_targets.push({ src: 'build/*', dest: 'examples/build' }); } +/** + * List out everything in a directory, but skip __pycache__ directory. Used to + * list out the directory paths and the [file path, file contents] pairs in the + * Python package. All paths are relative to the directory we are listing. The + * directories are sorted topologically so that a parent directory always + * appears before its children. + * + * This is consumed in main.ts which calls mkdir for each directory and then + * writeFile to create each file. + * + * @param {string} dir The path to the directory we want to list out + * @returns {dirs: string[], files: [string, string][]} + */ +async function directoryManifest(dir) { + const result = { dirs: [], files: [] }; + await _directoryManifestHelper(dir, '.', result); + return result; +} + +/** + * Recursive helper function for directoryManifest + */ +async function _directoryManifestHelper(root, dir, result) { + const dirObj = await opendir(join(root, dir)); + for await (const d of dirObj) { + const entry = join(dir, d.name); + if (d.isDirectory()) { + if (d.name === '__pycache__') { + continue; + } + result.dirs.push(entry); + await _directoryManifestHelper(root, entry, result); + } else if (d.isFile()) { + result.files.push([entry, await readFile(join(root, entry), { encoding: 'utf-8' })]); + } + } +} + +/** + * An esbuild plugin that injects the Pyscript Python package. + * + * It uses onResolve to attach our custom namespace to the import and then uses + * onLoad to inject the file contents. + */ +function bundlePyscriptPythonPlugin() { + const namespace = 'bundlePyscriptPythonPlugin'; + return { + name: namespace, + setup(build) { + // Resolve the pyscript_package to our custom namespace + // The path doesn't really matter, but we need a separate namespace + // or else the file system resolver will raise an error. + build.onResolve({ filter: /^pyscript_python_package.esbuild_injected.json$/ }, args => { + return { path: 'dummy', namespace }; + }); + // Inject our manifest as JSON contents, and use the JSON loader. + // Also tell esbuild to watch the files & directories we've listed + // for updates. + build.onLoad({ filter: /^dummy$/, namespace }, async args => { + const manifest = await directoryManifest('./src/python'); + return { + contents: JSON.stringify(manifest), + loader: 'json', + watchFiles: manifest.files.map(([k, v]) => k), + watchDirs: manifest.dirs, + }; + }); + }, + }; +} + const pyScriptConfig = { entryPoints: ['src/main.ts'], loader: { '.py': 'text' }, bundle: true, format: 'iife', globalName: 'pyscript', + plugins: [bundlePyscriptPythonPlugin()], }; const copyPath = (source, dest, ...rest) => cp(join(__dirname, source), join(__dirname, dest), ...rest); diff --git a/pyscriptjs/src/interpreter_client.ts b/pyscriptjs/src/interpreter_client.ts index a3d43eec..146535be 100644 --- a/pyscriptjs/src/interpreter_client.ts +++ b/pyscriptjs/src/interpreter_client.ts @@ -75,11 +75,11 @@ export class InterpreterClient extends Object { return this._remote.pyimport(mod_name); } - async mkdirTree(path: string) { - await this._remote.mkdirTree(path); + async mkdir(path: string) { + await this._remote.FS.mkdir(path); } async writeFile(path: string, content: string) { - await this._remote.writeFile(path, content); + await this._remote.FS.writeFile(path, content, { encoding: 'utf8' }); } } diff --git a/pyscriptjs/src/main.ts b/pyscriptjs/src/main.ts index aab01f18..faac25e2 100644 --- a/pyscriptjs/src/main.ts +++ b/pyscriptjs/src/main.ts @@ -17,12 +17,17 @@ import { SplashscreenPlugin } from './plugins/splashscreen'; import { ImportmapPlugin } from './plugins/importmap'; import { StdioDirector as StdioDirector } from './plugins/stdiodirector'; import type { PyProxy } from 'pyodide'; -import * as Synclink from 'synclink'; -// eslint-disable-next-line -// @ts-ignore -import pyscript from './python/pyscript/__init__.py'; -import { robustFetch } from './fetch'; import { RemoteInterpreter } from './remote_interpreter'; +import { robustFetch } from './fetch'; +import * as Synclink from 'synclink'; + +// pyscript_package is injected from src/python by bundlePyscriptPythonPlugin in +// esbuild.js + +// @ts-ignore +import python_package from 'pyscript_python_package.esbuild_injected.json'; + +declare const python_package: { dirs: string[]; files: [string, string] }; const logger = getLogger('pyscript/main'); @@ -268,9 +273,13 @@ export class PyScriptApp { // compatible with the old behavior. logger.info('importing pyscript'); - // Save and load pyscript.py from FS - await interpreter.mkdirTree('/home/pyodide/pyscript'); - await interpreter.writeFile('pyscript/__init__.py', pyscript as string); + // Write pyscript package into file system + for (const dir of python_package.dirs) { + await interpreter._remote.FS.mkdir('/home/pyodide/' + dir); + } + for (const [path, value] of python_package.files) { + await interpreter._remote.FS.writeFile('/home/pyodide/' + path, value); + } //Refresh the module cache so Python consistently finds pyscript module await interpreter._remote.invalidate_module_path_cache(); diff --git a/pyscriptjs/src/remote_interpreter.ts b/pyscriptjs/src/remote_interpreter.ts index eb4b2f08..5e13f97c 100644 --- a/pyscriptjs/src/remote_interpreter.ts +++ b/pyscriptjs/src/remote_interpreter.ts @@ -272,14 +272,6 @@ export class RemoteInterpreter extends Object { return Synclink.proxy(this.interface.pyimport(mod_name)); } - mkdirTree(path: string) { - this.FS.mkdirTree(path); - } - - writeFile(path: string, content: string) { - this.FS.writeFile(path, content, { encoding: 'utf8' }); - } - // eslint-disable-next-line @typescript-eslint/no-explicit-any setHandler(func_name: string, handler: any): void { const pyscript_module = this.interface.pyimport('pyscript');