diff --git a/pyproject.toml b/pyproject.toml index fdf96a22..0e73d381 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,6 +6,7 @@ build-backend = "setuptools.build_meta" dynamic = ["version"] [tool.codespell] +ignore-words-list = "afterall" skip = "pyscriptjs/node_modules/*,*.js,*.json" [tool.ruff] diff --git a/pyscriptjs/.eslintrc.js b/pyscriptjs/.eslintrc.js index 4eb95de9..0945b1a1 100644 --- a/pyscriptjs/.eslintrc.js +++ b/pyscriptjs/.eslintrc.js @@ -18,14 +18,20 @@ module.exports = { plugins: ['@typescript-eslint'], ignorePatterns: ['node_modules'], rules: { - 'no-prototype-builtins': 'error', - '@typescript-eslint/no-unused-vars': ['error', { args: 'none' }], + // ts-ignore is already an explicit override, no need to have a second lint + '@typescript-eslint/ban-ts-comment': 'off', + + // any related lints '@typescript-eslint/no-explicit-any': 'error', '@typescript-eslint/no-unsafe-assignment': 'error', - '@typescript-eslint/no-unsafe-argument': 'error', - '@typescript-eslint/no-unsafe-member-access': 'error', '@typescript-eslint/no-unsafe-call': 'error', + '@typescript-eslint/no-unsafe-member-access': 'error', + '@typescript-eslint/no-unsafe-argument': 'error', '@typescript-eslint/no-unsafe-return': 'error', + + // other rules + 'no-prototype-builtins': 'error', + '@typescript-eslint/no-unused-vars': ['error', { args: 'none' }], '@typescript-eslint/no-floating-promises': 'error', '@typescript-eslint/restrict-plus-operands': 'error', '@typescript-eslint/no-empty-function': 'error', diff --git a/pyscriptjs/Makefile b/pyscriptjs/Makefile index 6a875cdc..4e486319 100644 --- a/pyscriptjs/Makefile +++ b/pyscriptjs/Makefile @@ -61,6 +61,11 @@ build: build-fast: node esbuild.js +# use the following rule to do all the checks done by precommit: in +# particular, use this if you want to run eslint. +precommit-check: + pre-commit run --all-files + examples: mkdir -p ./examples cp -r ../examples/* ./examples diff --git a/pyscriptjs/jest-environment-jsdom.js b/pyscriptjs/jest-environment-jsdom.js new file mode 100644 index 00000000..4ff695fc --- /dev/null +++ b/pyscriptjs/jest-environment-jsdom.js @@ -0,0 +1,28 @@ +'use strict'; + +const { TextEncoder, TextDecoder } = require('util'); +const { MessageChannel } = require('node:worker_threads'); + +const { default: $JSDOMEnvironment, TestEnvironment } = require('jest-environment-jsdom'); + +Object.defineProperty(exports, '__esModule', { + value: true, +}); + +class JSDOMEnvironment extends $JSDOMEnvironment { + constructor(...args) { + const { global } = super(...args); + if (!global.TextEncoder) { + global.TextEncoder = TextEncoder; + } + if (!global.TextDecoder) { + global.TextDecoder = TextDecoder; + } + if (!global.MessageChannel) { + global.MessageChannel = MessageChannel; + } + } +} + +exports.default = JSDOMEnvironment; +exports.TestEnvironment = TestEnvironment === $JSDOMEnvironment ? JSDOMEnvironment : TestEnvironment; diff --git a/pyscriptjs/jest.config.js b/pyscriptjs/jest.config.js index d6f78c4d..f609a4ed 100644 --- a/pyscriptjs/jest.config.js +++ b/pyscriptjs/jest.config.js @@ -1,7 +1,7 @@ //jest.config.js module.exports = { preset: 'ts-jest', - testEnvironment: 'jest-environment-jsdom', + testEnvironment: './jest-environment-jsdom.js', extensionsToTreatAsEsm: ['.ts'], transform: { '^.+\\.tsx?$': [ diff --git a/pyscriptjs/package-lock.json b/pyscriptjs/package-lock.json index ca908f88..5d85e19e 100644 --- a/pyscriptjs/package-lock.json +++ b/pyscriptjs/package-lock.json @@ -15,7 +15,8 @@ "@codemirror/theme-one-dark": "^6.1.1", "@codemirror/view": "^6.9.3", "@hoodmane/toml-j0.4": "^1.1.2", - "codemirror": "^6.0.1" + "codemirror": "6.0.1", + "synclink": "^0.1.1" }, "devDependencies": { "@jest/globals": "29.1.2", @@ -5520,9 +5521,13 @@ }, "node_modules/symbol-tree": { "version": "3.2.4", - "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz", - "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==", - "dev": true + "dev": true, + "license": "MIT" + }, + "node_modules/synclink": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/synclink/-/synclink-0.1.1.tgz", + "integrity": "sha512-34eBqFHZn3f2PX/tz/z0E+817Um6ESFNsBkMoGqoGgBX0qlLNlRXFoHSNIUSjTK8l5Qs4gL99Xwrz0vNAUrPEw==" }, "node_modules/test-exclude": { "version": "6.0.0", @@ -10045,10 +10050,13 @@ }, "symbol-tree": { "version": "3.2.4", - "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz", - "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==", "dev": true }, + "synclink": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/synclink/-/synclink-0.1.1.tgz", + "integrity": "sha512-34eBqFHZn3f2PX/tz/z0E+817Um6ESFNsBkMoGqoGgBX0qlLNlRXFoHSNIUSjTK8l5Qs4gL99Xwrz0vNAUrPEw==" + }, "test-exclude": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", diff --git a/pyscriptjs/package.json b/pyscriptjs/package.json index 8b38ed45..ad2b79a7 100644 --- a/pyscriptjs/package.json +++ b/pyscriptjs/package.json @@ -39,6 +39,7 @@ "@codemirror/theme-one-dark": "^6.1.1", "@codemirror/view": "^6.9.3", "@hoodmane/toml-j0.4": "^1.1.2", - "codemirror": "^6.0.1" + "codemirror": "6.0.1", + "synclink": "^0.1.1" } } diff --git a/pyscriptjs/src/components/pyrepl.ts b/pyscriptjs/src/components/pyrepl.ts index 99a1b462..47ba551a 100644 --- a/pyscriptjs/src/components/pyrepl.ts +++ b/pyscriptjs/src/components/pyrepl.ts @@ -132,10 +132,10 @@ export function make_PyRepl(interpreter: InterpreterClient, app: PyScriptApp) { const outEl = this.outDiv; // execute the python code - app.plugins.beforePyReplExec({ interpreter: interpreter, src: pySrc, outEl: outEl, pyReplTag: this }); + await app.plugins.beforePyReplExec({ interpreter: interpreter, src: pySrc, outEl: outEl, pyReplTag: this }); // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment const { result } = await pyExec(interpreter, pySrc, outEl); - app.plugins.afterPyReplExec({ + await app.plugins.afterPyReplExec({ interpreter: interpreter, src: pySrc, outEl: outEl, diff --git a/pyscriptjs/src/components/pyscript.ts b/pyscriptjs/src/components/pyscript.ts index 1310bd5a..2728f41d 100644 --- a/pyscriptjs/src/components/pyscript.ts +++ b/pyscriptjs/src/components/pyscript.ts @@ -23,6 +23,7 @@ export function make_PyScript(interpreter: InterpreterClient, app: PyScriptApp) * * Concurrent access to the multiple py-script tags is thus avoided. */ + app.incrementPendingTags(); let releaseLock: () => void; try { releaseLock = await app.tagExecutionLock(); @@ -34,10 +35,10 @@ export function make_PyScript(interpreter: InterpreterClient, app: PyScriptApp) const pySrc = await this.getPySrc(); this.innerHTML = ''; - app.plugins.beforePyScriptExec({ interpreter: interpreter, src: pySrc, pyScriptTag: this }); + await app.plugins.beforePyScriptExec({ interpreter: interpreter, src: pySrc, pyScriptTag: this }); /* eslint-disable @typescript-eslint/no-unsafe-assignment */ const result = (await pyExec(interpreter, pySrc, this)).result; - app.plugins.afterPyScriptExec({ + await app.plugins.afterPyScriptExec({ interpreter: interpreter, src: pySrc, pyScriptTag: this, @@ -46,6 +47,7 @@ export function make_PyScript(interpreter: InterpreterClient, app: PyScriptApp) /* eslint-enable @typescript-eslint/no-unsafe-assignment */ } finally { releaseLock(); + app.decrementPendingTags(); } } diff --git a/pyscriptjs/src/components/pywidget.ts b/pyscriptjs/src/components/pywidget.ts index da0a86ff..86326e4d 100644 --- a/pyscriptjs/src/components/pywidget.ts +++ b/pyscriptjs/src/components/pywidget.ts @@ -2,6 +2,7 @@ import type { PyProxy, PyProxyCallable } from 'pyodide'; import { getLogger } from '../logger'; import { robustFetch } from '../fetch'; import { InterpreterClient } from '../interpreter_client'; +import type { Remote } from 'synclink'; const logger = getLogger('py-register-widget'); @@ -13,8 +14,8 @@ function createWidget(interpreter: InterpreterClient, name: string, code: string name: string = name; klass: string = klass; code: string = code; - proxy: PyProxy & { connect(): void }; - proxyClass: PyProxyCallable; + proxy: Remote; + proxyClass: Remote; constructor() { super(); @@ -28,15 +29,15 @@ function createWidget(interpreter: InterpreterClient, name: string, code: string async connectedCallback() { await interpreter.runButDontRaise(this.code); - this.proxyClass = interpreter.globals.get(this.klass) as PyProxyCallable; - this.proxy = this.proxyClass(this) as PyProxy & { connect(): void }; - this.proxy.connect(); - this.registerWidget(); + this.proxyClass = (await interpreter.globals.get(this.klass)) as Remote; + this.proxy = (await this.proxyClass(this)) as Remote; + await this.proxy.connect(); + await this.registerWidget(); } - registerWidget() { + async registerWidget() { logger.info('new widget registered:', this.name); - interpreter.globals.set(this.id, this.proxy); + await interpreter.globals.set(this.id, this.proxy); } } // eslint-disable-next-line @typescript-eslint/no-unused-vars diff --git a/pyscriptjs/src/exceptions.ts b/pyscriptjs/src/exceptions.ts index 3f9eec4b..540f19c1 100644 --- a/pyscriptjs/src/exceptions.ts +++ b/pyscriptjs/src/exceptions.ts @@ -31,6 +31,11 @@ export enum ErrorCode { export class UserError extends Error { messageType: MessageType; errorCode: ErrorCode; + /** + * `isinstance` doesn't work correctly across multiple realms. + * Hence, `$$isUserError` flag / marker is used to identify a `UserError`. + */ + $$isUserError: boolean; constructor(errorCode: ErrorCode, message: string, t: MessageType = 'text') { super(message); @@ -38,6 +43,7 @@ export class UserError extends Error { this.name = 'UserError'; this.messageType = t; this.message = `(${errorCode}): ${message}`; + this.$$isUserError = true; } } diff --git a/pyscriptjs/src/interpreter_client.ts b/pyscriptjs/src/interpreter_client.ts index b6701dde..a3d43eec 100644 --- a/pyscriptjs/src/interpreter_client.ts +++ b/pyscriptjs/src/interpreter_client.ts @@ -1,8 +1,9 @@ import type { AppConfig } from './pyconfig'; import { RemoteInterpreter } from './remote_interpreter'; -import type { PyProxyDict } from 'pyodide'; +import type { PyProxyDict, PyProxy } from 'pyodide'; import { getLogger } from './logger'; import type { Stdio } from './stdio'; +import * as Synclink from 'synclink'; const logger = getLogger('pyscript/interpreter'); @@ -11,18 +12,25 @@ InterpreterClient class is responsible to request code execution (among other things) from a `RemoteInterpreter` */ export class InterpreterClient extends Object { - _remote: RemoteInterpreter; + _remote: Synclink.Remote; + _unwrapped_remote: RemoteInterpreter; config: AppConfig; /** * global symbols table for the underlying interface. * */ - globals: PyProxyDict; + globals: Synclink.Remote; stdio: Stdio; - constructor(config: AppConfig, stdio: Stdio) { + constructor( + config: AppConfig, + stdio: Stdio, + remote: Synclink.Remote, + unwrapped_remote: RemoteInterpreter, + ) { super(); this.config = config; - this._remote = new RemoteInterpreter(this.config.interpreters[0].src); + this._remote = remote; + this._unwrapped_remote = unwrapped_remote; this.stdio = stdio; } @@ -31,8 +39,9 @@ export class InterpreterClient extends Object { * interface. * */ async initializeRemote(): Promise { - await this._remote.loadInterpreter(this.config, this.stdio); - this.globals = this._remote.globals as PyProxyDict; + await this._unwrapped_remote.loadInterpreter(this.config, this.stdio); + // await this._remote.loadInterpreter(this.config, Synclink.proxy(this.stdio)); + this.globals = this._remote.globals; } /** @@ -61,4 +70,16 @@ export class InterpreterClient extends Object { } return result; } + + async pyimport(mod_name: string): Promise> { + return this._remote.pyimport(mod_name); + } + + async mkdirTree(path: string) { + await this._remote.mkdirTree(path); + } + + async writeFile(path: string, content: string) { + await this._remote.writeFile(path, content); + } } diff --git a/pyscriptjs/src/main.ts b/pyscriptjs/src/main.ts index 1c46a170..68ff926d 100644 --- a/pyscriptjs/src/main.ts +++ b/pyscriptjs/src/main.ts @@ -17,13 +17,31 @@ import { SplashscreenPlugin } from './plugins/splashscreen'; import { ImportmapPlugin } from './plugins/importmap'; import { StdioDirector as StdioDirector } from './plugins/stdiodirector'; import type { PyProxy } from 'pyodide'; +import * as Synclink from 'synclink'; // eslint-disable-next-line // @ts-ignore import pyscript from './python/pyscript/__init__.py'; import { robustFetch } from './fetch'; +import { RemoteInterpreter } from './remote_interpreter'; const logger = getLogger('pyscript/main'); +/** + * Monkey patching the error transfer handler to preserve the `$$isUserError` + * marker so as to detect `UserError` subclasses in the error handling code. + */ +const throwHandler = Synclink.transferHandlers.get('throw') as Synclink.TransferHandler< + { value: unknown }, + { value: { $$isUserError: boolean } } +>; +const old_error_transfer_handler = throwHandler.serialize.bind(throwHandler) as typeof throwHandler.serialize; +function new_error_transfer_handler({ value }: { value: { $$isUserError: boolean } }) { + const result = old_error_transfer_handler({ value }); + result[0].value.$$isUserError = value.$$isUserError; + return result; +} +throwHandler.serialize = new_error_transfer_handler; + /* High-level overview of the lifecycle of a PyScript App: 1. pyscript.js is loaded by the browser. PyScriptApp().main() is called @@ -58,13 +76,22 @@ More concretely: - PyScriptApp.afterInterpreterLoad() implements all the points >= 5. */ +export let interpreter; +// TODO: This is for backwards compatibility, it should be removed +// when we finish the deprecation cycle of `runtime` +export let runtime; + export class PyScriptApp { config: AppConfig; interpreter: InterpreterClient; + readyPromise: Promise; PyScript: ReturnType; plugins: PluginManager; _stdioMultiplexer: StdioMultiplexer; - tagExecutionLock: ReturnType; // this is used to ensure that py-script tags are executed sequentially + tagExecutionLock: () => Promise<() => void>; // this is used to ensure that py-script tags are executed sequentially + _numPendingTags: number; + scriptTagsPromise: Promise; + resolvedScriptTags: () => void; constructor() { // initialize the builtin plugins @@ -76,6 +103,8 @@ export class PyScriptApp { this.plugins.add(new StdioDirector(this._stdioMultiplexer)); this.tagExecutionLock = createLock(); + this._numPendingTags = 0; + this.scriptTagsPromise = new Promise(res => (this.resolvedScriptTags = res)); } // Error handling logic: if during the execution we encounter an error @@ -83,18 +112,34 @@ export class PyScriptApp { // config, file not found in fetch, etc.), we can throw UserError(). It is // responsibility of main() to catch it and show it to the user in a // proper way (e.g. by using a banner at the top of the page). - main() { + async main() { try { - this._realMain(); + await this._realMain(); } catch (error) { - this._handleUserErrorMaybe(error); + await this._handleUserErrorMaybe(error); } } - _handleUserErrorMaybe(error) { - if (error instanceof UserError) { - _createAlertBanner(error.message, 'error', error.messageType); - this.plugins.onUserError(error); + incrementPendingTags() { + this._numPendingTags += 1; + } + + decrementPendingTags() { + if (this._numPendingTags <= 0) { + throw new Error('INTERNAL ERROR: assertion _numPendingTags > 0 failed'); + } + this._numPendingTags -= 1; + if (this._numPendingTags === 0) { + this.resolvedScriptTags(); + } + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + async _handleUserErrorMaybe(error: any) { + const e = error as UserError; + if (e && e.$$isUserError) { + _createAlertBanner(e.message, 'error', e.messageType); + await this.plugins.onUserError(e); } else { throw error; } @@ -103,11 +148,15 @@ export class PyScriptApp { // ============ lifecycle ============ // lifecycle (1) - _realMain() { + async _realMain() { this.loadConfig(); - this.plugins.configure(this.config); + await this.plugins.configure(this.config); this.plugins.beforeLaunch(this.config); - this.loadInterpreter(); + await this.loadInterpreter(); + interpreter = this.interpreter; + // TODO: This is for backwards compatibility, it should be removed + // when we finish the deprecation cycle of `runtime` + runtime = this.interpreter; } // lifecycle (2) @@ -131,7 +180,7 @@ export class PyScriptApp { } // lifecycle (4) - loadInterpreter() { + async loadInterpreter() { logger.info('Initializing interpreter'); if (this.config.interpreters.length == 0) { throw new UserError(ErrorCode.BAD_CONFIG, 'Fatal error: config.interpreter is empty'); @@ -143,7 +192,18 @@ export class PyScriptApp { const interpreter_cfg = this.config.interpreters[0]; - this.interpreter = new InterpreterClient(this.config, this._stdioMultiplexer); + const remote_interpreter = new RemoteInterpreter(interpreter_cfg.src); + const { port1, port2 } = new MessageChannel(); + port1.start(); + port2.start(); + Synclink.expose(remote_interpreter, port2); + const wrapped_remote_interpreter = Synclink.wrap(port1); + this.interpreter = new InterpreterClient( + this.config, + this._stdioMultiplexer, + wrapped_remote_interpreter as Synclink.Remote, + remote_interpreter, + ); this.logStatus(`Downloading ${interpreter_cfg.name}...`); @@ -154,10 +214,10 @@ export class PyScriptApp { // by the try/catch inside main(): that's why we need to .catch() it // explicitly and call _handleUserErrorMaybe also there. const script = document.createElement('script'); // create a script DOM node - script.src = this.interpreter._remote.src; + script.src = await this.interpreter._remote.src; script.addEventListener('load', () => { - this.afterInterpreterLoad(this.interpreter).catch(error => { - this._handleUserErrorMaybe(error); + this.afterInterpreterLoad(this.interpreter).catch(async error => { + await this._handleUserErrorMaybe(error); }); }); document.head.appendChild(script); @@ -180,12 +240,12 @@ export class PyScriptApp { await mountElements(interpreter); // lifecycle (6.5) - this.plugins.afterSetup(interpreter); + await this.plugins.afterSetup(interpreter); //Refresh module cache in case plugins have modified the filesystem - interpreter._remote.invalidate_module_path_cache(); + await interpreter._remote.invalidate_module_path_cache(); this.logStatus('Executing tags...'); - this.executeScripts(interpreter); + await this.executeScripts(interpreter); this.logStatus('Initializing web components...'); // lifecycle (8) @@ -198,7 +258,7 @@ export class PyScriptApp { // pyscript initialization has complete. If you change it, you need to // change it also in tests/integration/support.py this.logStatus('Startup complete'); - this.plugins.afterStartup(interpreter); + await this.plugins.afterStartup(interpreter); logger.info('PyScript page fully initialized'); } @@ -210,19 +270,23 @@ export class PyScriptApp { logger.info('importing pyscript'); // Save and load pyscript.py from FS - interpreter._remote.FS.mkdirTree('/home/pyodide/pyscript'); - interpreter._remote.FS.writeFile('pyscript/__init__.py', pyscript as string); + await interpreter.mkdirTree('/home/pyodide/pyscript'); + await interpreter.writeFile('pyscript/__init__.py', pyscript as string); //Refresh the module cache so Python consistently finds pyscript module - interpreter._remote.invalidate_module_path_cache(); + await interpreter._remote.invalidate_module_path_cache(); // inject `define_custom_element` and showWarning it into the PyScript // module scope - const pyscript_module = interpreter._remote.interface.pyimport('pyscript'); - pyscript_module.define_custom_element = define_custom_element; - pyscript_module.showWarning = showWarning; - // eslint-disable-next-line @typescript-eslint/no-unsafe-call - pyscript_module._set_version_info(version); - pyscript_module.destroy(); + // eventually replace the setHandler calls with interpreter._remote.setHandler i.e. the ones mentioned below + // await interpreter._remote.setHandler('define_custom_element', Synclink.proxy(define_custom_element)); + // await interpreter._remote.setHandler('showWarning', Synclink.proxy(showWarning)); + interpreter._unwrapped_remote.setHandler('define_custom_element', define_custom_element); + interpreter._unwrapped_remote.setHandler('showWarning', showWarning); + const pyscript_module = (await interpreter.pyimport('pyscript')) as Synclink.Remote< + PyProxy & { _set_version_info(string): void } + >; + await pyscript_module._set_version_info(version); + await pyscript_module.destroy(); // import some carefully selected names into the global namespace await interpreter.run(` @@ -239,7 +303,7 @@ export class PyScriptApp { await this.fetchPaths(interpreter); //This may be unnecessary - only useful if plugins try to import files fetch'd in fetchPaths() - interpreter._remote.invalidate_module_path_cache(); + await interpreter._remote.invalidate_module_path_cache(); // Finally load plugins await this.fetchUserPlugins(interpreter); } @@ -339,7 +403,7 @@ export class PyScriptApp { await interpreter._remote.loadFromFile(filename, filePath); //refresh module cache before trying to import module files into interpreter - interpreter._remote.invalidate_module_path_cache(); + await interpreter._remote.invalidate_module_path_cache(); const modulename = filePath.replace(/^.*[\\/]/, '').replace('.py', ''); @@ -347,8 +411,10 @@ export class PyScriptApp { // TODO: This is very specific to Pyodide API and will not work for other interpreters, // when we add support for other interpreters we will need to move this to the // interpreter API level and allow each one to implement it in its own way - const module = interpreter._remote.interface.pyimport(modulename); - if (typeof module.plugin !== 'undefined') { + + // eventually replace with interpreter.pyimport(modulename); + const module = interpreter._unwrapped_remote.pyimport(modulename); + if (typeof (await module.plugin) !== 'undefined') { const py_plugin = module.plugin as PyProxy & { init(app: PyScriptApp): void }; py_plugin.init(this); this.plugins.addPythonPlugin(py_plugin); @@ -359,10 +425,13 @@ modules must contain a "plugin" attribute. For more information check the plugin } // lifecycle (7) - executeScripts(interpreter: InterpreterClient) { + async executeScripts(interpreter: InterpreterClient) { // make_PyScript takes an interpreter and a PyScriptApp as arguments this.PyScript = make_PyScript(interpreter, this); customElements.define('py-script', this.PyScript); + this.incrementPendingTags(); + this.decrementPendingTags(); + await this.scriptTagsPromise; } // ================= registraton API ==================== @@ -385,10 +454,6 @@ globalExport('pyscript_get_config', pyscript_get_config); // main entry point of execution const globalApp = new PyScriptApp(); -globalApp.main(); +globalApp.readyPromise = globalApp.main(); export { version }; -export const interpreter = globalApp.interpreter; -// TODO: This is for backwards compatibility, it should be removed -// when we finish the deprecation cycle of `runtime` -export const runtime = globalApp.interpreter; diff --git a/pyscriptjs/src/plugin.ts b/pyscriptjs/src/plugin.ts index 7052d356..ef4eeb6c 100644 --- a/pyscriptjs/src/plugin.ts +++ b/pyscriptjs/src/plugin.ts @@ -142,10 +142,10 @@ export class PluginManager { this._pythonPlugins.push(plugin); } - configure(config: AppConfig) { + async configure(config: AppConfig) { for (const p of this._plugins) p.configure?.(config); - for (const p of this._pythonPlugins) p.configure?.(config); + for (const p of this._pythonPlugins) await p.configure?.(config); } beforeLaunch(config: AppConfig) { @@ -158,7 +158,7 @@ export class PluginManager { } } - afterSetup(interpreter: InterpreterClient) { + async afterSetup(interpreter: InterpreterClient) { for (const p of this._plugins) { try { p.afterSetup?.(interpreter); @@ -167,43 +167,63 @@ export class PluginManager { } } - for (const p of this._pythonPlugins) p.afterSetup?.(interpreter); + for (const p of this._pythonPlugins) await p.afterSetup?.(interpreter); } - afterStartup(interpreter: InterpreterClient) { + async afterStartup(interpreter: InterpreterClient) { for (const p of this._plugins) p.afterStartup?.(interpreter); - for (const p of this._pythonPlugins) p.afterStartup?.(interpreter); + for (const p of this._pythonPlugins) await p.afterStartup?.(interpreter); } - beforePyScriptExec(options: { interpreter: InterpreterClient; src: string; pyScriptTag: PyScriptTag }) { + async beforePyScriptExec(options: { interpreter: InterpreterClient; src: string; pyScriptTag: PyScriptTag }) { for (const p of this._plugins) p.beforePyScriptExec?.(options); - for (const p of this._pythonPlugins) p.beforePyScriptExec?.callKwargs(options); + for (const p of this._pythonPlugins) + await p.beforePyScriptExec(options.interpreter, options.src, options.pyScriptTag); } - afterPyScriptExec(options: { interpreter: InterpreterClient; src: string; pyScriptTag: PyScriptTag; result: any }) { + async afterPyScriptExec(options: { + interpreter: InterpreterClient; + src: string; + pyScriptTag: PyScriptTag; + result: any; + }) { for (const p of this._plugins) p.afterPyScriptExec?.(options); - for (const p of this._pythonPlugins) p.afterPyScriptExec?.callKwargs(options); + for (const p of this._pythonPlugins) + await p.afterPyScriptExec(options.interpreter, options.src, options.pyScriptTag, options.result); } - beforePyReplExec(options: { interpreter: InterpreterClient; src: string; outEl: HTMLElement; pyReplTag: any }) { + async beforePyReplExec(options: { + interpreter: InterpreterClient; + src: string; + outEl: HTMLElement; + pyReplTag: any; + }) { for (const p of this._plugins) p.beforePyReplExec?.(options); - for (const p of this._pythonPlugins) p.beforePyReplExec?.callKwargs(options); + for (const p of this._pythonPlugins) + await p.beforePyReplExec?.(options.interpreter, options.src, options.outEl, options.pyReplTag); } - afterPyReplExec(options: { interpreter: InterpreterClient; src: string; outEl; pyReplTag; result }) { + async afterPyReplExec(options: { interpreter: InterpreterClient; src: string; outEl; pyReplTag; result }) { for (const p of this._plugins) p.afterPyReplExec?.(options); - for (const p of this._pythonPlugins) p.afterPyReplExec?.callKwargs(options); + for (const p of this._pythonPlugins) + await p.afterPyReplExec?.( + options.interpreter, + options.src, + options.outEl, + options.pyReplTag, + options.result, + ); } - onUserError(error: UserError) { + async onUserError(error: UserError) { for (const p of this._plugins) p.onUserError?.(error); - for (const p of this._pythonPlugins) p.onUserError?.(error); + for (const p of this._pythonPlugins) await p.onUserError?.(error); } } diff --git a/pyscriptjs/src/plugins/importmap.ts b/pyscriptjs/src/plugins/importmap.ts index 03c013c8..9d8bf928 100644 --- a/pyscriptjs/src/plugins/importmap.ts +++ b/pyscriptjs/src/plugins/importmap.ts @@ -47,7 +47,7 @@ export class ImportmapPlugin extends Plugin { } logger.info('Registering JS module', name); - interpreter._remote.registerJsModule(name, exports); + await interpreter._remote.registerJsModule(name, exports); } } } diff --git a/pyscriptjs/src/plugins/python/py_markdown.py b/pyscriptjs/src/plugins/python/py_markdown.py index 95322459..093e4af5 100644 --- a/pyscriptjs/src/plugins/python/py_markdown.py +++ b/pyscriptjs/src/plugins/python/py_markdown.py @@ -16,7 +16,7 @@ class MyPlugin(Plugin): console.log(f"configuration received: {config}") def afterStartup(self, interpreter): - console.log(f"interpreter received: {interpreter}") + console.log("interpreter received:", interpreter) plugin = MyPlugin("py-markdown") diff --git a/pyscriptjs/src/plugins/stdiodirector.ts b/pyscriptjs/src/plugins/stdiodirector.ts index 9f7c463c..68cd4e10 100644 --- a/pyscriptjs/src/plugins/stdiodirector.ts +++ b/pyscriptjs/src/plugins/stdiodirector.ts @@ -94,13 +94,13 @@ export class StdioDirector extends Plugin { } } - afterPyReplExec(options: { + async afterPyReplExec(options: { interpreter: InterpreterClient; src: string; outEl: HTMLElement; pyReplTag: InstanceType>; result: any; // eslint-disable-line @typescript-eslint/no-explicit-any - }): void { + }): Promise { // display the value of the last-evaluated expression in the REPL if (options.result !== undefined) { const outputId: string | undefined = options.pyReplTag.getAttribute('output'); @@ -108,14 +108,14 @@ export class StdioDirector extends Plugin { // 'output' attribute also used as location to send // result of REPL if (document.getElementById(outputId)) { - pyDisplay(options.interpreter, options.result, { target: outputId }); + await pyDisplay(options.interpreter, options.result, { target: outputId }); } else { //no matching element on page createSingularWarning(`output = "${outputId}" does not match the id of any element on the page.`); } } else { // 'otuput atribuite not provided - pyDisplay(options.interpreter, options.result, { target: options.outEl.id }); + await pyDisplay(options.interpreter, options.result, { target: options.outEl.id }); } } diff --git a/pyscriptjs/src/pyexec.ts b/pyscriptjs/src/pyexec.ts index e01036b7..69006298 100644 --- a/pyscriptjs/src/pyexec.ts +++ b/pyscriptjs/src/pyexec.ts @@ -2,7 +2,8 @@ import { getLogger } from './logger'; import { ensureUniqueId } from './utils'; import { UserError, ErrorCode } from './exceptions'; import { InterpreterClient } from './interpreter_client'; -import type { PyProxy, PyProxyCallable } from 'pyodide'; +import type { PyProxyCallable, PyProxy } from 'pyodide'; +import type { Remote } from 'synclink'; const logger = getLogger('pyexec'); @@ -12,15 +13,17 @@ export async function pyExec( outElem: HTMLElement, // eslint-disable-next-line @typescript-eslint/no-explicit-any ): Promise<{ result: any }> { - const pyscript_py = interpreter._remote.interface.pyimport('pyscript') as PyProxy & { - set_current_display_target(id: string): void; - uses_top_level_await(code: string): boolean; - }; + const pyscript_py = (await interpreter.pyimport('pyscript')) as Remote< + PyProxy & { + set_current_display_target(id: string): void; + uses_top_level_await(code: string): boolean; + } + >; ensureUniqueId(outElem); - pyscript_py.set_current_display_target(outElem.id); + await pyscript_py.set_current_display_target(outElem.id); try { try { - if (pyscript_py.uses_top_level_await(pysrc)) { + if (await pyscript_py.uses_top_level_await(pysrc)) { throw new UserError( ErrorCode.TOP_LEVEL_AWAIT, 'The use of top-level "await", "async for", and ' + @@ -41,8 +44,8 @@ export async function pyExec( return { result: undefined }; } } finally { - pyscript_py.set_current_display_target(undefined); - pyscript_py.destroy(); + await pyscript_py.set_current_display_target(undefined); + await pyscript_py.destroy(); } } @@ -54,10 +57,10 @@ export async function pyExec( * pyDisplay(interpreter, obj, { target: targetID }); */ // eslint-disable-next-line @typescript-eslint/no-explicit-any -export function pyDisplay(interpreter: InterpreterClient, obj: any, kwargs: { [k: string]: any } = {}) { - const display = interpreter.globals.get('display') as PyProxyCallable; +export async function pyDisplay(interpreter: InterpreterClient, obj: any, kwargs: { [k: string]: any } = {}) { + const display = (await interpreter.globals.get('display')) as PyProxyCallable; try { - display.callKwargs(obj, kwargs); + await display.callKwargs(obj, kwargs); } finally { display.destroy(); } diff --git a/pyscriptjs/src/python/pyscript/__init__.py b/pyscriptjs/src/python/pyscript/__init__.py index 32fe669f..93667f98 100644 --- a/pyscriptjs/src/python/pyscript/__init__.py +++ b/pyscriptjs/src/python/pyscript/__init__.py @@ -496,6 +496,30 @@ class Plugin: def init(self, app): self.app = app + def configure(self, config): + pass + + def afterSetup(self, interpreter): + pass + + def afterStartup(self, interpreter): + pass + + def beforePyScriptExec(self, interpreter, src, pyScriptTag): + pass + + def afterPyScriptExec(self, interpreter, src, pyScriptTag, result): + pass + + def beforePyReplExec(self, interpreter, src, outEl, pyReplTag): + pass + + def afterPyReplExec(self, interpreter, src, outEl, pyReplTag, result): + pass + + def onUserError(self, error): + pass + def register_custom_element(self, tag): """ Decorator to register a new custom element as part of a Plugin and associate diff --git a/pyscriptjs/src/remote_interpreter.ts b/pyscriptjs/src/remote_interpreter.ts index 3e0fcbab..eb4b2f08 100644 --- a/pyscriptjs/src/remote_interpreter.ts +++ b/pyscriptjs/src/remote_interpreter.ts @@ -3,30 +3,32 @@ import { getLogger } from './logger'; import { Stdio } from './stdio'; import { InstallError, ErrorCode } from './exceptions'; import { robustFetch } from './fetch'; -import type { loadPyodide as loadPyodideDeclaration, PyodideInterface, PyProxy } from 'pyodide'; +import type { loadPyodide as loadPyodideDeclaration, PyodideInterface, PyProxy, PyProxyDict } from 'pyodide'; +import type { ProxyMarked } from 'synclink'; +import * as Synclink from 'synclink'; declare const loadPyodide: typeof loadPyodideDeclaration; const logger = getLogger('pyscript/pyodide'); -export type InterpreterInterface = PyodideInterface | null; +export type InterpreterInterface = (PyodideInterface & ProxyMarked) | null; interface Micropip extends PyProxy { install(packageName: string | string[]): Promise; } type FSInterface = { - writeFile(path: string, data: Uint8Array | string, options?: { canOwn: boolean }): void; + writeFile(path: string, data: Uint8Array | string, options?: { canOwn?: boolean; encoding?: string }): void; mkdirTree(path: string): void; mkdir(path: string): void; -}; +} & ProxyMarked; type PATHFSInterface = { resolve(path: string): string; -}; +} & ProxyMarked; type PATHInterface = { dirname(path: string): string; -}; +} & ProxyMarked; /* RemoteInterpreter class is responsible to process requests from the @@ -50,9 +52,9 @@ export class RemoteInterpreter extends Object { PATH: PATHInterface; PATH_FS: PATHFSInterface; - globals: PyProxy; + globals: PyProxyDict & ProxyMarked; // TODO: Remove this once `runtimes` is removed! - interpreter: InterpreterInterface; + interpreter: InterpreterInterface & ProxyMarked; constructor(src = 'https://cdn.jsdelivr.net/pyodide/v0.22.1/full/pyodide.js') { super(); @@ -82,15 +84,18 @@ export class RemoteInterpreter extends Object { * path. */ async loadInterpreter(config: AppConfig, stdio: Stdio): Promise { - this.interface = await loadPyodide({ - stdout: (msg: string) => { - stdio.stdout_writeline(msg); - }, - stderr: (msg: string) => { - stdio.stderr_writeline(msg); - }, - fullStdLib: false, - }); + this.interface = Synclink.proxy( + await loadPyodide({ + stdout: (msg: string) => { + // TODO: add syncify when moved to worker + stdio.stdout_writeline(msg); + }, + stderr: (msg: string) => { + stdio.stderr_writeline(msg); + }, + fullStdLib: false, + }), + ); // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment this.FS = this.interface.FS; // eslint-disable-next-line @@ -101,7 +106,7 @@ export class RemoteInterpreter extends Object { // TODO: Remove this once `runtimes` is removed! this.interpreter = this.interface; - this.globals = this.interface.globals; + this.globals = Synclink.proxy(this.interface.globals as PyProxyDict); if (config.packages) { logger.info('Found packages in configuration to install. Loading micropip...'); @@ -262,4 +267,23 @@ export class RemoteInterpreter extends Object { const importlib = this.interface.pyimport('importlib') as PyProxy & { invalidate_caches(): void }; importlib.invalidate_caches(); } + + pyimport(mod_name: string): PyProxy & Synclink.ProxyMarked { + return Synclink.proxy(this.interface.pyimport(mod_name)); + } + + mkdirTree(path: string) { + this.FS.mkdirTree(path); + } + + writeFile(path: string, content: string) { + this.FS.writeFile(path, content, { encoding: 'utf8' }); + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + setHandler(func_name: string, handler: any): void { + const pyscript_module = this.interface.pyimport('pyscript'); + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + pyscript_module[func_name] = handler; + } } diff --git a/pyscriptjs/tests/integration/conftest.py b/pyscriptjs/tests/integration/conftest.py index e03272ea..2fe14436 100644 --- a/pyscriptjs/tests/integration/conftest.py +++ b/pyscriptjs/tests/integration/conftest.py @@ -142,10 +142,18 @@ class HTTPServer(SuperHTTPServer): @pytest.fixture(scope="session") def http_server(logger): class MyHTTPRequestHandler(SimpleHTTPRequestHandler): + def end_headers(self): + self.send_my_headers() + SimpleHTTPRequestHandler.end_headers(self) + + def send_my_headers(self): + self.send_header("Cross-Origin-Embedder-Policy", "require-corp") + self.send_header("Cross-Origin-Opener-Policy", "same-origin") + def log_message(self, fmt, *args): logger.log("http_server", fmt % args, color="blue") - host, port = "127.0.0.1", 8080 + host, port = "localhost", 8080 base_url = f"http://{host}:{port}" # serve_Run forever under thread diff --git a/pyscriptjs/tests/integration/support.py b/pyscriptjs/tests/integration/support.py index 4da6a4a7..96dcf5b0 100644 --- a/pyscriptjs/tests/integration/support.py +++ b/pyscriptjs/tests/integration/support.py @@ -126,6 +126,20 @@ class PyScriptTest: page.on("console", self._on_console) page.on("pageerror", self._on_pageerror) + def run_js(self, code): + """ + allows top level await to be present in the `code` parameter + """ + self.page.evaluate( + """(async () => { + try {%s} + catch(e) { + console.error(e); + } + })();""" + % code + ) + def teardown_method(self): # we call check_js_errors on teardown: this means that if there are still # non-cleared errors, the test will fail. If you expect errors in your diff --git a/pyscriptjs/tests/integration/test_00_support.py b/pyscriptjs/tests/integration/test_00_support.py index f22a6c9b..078bb33f 100644 --- a/pyscriptjs/tests/integration/test_00_support.py +++ b/pyscriptjs/tests/integration/test_00_support.py @@ -32,6 +32,24 @@ class TestSupport(PyScriptTest): content = self.page.content() assert "

Hello world

" in content + def test_await_with_run_js(self): + self.run_js( + """ + function resolveAfter200MilliSeconds(x) { + return new Promise((resolve) => { + setTimeout(() => { + resolve(x); + }, 200); + }); + } + + const x = await resolveAfter200MilliSeconds(10); + console.log(x); + """ + ) + + assert self.console.log.lines[-1] == "10" + def test_console(self): """ Test that we capture console.log messages correctly. diff --git a/pyscriptjs/tests/integration/test_01_basic.py b/pyscriptjs/tests/integration/test_01_basic.py index 5f6b6903..4104b1db 100644 --- a/pyscriptjs/tests/integration/test_01_basic.py +++ b/pyscriptjs/tests/integration/test_01_basic.py @@ -176,8 +176,8 @@ class TestBasic(PyScriptTest): """ ) self.page.locator("button").click() - self.page.locator("py-script") # wait until appears + self.page.wait_for_selector("py-terminal") assert self.console.log.lines[0] == self.PY_COMPLETE assert self.console.log.lines[-1] == "hello world" diff --git a/pyscriptjs/tests/integration/test_02_display.py b/pyscriptjs/tests/integration/test_02_display.py index b51ab771..670eed6c 100644 --- a/pyscriptjs/tests/integration/test_02_display.py +++ b/pyscriptjs/tests/integration/test_02_display.py @@ -1,11 +1,13 @@ import base64 import html import io +import os import re +import numpy as np from PIL import Image -from .support import PyScriptTest +from .support import PyScriptTest, wait_for_render class TestOutput(PyScriptTest): @@ -298,19 +300,30 @@ class TestOutput(PyScriptTest): def test_image_display(self): self.pyscript_run( """ - packages = [ "matplotlib"] + packages = ["matplotlib"] import matplotlib.pyplot as plt xpoints = [3, 6, 9] ypoints = [1, 2, 3] plt.plot(xpoints, ypoints) - plt.show() + display(plt) """ ) - inner_html = self.page.content() - pattern = r'