jprdonnelly/pyscript
1
0
Fork 0
mirror of https://github.com/pyscript/pyscript.git synced 2025-12-19 18:27:29 -05:00
Code Issues Projects Releases Wiki Activity

Allow fetching plugins from URL (#1065)

Browse Source
This commit is contained in:
Fábio Rosado
2023-01-11 17:03:53 +00:00
committed by GitHub
parent 470c3489dd
commit cc4b460183
14 changed files with 568 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
pyscriptjs/src/exceptions.ts
View File

@@ -7,6 +7,7 @@ These error codes are used to identify the type of error that occurred.
The convention is:
* PY0 - errors that occur when fetching
* PY1 - errors that occur in config
* Py2 - errors that occur in plugins
* PY9 - Deprecation errors
*/
@@ -22,6 +23,8 @@ export enum ErrorCode {
FETCH_UNAVAILABLE_ERROR = "PY0503",
BAD_CONFIG = "PY1000",
MICROPIP_INSTALL_ERROR = "PY1001",
BAD_PLUGIN_FILE_EXTENSION = "PY2000",
NO_DEFAULT_EXPORT = "PY2001",
TOP_LEVEL_AWAIT = "PY9000"
}
@@ -39,13 +42,11 @@ export class UserError extends Error {
}
export class FetchError extends Error {
export class FetchError extends UserError {
errorCode: ErrorCode;
constructor(errorCode: ErrorCode, message: string) {
super(message)
super(errorCode, message)
this.name = "FetchError";
this.errorCode = errorCode;
this.message = `(${errorCode}): ${message}`;
}
}

37
pyscriptjs/src/fetch.ts
View File

@@ -1,10 +1,43 @@
import { FetchError, ErrorCode } from './exceptions';
/**
* This is a fetch wrapper that handles any non 200 responses and throws a
* FetchError with the right ErrorCode. This is useful because our FetchError
* will automatically create an alert banner.
*
* @param url - URL to fetch
* @param options - options to pass to fetch
* @returns Response
*/
export async function robustFetch(url: string, options?: RequestInit): Promise<Response> {
const response = await fetch(url, options);
let response: Response;
// Note: We need to wrap fetch into a try/catch block because fetch
// throws a TypeError if the URL is invalid such as http://blah.blah
try {
response = await fetch(url, options);
} catch (err) {
const error = err as Error;
let errMsg: string;
if (url.startsWith('http')) {
errMsg =
`Fetching from URL ${url} failed with error ` +
`'${error.message}'. Are your filename and path correct?`;
} else {
errMsg = `PyScript: Access to local files
(using "Paths:" in &lt;py-config&gt;)
is not available when directly opening a HTML file;
you must use a webserver to serve the additional files.
See <a style="text-decoration: underline;" href="https://github.com/pyscript/pyscript/issues/257#issuecomment-1119595062">this reference</a>
on starting a simple webserver with Python.
`;
}
throw new FetchError(ErrorCode.FETCH_ERROR, errMsg);
}
// Note that response.ok is true for 200-299 responses
if (!response.ok) {
const errorMsg = `Fetching from URL ${url} failed with error ${response.status} (${response.statusText}).`;
const errorMsg = `Fetching from URL ${url} failed with error ${response.status} (${response.statusText}). Are your filename and path correct?`;
switch (response.status) {
case 404:
throw new FetchError(ErrorCode.FETCH_NOT_FOUND_ERROR, errorMsg);

141
pyscriptjs/src/main.ts
View File

@@ -8,7 +8,7 @@ import { PluginManager, define_custom_element } from './plugin';
import { make_PyScript, initHandlers, mountElements } from './components/pyscript';
import { PyodideRuntime } from './pyodide';
import { getLogger } from './logger';
import { handleFetchError, showWarning, globalExport } from './utils';
import { showWarning, globalExport } from './utils';
import { calculatePaths } from './plugins/fetch';
import { createCustomElements } from './components/elements';
import { UserError, ErrorCode, _createAlertBanner } from './exceptions';
@@ -20,6 +20,8 @@ import { StdioDirector as StdioDirector } from './plugins/stdiodirector';
// eslint-disable-next-line
// @ts-ignore
import pyscript from './python/pyscript.py';
import { robustFetch } from './fetch';
import type { Plugin } from './plugin';
const logger = getLogger('pyscript/main');
@@ -238,7 +240,7 @@ export class PyScriptApp {
//This may be unnecessary - only useful if plugins try to import files fetch'd in fetchPaths()
runtime.invalidate_module_path_cache();
// Finally load plugins
await this.fetchPythonPlugins(runtime);
await this.fetchUserPlugins(runtime);
}
async fetchPaths(runtime: Runtime) {
@@ -251,63 +253,108 @@ export class PyScriptApp {
logger.info('Paths to fetch: ', fetchPaths);
for (let i = 0; i < paths.length; i++) {
logger.info(` fetching path: ${fetchPaths[i]}`);
try {
await runtime.loadFromFile(paths[i], fetchPaths[i]);
} catch (e) {
// The 'TypeError' here happens when running pytest
// I'm not particularly happy with this solution.
if (e.name === 'FetchError' || e.name === 'TypeError') {
handleFetchError(<Error>e, fetchPaths[i]);
} else {
throw e;
}
}
// Exceptions raised from here will create an alert banner
await runtime.loadFromFile(paths[i], fetchPaths[i]);
}
logger.info('All paths fetched');
}
/**
* Fetches all the python plugins specified in this.config, saves them on the FS and import
* them as modules, executing any plugin define the module scope
* Fetch user plugins and adds them to `this.plugins` so they can
* be loaded by the PluginManager. Currently, we are just looking
* for .py and .js files and calling the appropriate methods.
*
* @param runtime - runtime that will execute the plugins
* @param runtime - runtime that will be used to execute the plugins that need it.
*/
async fetchPythonPlugins(runtime: Runtime) {
async fetchUserPlugins(runtime: Runtime) {
const plugins = this.config.plugins;
logger.info('Python plugins to fetch: ', plugins);
logger.info('Plugins to fetch: ', plugins);
for (const singleFile of plugins) {
logger.info(` fetching plugins: ${singleFile}`);
try {
const pathArr = singleFile.split('/');
const filename = pathArr.pop();
// TODO: Would be probably be better to store plugins somewhere like /plugins/python/ or similar
const destPath = `./${filename}`;
await runtime.loadFromFile(destPath, singleFile);
//refresh module cache before trying to import module files into runtime
runtime.invalidate_module_path_cache();
const modulename = singleFile.replace(/^.*[\\/]/, '').replace('.py', '');
console.log(`importing ${modulename}`);
// TODO: This is very specific to Pyodide API and will not work for other interpreters,
// when we add support for other interpreters we will need to move this to the
// runtime (interpreter) API level and allow each one to implement it in its own way
const module = runtime.interpreter.pyimport(modulename);
if (typeof module.plugin !== 'undefined') {
const py_plugin = module.plugin;
py_plugin.init(this);
this.plugins.addPythonPlugin(py_plugin);
} else {
logger.error(`Cannot find plugin on Python module ${modulename}! Python plugins \
modules must contain a "plugin" attribute. For more information check the plugins documentation.`);
}
} catch (e) {
//Should we still export full error contents to console?
handleFetchError(<Error>e, singleFile);
if (singleFile.endsWith('.py')) {
await this.fetchPythonPlugin(runtime, singleFile);
} else if (singleFile.endsWith('.js')) {
await this.fetchJSPlugin(singleFile);
} else {
throw new UserError(
ErrorCode.BAD_PLUGIN_FILE_EXTENSION,
`Unable to load plugin from '${singleFile}'. ` +
`Plugins need to contain a file extension and be ` +
`either a python or javascript file.`,
);
}
logger.info('All plugins fetched');
}
}
/**
* Fetch a javascript plugin from a filePath, it gets a blob from the
* fetch and creates a file from it, then we create a URL from the file
* so we can import it as a module.
*
* This allow us to instantiate the imported plugin with the default
* export in the module (the plugin class) and add it to the plugins
* list with `new importedPlugin()`.
*
* @param filePath - URL of the javascript file to fetch.
*/
async fetchJSPlugin(filePath: string) {
const pluginBlob = await (await robustFetch(filePath)).blob();
const blobFile = new File([pluginBlob], 'plugin.js', { type: 'text/javascript' });
const fileUrl = URL.createObjectURL(blobFile);
const module = await import(fileUrl);
// Note: We have to put module.default in a variable
// because we have seen weird behaviour when doing
// new module.default() directly.
const importedPlugin = module.default;
// If the imported plugin doesn't have a default export
// it will be undefined, so we throw a user error, so
// an alter banner will be created.
if (importedPlugin === undefined) {
throw new UserError(
ErrorCode.NO_DEFAULT_EXPORT,
`Unable to load plugin from '${filePath}'. ` + `Plugins need to contain a default export.`,
);
} else {
this.plugins.add(new importedPlugin());
}
}
/**
* Fetch python plugins from a filePath, saves it on the FS and import
* it as a module, executing any plugin define the module scope.
*
* @param runtime - runtime that will execute the plugins
* @param filePath - path to the python file to fetch
*/
async fetchPythonPlugin(runtime: Runtime, filePath: string) {
const pathArr = filePath.split('/');
const filename = pathArr.pop();
// TODO: Would be probably be better to store plugins somewhere like /plugins/python/ or similar
const destPath = `./${filename}`;
await runtime.loadFromFile(destPath, filePath);
//refresh module cache before trying to import module files into runtime
runtime.invalidate_module_path_cache();
const modulename = filePath.replace(/^.*[\\/]/, '').replace('.py', '');
console.log(`importing ${modulename}`);
// TODO: This is very specific to Pyodide API and will not work for other interpreters,
// when we add support for other interpreters we will need to move this to the
// runtime (interpreter) API level and allow each one to implement it in its own way
const module = runtime.interpreter.pyimport(modulename);
if (typeof module.plugin !== 'undefined') {
const py_plugin = module.plugin;
py_plugin.init(this);
this.plugins.addPythonPlugin(py_plugin);
} else {
logger.error(`Cannot find plugin on Python module ${modulename}! Python plugins \
modules must contain a "plugin" attribute. For more information check the plugins documentation.`);
}
logger.info('All plugins fetched');
}
// lifecycle (7)

16
pyscriptjs/src/plugin.ts
View File

@@ -93,11 +93,23 @@ export class PluginManager {
}
beforeLaunch(config: AppConfig) {
for (const p of this._plugins) p.beforeLaunch(config);
for (const p of this._plugins) {
try {
p.beforeLaunch(config);
} catch (e) {
logger.error(`Error while calling beforeLaunch hook of plugin ${p.constructor.name}`, e);
}
}
}
afterSetup(runtime: Runtime) {
for (const p of this._plugins) p.afterSetup(runtime);
for (const p of this._plugins) {
try {
p.afterSetup(runtime);
} catch (e) {
logger.error(`Error while calling afterSetup hook of plugin ${p.constructor.name}`, e);
}
}
for (const p of this._pythonPlugins) p.afterSetup?.(runtime);
}

24
pyscriptjs/src/utils.ts
View File

@@ -1,4 +1,4 @@
import { _createAlertBanner, UserError, FetchError, ErrorCode } from "./exceptions"
import { _createAlertBanner } from "./exceptions"
export function addClasses(element: HTMLElement, classes: string[]) {
for (const entry of classes) {
@@ -45,28 +45,6 @@ export function showWarning(msg: string, messageType: 'text' | 'html' = 'text'):
_createAlertBanner(msg, 'warning', messageType);
}
export function handleFetchError(e: Error, singleFile: string) {
// XXX: inspecting the error message to understand what happened is very
// fragile. We need a better solution.
let errorContent: string;
if (e.message.includes('Failed to fetch')) {
errorContent = `PyScript: Access to local files
(using "Paths:" in &lt;py-config&gt;)
is not available when directly opening a HTML file;
you must use a webserver to serve the additional files.
See <a style="text-decoration: underline;" href="https://github.com/pyscript/pyscript/issues/257#issuecomment-1119595062">this reference</a>
on starting a simple webserver with Python.`;
} else if (e.message.includes('404')) {
errorContent =
`PyScript: Loading from file <u>` +
singleFile +
`</u> failed with error 404 (File not Found). Are your filename and path are correct?`;
} else {
errorContent = `PyScript encountered an error while loading from file: ${e.message}`;
}
throw new UserError(ErrorCode.FETCH_ERROR, errorContent, 'html');
}
export function readTextFromPath(path: string) {
const request = new XMLHttpRequest();
request.open('GET', path, false);

15
pyscriptjs/tests/integration/support.py
View File

@@ -304,6 +304,21 @@ class PyScriptTest:
text = "\n".join(loc.all_inner_texts())
raise AssertionError(f"Found {n} alert banners:\n" + text)
def assert_banner_message(self, expected_message):
"""
Ensure that there is an alert banner on the page with the given message.
Currently it only handles a single.
"""
banner = self.page.wait_for_selector(".alert-banner")
banner_text = banner.inner_text()
if expected_message not in banner_text:
raise AssertionError(
f"Expected message '{expected_message}' does not "
f"match banner text '{banner_text}'"
)
return True
def check_tutor_generated_code(self, modules_to_check=None):
"""
Ensure that the source code viewer injected by the PyTutor plugin

18
pyscriptjs/tests/integration/test_01_basic.py
View File

@@ -179,18 +179,12 @@ class TestBasic(PyScriptTest):
with pytest.raises(JsErrors) as exc:
self.check_js_errors()
error_msg = str(exc.value)
print(error_msg)
if self.is_fake_server:
assert (
"Fetching from URL foo.py failed with error 404 (Not Found)."
in error_msg
)
else:
assert (
"Fetching from URL foo.py failed with error 404 (File not found)"
in error_msg
)
error_msgs = str(exc.value)
expected_msg = "(PY0404): Fetching from URL foo.py failed with error 404"
assert expected_msg in error_msgs
assert self.assert_banner_message(expected_msg)
pyscript_tag = self.page.locator("py-script")
assert pyscript_tag.inner_html() == ""

112
pyscriptjs/tests/integration/test_plugins.py
View File

@@ -244,3 +244,115 @@ class TestPlugin(PyScriptTest):
)
# EXPECT an error for the missing attribute
assert error_msg in self.console.error.lines
def test_fetch_python_plugin(self):
"""
Test that we can fetch a plugin from a remote URL. Note we need to use
the 'raw' URL for the plugin, otherwise the request will be rejected
by cors policy.
"""
self.pyscript_run(
"""
<py-config>
plugins = [
"https://raw.githubusercontent.com/FabioRosado/pyscript-plugins/main/python/hello-world.py"
]
</py-config>
<py-hello-world></py-hello-world>
"""
)
hello_element = self.page.locator("py-hello-world")
assert hello_element.inner_html() == '<div id="hello">Hello World!</div>'
def test_fetch_js_plugin(self):
self.pyscript_run(
"""
<py-config>
plugins = [
"https://raw.githubusercontent.com/FabioRosado/pyscript-plugins/main/js/hello-world.js"
]
</py-config>
"""
)
hello_element = self.page.locator("py-hello-world")
assert hello_element.inner_html() == "<h1>Hello, world!</h1>"
def test_fetch_js_plugin_bare(self):
self.pyscript_run(
"""
<py-config>
plugins = [
"https://raw.githubusercontent.com/FabioRosado/pyscript-plugins/main/js/hello-world-base.js"
]
</py-config>
"""
)
hello_element = self.page.locator("py-hello-world")
assert hello_element.inner_html() == "<h1>Hello, world!</h1>"
def test_fetch_plugin_no_file_extension(self):
self.pyscript_run(
"""
<py-config>
plugins = [
"http://non-existent.blah/hello-world"
]
</py-config>
""",
wait_for_pyscript=False,
)
expected_msg = (
"(PY2000): Unable to load plugin from "
"'http://non-existent.blah/hello-world'. Plugins "
"need to contain a file extension and be either a "
"python or javascript file."
)
assert self.assert_banner_message(expected_msg)
def test_fetch_js_plugin_non_existent(self):
self.pyscript_run(
"""
<py-config>
plugins = [
"http://non-existent.example.com/hello-world.js"
]
</py-config>
""",
wait_for_pyscript=False,
)
expected_msg = (
"(PY0001): Fetching from URL "
"http://non-existent.example.com/hello-world.js failed "
"with error 'Failed to fetch'. Are your filename and "
"path correct?"
)
assert self.assert_banner_message(expected_msg)
def test_fetch_js_no_export(self):
self.pyscript_run(
"""
<py-config>
plugins = [
"https://raw.githubusercontent.com/FabioRosado/pyscript-plugins/main/js/hello-world-no-export.js"
]
</py-config>
""",
wait_for_pyscript=False,
)
expected_message = (
"(PY2001): Unable to load plugin from "
"'https://raw.githubusercontent.com/FabioRosado/pyscript-plugins"
"/main/js/hello-world-no-export.js'. "
"Plugins need to contain a default export."
)
assert self.assert_banner_message(expected_message)

6
pyscriptjs/tests/integration/test_py_config.py
View File

@@ -245,12 +245,10 @@ class TestConfig(PyScriptTest):
wait_for_pyscript=False,
)
expected = (
"Loading from file <u>./f.py</u> failed with error 404 (File not Found). "
"Are your filename and path are correct?"
)
expected = "(PY0404): Fetching from URL ./f.py failed with " "error 404"
inner_html = self.page.locator(".py-error").inner_html()
assert expected in inner_html
assert expected in self.console.error.lines[-1]

58
pyscriptjs/tests/unit/fetch.test.ts
View File

@@ -19,19 +19,8 @@ describe("robustFetch", () => {
const url = "https://pyscript.net/non-existent-page"
const expectedError = new FetchError(
ErrorCode.FETCH_NOT_FOUND_ERROR,
`Fetching from URL ${url} failed with error 404 (Not Found).`
)
expect(() => robustFetch(url)).rejects.toThrow(expectedError);
})
it('receiving a 404 when fetching should throw FetchError with the right errorCode', async () => {
global.fetch = jest.fn(() => (Promise.resolve(new Response("Not Found", {status: 404}))));
const url = "https://pyscript.net/non-existent-page"
const expectedError = new FetchError(
ErrorCode.FETCH_NOT_FOUND_ERROR,
`Fetching from URL ${url} failed with error 404 (Not Found).`
`Fetching from URL ${url} failed with error 404 (Not Found). ` +
`Are your filename and path correct?`
)
expect(() => robustFetch(url)).rejects.toThrow(expectedError);
@@ -43,7 +32,8 @@ describe("robustFetch", () => {
const url = "https://pyscript.net/protected-page"
const expectedError = new FetchError(
ErrorCode.FETCH_UNAUTHORIZED_ERROR,
`Fetching from URL ${url} failed with error 401 (Unauthorized).`
`Fetching from URL ${url} failed with error 401 (Unauthorized). ` +
`Are your filename and path correct?`
)
expect(() => robustFetch(url)).rejects.toThrow(expectedError);
@@ -55,7 +45,8 @@ describe("robustFetch", () => {
const url = "https://pyscript.net/secret-page"
const expectedError = new FetchError(
ErrorCode.FETCH_FORBIDDEN_ERROR,
`Fetching from URL ${url} failed with error 403 (Forbidden).`
`Fetching from URL ${url} failed with error 403 (Forbidden). ` +
`Are your filename and path correct?`
)
expect(() => robustFetch(url)).rejects.toThrow(expectedError);
@@ -67,7 +58,8 @@ describe("robustFetch", () => {
const url = "https://pyscript.net/protected-page"
const expectedError = new FetchError(
ErrorCode.FETCH_SERVER_ERROR,
`Fetching from URL ${url} failed with error 500 (Internal Server Error).`
`Fetching from URL ${url} failed with error 500 (Internal Server Error). ` +
`Are your filename and path correct?`
)
expect(() => robustFetch(url)).rejects.toThrow(expectedError);
@@ -79,9 +71,41 @@ describe("robustFetch", () => {
const url = "https://pyscript.net/protected-page"
const expectedError = new FetchError(
ErrorCode.FETCH_UNAVAILABLE_ERROR,
`Fetching from URL ${url} failed with error 503 (Service Unavailable).`
`Fetching from URL ${url} failed with error 503 (Service Unavailable). ` +
`Are your filename and path correct?`
)
expect(() => robustFetch(url)).rejects.toThrow(expectedError);
})
it('handle TypeError when using a bad url', async () => {
global.fetch = jest.fn(() => (Promise.reject(new TypeError("Failed to fetch"))));
const url = "https://pyscript.net/protected-page"
const expectedError = new FetchError(
ErrorCode.FETCH_ERROR,
`Fetching from URL ${url} failed with error 'Failed to fetch'. Are your filename and path correct?`
)
expect(() => robustFetch(url)).rejects.toThrow(expectedError);
})
it('handle failed to fetch when using local file', async () => {
global.fetch = jest.fn(() => (Promise.reject(new TypeError("Failed to fetch"))));
const url = "./my-awesome-pyscript.py"
const expectedError = new FetchError(
ErrorCode.FETCH_ERROR,
`PyScript: Access to local files
(using "Paths:" in &lt;py-config&gt;)
is not available when directly opening a HTML file;
you must use a webserver to serve the additional files.
See <a style="text-decoration: underline;" href="https://github.com/pyscript/pyscript/issues/257#issuecomment-1119595062">this reference</a>
on starting a simple webserver with Python.
`
)
expect(() => robustFetch(url)).rejects.toThrow(expectedError);
})
});