From 0b23310a06a32071efe6b45aae801429a125cdb4 Mon Sep 17 00:00:00 2001 From: Jeff Glass Date: Wed, 16 Nov 2022 13:11:40 -0600 Subject: [PATCH] Remove 'Implicit Async', Don't Await `runtime.run()` (#928) * Revert to runPython instead of await runPythonAsync * "Implicit Coroutines" are no longer permitted in py-script tags * Tests added for the above * xfail test_importmap (See #938) --- docs/guides/asyncio.md | 35 +++++ docs/guides/index.md | 1 + docs/index.md | 3 + examples/bokeh_interactive.html | 2 +- examples/numpy_canvas_fractals.html | 13 +- examples/webgl/raycaster/index.html | 50 +++---- pyscriptjs/src/components/pyrepl.ts | 4 +- pyscriptjs/src/components/pyscript.ts | 2 +- pyscriptjs/src/pyexec.ts | 20 ++- pyscriptjs/src/pyodide.ts | 4 +- pyscriptjs/src/python/pyscript.py | 25 ++++ pyscriptjs/src/runtime.ts | 17 ++- pyscriptjs/tests/integration/test_03_async.py | 134 +++++++++++++++--- .../tests/integration/test_importmap.py | 3 + .../tests/integration/test_py_config.py | 2 +- pyscriptjs/tests/integration/test_py_repl.py | 8 +- pyscriptjs/tests/py-unit/test_pyscript.py | 69 +++++++++ 17 files changed, 318 insertions(+), 74 deletions(-) create mode 100644 docs/guides/asyncio.md diff --git a/docs/guides/asyncio.md b/docs/guides/asyncio.md new file mode 100644 index 00000000..76183153 --- /dev/null +++ b/docs/guides/asyncio.md @@ -0,0 +1,35 @@ +# Using Async/Await and Asyncio + +## {bdg-warning-line}`Deprecated` Implicit Coroutine Scheduling / Top-Level Await + +In PyScript versions 2022.09.1 and earlier, \ tags could be written in a way that enabled "Implicit Coroutine Scheduling." The keywords `await`, `async for` and `await with` were permitted to be used outside of `async` functions. Any \ tags with these keywords at the top level were compiled into coroutines and automatically scheuled to run in the browser's event loop. This functionality was deprecated, and these keywords are no longer allowed outside of `async` functions. + +To transition code from using top-level await statements to the currently-acceptable syntax, wrap the code into a coroutine using `async def()` and schedule it to run in the browser's event looping using `asyncio.ensure_future()` or `asyncio.create_task()`. + +The following two pieces of code are functionally equivalent - the first only works in versions 2022.09.1, the latter is the currently acceptable equivalent. + +```python +# This version is deprecated, since +# it uses 'await' outside an async function + +import asyncio + +for i in range(3): + print(i) + await asyncio.sleep(1) + +``` + +```python +# This version is acceptable + +import asyncio + +async def main(): + for i in range(3): + print(i) + await asyncio.sleep(1) + +asyncio.ensure_future(main()) + +``` diff --git a/docs/guides/index.md b/docs/guides/index.md index eb8e47fe..ea1c2c87 100644 --- a/docs/guides/index.md +++ b/docs/guides/index.md @@ -16,4 +16,5 @@ caption: 'Contents:' --- passing-objects http-requests +asyncio ``` diff --git a/docs/index.md b/docs/index.md index 13e56bf9..55f6f43a 100644 --- a/docs/index.md +++ b/docs/index.md @@ -22,8 +22,11 @@ Check out our [getting started guide](tutorials/getting-started.md)! You already know the basics and want to learn specifics! [Passing Objects between JavaScript and Python](guides/passing-objects.md) + [Making async HTTP requests in pure Python](guides/http-requests.md) +[Async/Await and Asyncio](guides/asyncio.md) + ::: :::{grid-item-card} [Concepts](concepts/index.md) diff --git a/examples/bokeh_interactive.html b/examples/bokeh_interactive.html index ae9205a5..9bb35098 100644 --- a/examples/bokeh_interactive.html +++ b/examples/bokeh_interactive.html @@ -93,7 +93,7 @@ async def show(plot, target): jsdoc = views[0].model.document _link_docs(pydoc, jsdoc) -await show(row, 'myplot') +asyncio.ensure_future(show(row, 'myplot')) diff --git a/examples/numpy_canvas_fractals.html b/examples/numpy_canvas_fractals.html index 6153b72e..678967bf 100644 --- a/examples/numpy_canvas_fractals.html +++ b/examples/numpy_canvas_fractals.html @@ -316,11 +316,14 @@ canvas.addEventListener("mousemove", create_proxy(mousemove)) import asyncio -_ = await asyncio.gather( - draw_mandelbrot(), - draw_julia(), - draw_newton(), -) +async def main(): + _ = await asyncio.gather( + draw_mandelbrot(), + draw_julia(), + draw_newton(), + ) + +asyncio.ensure_future(main()) diff --git a/examples/webgl/raycaster/index.html b/examples/webgl/raycaster/index.html index c663d859..5d09cc13 100644 --- a/examples/webgl/raycaster/index.html +++ b/examples/webgl/raycaster/index.html @@ -158,36 +158,38 @@ uSpeed = 0.1 time = 0.0003; camera.lookAt(scene.position) -while True: - time = performance.now() * 0.0003; - i = 0 - while i < particularGroup.children.length: - newObject = particularGroup.children[i]; - newObject.rotation.x += newObject.speedValue/10; - newObject.rotation.y += newObject.speedValue/10; - newObject.rotation.z += newObject.speedValue/10; - i += 1 +async def main(): + while True: + time = performance.now() * 0.0003; + i = 0 + while i < particularGroup.children.length: + newObject = particularGroup.children[i]; + newObject.rotation.x += newObject.speedValue/10; + newObject.rotation.y += newObject.speedValue/10; + newObject.rotation.z += newObject.speedValue/10; + i += 1 - i = 0 - while i < modularGroup.children.length: - newCubes = modularGroup.children[i]; - newCubes.rotation.x += 0.008; - newCubes.rotation.y += 0.005; - newCubes.rotation.z += 0.003; + i = 0 + while i < modularGroup.children.length: + newCubes = modularGroup.children[i]; + newCubes.rotation.x += 0.008; + newCubes.rotation.y += 0.005; + newCubes.rotation.z += 0.003; - newCubes.position.x = Math.sin(time * newCubes.positionZ) * newCubes.positionY; - newCubes.position.y = Math.cos(time * newCubes.positionX) * newCubes.positionZ; - newCubes.position.z = Math.sin(time * newCubes.positionY) * newCubes.positionX; - i += 1 + newCubes.position.x = Math.sin(time * newCubes.positionZ) * newCubes.positionY; + newCubes.position.y = Math.cos(time * newCubes.positionX) * newCubes.positionZ; + newCubes.position.z = Math.sin(time * newCubes.positionY) * newCubes.positionX; + i += 1 - particularGroup.rotation.y += 0.005; + particularGroup.rotation.y += 0.005; - modularGroup.rotation.y -= ((mouse.x * 4) + modularGroup.rotation.y) * uSpeed; - modularGroup.rotation.x -= ((-mouse.y * 4) + modularGroup.rotation.x) * uSpeed; + modularGroup.rotation.y -= ((mouse.x * 4) + modularGroup.rotation.y) * uSpeed; + modularGroup.rotation.x -= ((-mouse.y * 4) + modularGroup.rotation.x) * uSpeed; - renderer.render( scene, camera ) - await asyncio.sleep(0.02) + renderer.render( scene, camera ) + await asyncio.sleep(0.02) + asyncio.ensure_future(main()) diff --git a/pyscriptjs/src/components/pyrepl.ts b/pyscriptjs/src/components/pyrepl.ts index b51fcc3a..03da0aa7 100644 --- a/pyscriptjs/src/components/pyrepl.ts +++ b/pyscriptjs/src/components/pyrepl.ts @@ -150,7 +150,7 @@ export function make_PyRepl(runtime: Runtime) { /** Execute the python code written in the editor, and automatically * display() the last evaluated expression */ - async execute(): Promise { + execute(): void { const pySrc = this.getPySrc(); // determine the output element @@ -166,7 +166,7 @@ export function make_PyRepl(runtime: Runtime) { outEl.innerHTML = ''; // execute the python code - const pyResult = await pyExec(runtime, pySrc, outEl); + const pyResult = pyExec(runtime, pySrc, outEl); // display the value of the last evaluated expression (REPL-style) if (pyResult !== undefined) { diff --git a/pyscriptjs/src/components/pyscript.ts b/pyscriptjs/src/components/pyscript.ts index 6d645b70..f55e300f 100644 --- a/pyscriptjs/src/components/pyscript.ts +++ b/pyscriptjs/src/components/pyscript.ts @@ -12,7 +12,7 @@ export function make_PyScript(runtime: Runtime) { ensureUniqueId(this); const pySrc = await this.getPySrc(); this.innerHTML = ''; - await pyExec(runtime, pySrc, this); + pyExec(runtime, pySrc, this); } async getPySrc(): Promise { diff --git a/pyscriptjs/src/pyexec.ts b/pyscriptjs/src/pyexec.ts index 0588abd2..0dc2f6a9 100644 --- a/pyscriptjs/src/pyexec.ts +++ b/pyscriptjs/src/pyexec.ts @@ -1,18 +1,30 @@ import { getLogger } from './logger'; -import { ensureUniqueId } from './utils'; +import { ensureUniqueId, ltrim } from './utils'; +import { UserError } from './exceptions'; import type { Runtime } from './runtime'; const logger = getLogger('pyexec'); -export async function pyExec(runtime: Runtime, pysrc: string, outElem: HTMLElement) { +export function pyExec(runtime: Runtime, pysrc: string, outElem: HTMLElement) { // this is the python function defined in pyscript.py const set_current_display_target = runtime.globals.get('set_current_display_target'); ensureUniqueId(outElem); set_current_display_target(outElem.id); + //This is the python function defined in pyscript.py + const usesTopLevelAwait = runtime.globals.get('uses_top_level_await') try { try { - return await runtime.run(pysrc); - } catch (err) { + if (usesTopLevelAwait(pysrc)){ + throw new UserError( + 'The use of top-level "await", "async for", and ' + + '"async with" is deprecated.' + + '\nPlease write a coroutine containing ' + + 'your code and schedule it using asyncio.ensure_future() or similar.' + + '\nSee https://docs.pyscript.net/latest/guides/asyncio.html for more information.' + ) + } + return runtime.run(pysrc); + } catch (err) { // XXX: currently we display exceptions in the same position as // the output. But we probably need a better way to do that, // e.g. allowing plugins to intercept exceptions and display them diff --git a/pyscriptjs/src/pyodide.ts b/pyscriptjs/src/pyodide.ts index b8c90e6e..b2d2d6f8 100644 --- a/pyscriptjs/src/pyodide.ts +++ b/pyscriptjs/src/pyodide.ts @@ -75,8 +75,8 @@ export class PyodideRuntime extends Runtime { logger.info('pyodide loaded and initialized'); } - async run(code: string): Promise { - return await this.interpreter.runPythonAsync(code); + run(code: string) { + return this.interpreter.runPython(code); } registerJsModule(name: string, module: object): void { diff --git a/pyscriptjs/src/python/pyscript.py b/pyscriptjs/src/python/pyscript.py index 816082a4..d31e3684 100644 --- a/pyscriptjs/src/python/pyscript.py +++ b/pyscriptjs/src/python/pyscript.py @@ -1,3 +1,4 @@ +import ast import asyncio import base64 import html @@ -404,4 +405,28 @@ class PyListTemplate: pass +class TopLevelAsyncFinder(ast.NodeVisitor): + def is_source_top_level_await(self, source): + self.async_found = False + node = ast.parse(source) + self.generic_visit(node) + return self.async_found + + def visit_Await(self, node): + self.async_found = True + + def visit_AsyncFor(self, node): + self.async_found = True + + def visit_AsyncWith(self, node): + self.async_found = True + + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef): + pass # Do not visit children of async function defs + + +def uses_top_level_await(source: str) -> bool: + return TopLevelAsyncFinder().is_source_top_level_await(source) + + pyscript = PyScript() diff --git a/pyscriptjs/src/runtime.ts b/pyscriptjs/src/runtime.ts index 196de8c1..4bb57c5b 100644 --- a/pyscriptjs/src/runtime.ts +++ b/pyscriptjs/src/runtime.ts @@ -55,7 +55,7 @@ export abstract class Runtime extends Object { * (asynchronously) which can call its own API behind the scenes. * Python exceptions are turned into JS exceptions. * */ - abstract run(code: string): Promise; + abstract run(code: string); /** * Same as run, but Python exceptions are not propagated: instead, they @@ -64,11 +64,16 @@ export abstract class Runtime extends Object { * This is a bad API and should be killed/refactored/changed eventually, * but for now we have code which relies on it. * */ - async runButDontRaise(code: string): Promise { - return this.run(code).catch(err => { - const error = err as Error; - logger.error('Error:', error); - }); + runButDontRaise(code: string) { + let result + try{ + result = this.run(code) + } + catch (err){ + const error = err as Error + logger.error('Error:', error) + } + return result } /** diff --git a/pyscriptjs/tests/integration/test_03_async.py b/pyscriptjs/tests/integration/test_03_async.py index 84fc91ee..add002a7 100644 --- a/pyscriptjs/tests/integration/test_03_async.py +++ b/pyscriptjs/tests/integration/test_03_async.py @@ -2,56 +2,146 @@ from .support import PyScriptTest class TestAsync(PyScriptTest): + # ensure_future() and create_task() should behave similarly; + # we'll use the same source code to test both + coroutine_script = """ + + import js + import asyncio + js.console.log("first") + async def main(): + await asyncio.sleep(1) + js.console.log("third") + asyncio.{func}(main()) + js.console.log("second") + + """ + + def test_asyncio_ensure_future(self): + self.pyscript_run(self.coroutine_script.format(func="ensure_future")) + self.wait_for_console("third") + assert self.console.log.lines == [self.PY_COMPLETE, "first", "second", "third"] + + def test_asyncio_create_task(self): + self.pyscript_run(self.coroutine_script.format(func="create_task")) + self.wait_for_console("third") + assert self.console.log.lines == [self.PY_COMPLETE, "first", "second", "third"] + + def test_asyncio_gather(self): + self.pyscript_run( + """ + + import asyncio + import js + from pyodide.ffi import to_js + + async def coro(delay): + await asyncio.sleep(delay) + return(delay) + + async def get_results(): + results = await asyncio.gather(*[coro(d) for d in range(3,0,-1)]) + js.console.log(to_js(results)) + js.console.log("DONE") + + asyncio.ensure_future(get_results()) + + """ + ) + self.wait_for_console("DONE") + assert self.console.log.lines[-2:] == ["[3,2,1]", "DONE"] + def test_multiple_async(self): self.pyscript_run( """ import js import asyncio - for i in range(3): - js.console.log('A', i) - await asyncio.sleep(0.1) + async def a_func(): + for i in range(3): + js.console.log('A', i) + await asyncio.sleep(0.1) + asyncio.ensure_future(a_func()) import js import asyncio - for i in range(3): - js.console.log('B', i) - await asyncio.sleep(0.1) - js.console.log("async tadone") + async def b_func(): + for i in range(3): + js.console.log('B', i) + await asyncio.sleep(0.1) + js.console.log('b func done') + asyncio.ensure_future(b_func()) """ ) - self.wait_for_console("async tadone") + self.wait_for_console("b func done") assert self.console.log.lines == [ - "Python initialization complete", + self.PY_COMPLETE, "A 0", "B 0", "A 1", "B 1", "A 2", "B 2", - "async tadone", + "b func done", ] - def test_multiple_async_multiple_display(self): + def test_multiple_async_multiple_display_targetted(self): + self.pyscript_run( + """ + + import js + import asyncio + + async def a_func(): + for i in range(2): + display(f'A{i}', target='pyA') + await asyncio.sleep(0.1) + asyncio.ensure_future(a_func()) + + + + import js + import asyncio + + async def a_func(): + for i in range(2): + display(f'B{i}', target='pyB') + await asyncio.sleep(0.1) + js.console.log("B DONE") + + asyncio.ensure_future(a_func()) + + """ + ) + self.wait_for_console("B DONE") + inner_text = self.page.inner_text("html") + assert "A0\nA1\nB0\nB1" in inner_text + + def test_async_display_untargetted(self): self.pyscript_run( """ import asyncio - for i in range(2): - display('A') - await asyncio.sleep(0) - + import js - - import asyncio - for i in range(2): - display('B') - await asyncio.sleep(0) + async def a_func(): + try: + display('A') + await asyncio.sleep(0.1) + except Exception as err: + js.console.error(str(err)) + await asyncio.sleep(1) + js.console.log("DONE") + + asyncio.ensure_future(a_func()) """ ) - inner_text = self.page.inner_text("html") - assert "A\nB\nA\nB" in inner_text + self.wait_for_console("DONE") + assert ( + self.console.error.lines[-1] + == "Implicit target not allowed here. Please use display(..., target=...)" + ) diff --git a/pyscriptjs/tests/integration/test_importmap.py b/pyscriptjs/tests/integration/test_importmap.py index 7acd7b8c..a809b249 100644 --- a/pyscriptjs/tests/integration/test_importmap.py +++ b/pyscriptjs/tests/integration/test_importmap.py @@ -1,6 +1,9 @@ +import pytest + from .support import PyScriptTest +@pytest.mark.xfail(reason="See PR #938") class TestImportmap(PyScriptTest): def test_importmap(self): src = """ diff --git a/pyscriptjs/tests/integration/test_py_config.py b/pyscriptjs/tests/integration/test_py_config.py index 38b88ac3..e8a8aa36 100644 --- a/pyscriptjs/tests/integration/test_py_config.py +++ b/pyscriptjs/tests/integration/test_py_config.py @@ -100,7 +100,7 @@ class TestConfig(PyScriptTest): """, ) - assert self.console.log.lines == [self.PY_COMPLETE, "version 0.20.0"] + assert self.console.log.lines[-1] == "version 0.20.0" version = self.page.locator("py-script").inner_text() assert version == "0.20.0" diff --git a/pyscriptjs/tests/integration/test_py_repl.py b/pyscriptjs/tests/integration/test_py_repl.py index 6209d6ee..9a708789 100644 --- a/pyscriptjs/tests/integration/test_py_repl.py +++ b/pyscriptjs/tests/integration/test_py_repl.py @@ -63,13 +63,9 @@ class TestPyRepl(PyScriptTest): """ ) + self.page.wait_for_selector("#runButton") self.page.keyboard.press("Shift+Enter") - - # when we use locator('button').click() the message appears - # immediately, with keyboard.press we need to wait for it. I don't - # really know why it has a different behavior, I didn't investigate - # further. - self.wait_for_console("hello world") + assert self.console.log.lines == [self.PY_COMPLETE, "hello world"] def test_display(self): self.pyscript_run( diff --git a/pyscriptjs/tests/py-unit/test_pyscript.py b/pyscriptjs/tests/py-unit/test_pyscript.py index a7b7c09d..b90acd3c 100644 --- a/pyscriptjs/tests/py-unit/test_pyscript.py +++ b/pyscriptjs/tests/py-unit/test_pyscript.py @@ -1,4 +1,5 @@ import sys +import textwrap from unittest.mock import Mock import pyscript @@ -48,3 +49,71 @@ def test_format_mime_HTML(): out, mime = pyscript.format_mime(obj) assert out == "

hello

" assert mime == "text/html" + + +def test_uses_top_level_await(): + # Basic Case + src = "x = 1" + assert pyscript.uses_top_level_await(src) is False + + # Comments are not top-level await + src = textwrap.dedent( + """ + #await async for async with asyncio + """ + ) + + assert pyscript.uses_top_level_await(src) is False + + # Top-level-await cases + src = textwrap.dedent( + """ + async def foo(): + pass + await foo + """ + ) + assert pyscript.uses_top_level_await(src) is True + + src = textwrap.dedent( + """ + async with object(): + pass + """ + ) + assert pyscript.uses_top_level_await(src) is True + + src = textwrap.dedent( + """ + async for _ in range(10): + pass + """ + ) + assert pyscript.uses_top_level_await(src) is True + + # Acceptable await/async for/async with cases + src = textwrap.dedent( + """ + async def foo(): + await foo() + """ + ) + assert pyscript.uses_top_level_await(src) is False + + src = textwrap.dedent( + """ + async def foo(): + async with object(): + pass + """ + ) + assert pyscript.uses_top_level_await(src) is False + + src = textwrap.dedent( + """ + async def foo(): + async for _ in range(10): + pass + """ + ) + assert pyscript.uses_top_level_await(src) is False