Remove 'Implicit Async', Don't Await runtime.run() (#928)

* Revert to runPython instead of await runPythonAsync

* "Implicit Coroutines" are no longer permitted in py-script tags

* Tests added for the above

* xfail test_importmap (See #938)

docs/guides/asyncio.md

@@ -0,0 +1,35 @@
+# Using Async/Await and Asyncio
+
+## {bdg-warning-line}`Deprecated` Implicit Coroutine Scheduling / Top-Level Await
+
+In PyScript versions 2022.09.1 and earlier, \<py-script\> tags could be written in a way that enabled "Implicit Coroutine Scheduling." The keywords `await`, `async for` and `await with` were permitted to be used outside of `async` functions. Any \<py-script\> tags with these keywords at the top level were compiled into coroutines and automatically scheuled to run in the browser's event loop. This functionality was deprecated, and these keywords are no longer allowed outside of `async` functions.
+
+To transition code from using top-level await statements to the currently-acceptable syntax, wrap the code into a coroutine using `async def()` and schedule it to run in the browser's event looping using `asyncio.ensure_future()` or `asyncio.create_task()`.
+
+The following two pieces of code are functionally equivalent - the first only works in versions 2022.09.1, the latter is the currently acceptable equivalent.
+
+```python
+# This version is deprecated, since
+# it uses 'await' outside an async function
+<py-script>
+import asyncio
+
+for i in range(3):
+    print(i)
+    await asyncio.sleep(1)
+</py-script>
+```
+
+```python
+# This version is acceptable
+<py-script>
+import asyncio
+
+async def main():
+    for i in range(3):
+        print(i)
+        await asyncio.sleep(1)
+
+asyncio.ensure_future(main())
+</py-script>
+```

docs/guides/index.md

@@ -16,4 +16,5 @@ caption: 'Contents:'
 ---
 passing-objects
 http-requests
+asyncio
 ```

docs/index.md

@@ -22,8 +22,11 @@ Check out our [getting started guide](tutorials/getting-started.md)!
 You already know the basics and want to learn specifics!
 
 [Passing Objects between JavaScript and Python](guides/passing-objects.md)
+
 [Making async HTTP requests in pure Python](guides/http-requests.md)
 
+[Async/Await and Asyncio](guides/asyncio.md)
+
 :::
 :::{grid-item-card} [Concepts](concepts/index.md)
 

examples/bokeh_interactive.html

@@ -93,7 +93,7 @@ async def show(plot, target):
     jsdoc = views[0].model.document
     _link_docs(pydoc, jsdoc)
 
-await show(row, 'myplot')
+asyncio.ensure_future(show(row, 'myplot'))
     </py-script>
 
 </body>

examples/numpy_canvas_fractals.html

@@ -316,11 +316,14 @@ canvas.addEventListener("mousemove", create_proxy(mousemove))
 
 import asyncio
 
-_ = await asyncio.gather(
+async def main():
-  draw_mandelbrot(),
+  _ = await asyncio.gather(
-  draw_julia(),
+    draw_mandelbrot(),
-  draw_newton(),
+    draw_julia(),
-)
+    draw_newton(),
+  )
+
+asyncio.ensure_future(main())
 </py-script>
 
 </body>

examples/webgl/raycaster/index.html

@@ -158,36 +158,38 @@ uSpeed = 0.1
 time = 0.0003;
 camera.lookAt(scene.position)
 
-while True:
+async def main():
-  time = performance.now() * 0.0003;
+  while True:
-  i = 0
+    time = performance.now() * 0.0003;
-  while i < particularGroup.children.length:
+    i = 0
-    newObject = particularGroup.children[i];
+    while i < particularGroup.children.length:
-    newObject.rotation.x += newObject.speedValue/10;
+      newObject = particularGroup.children[i];
-    newObject.rotation.y += newObject.speedValue/10;
+      newObject.rotation.x += newObject.speedValue/10;
-    newObject.rotation.z += newObject.speedValue/10;
+      newObject.rotation.y += newObject.speedValue/10;
-    i += 1
+      newObject.rotation.z += newObject.speedValue/10;
+      i += 1
 
-  i = 0
+    i = 0
-  while i < modularGroup.children.length:
+    while i < modularGroup.children.length:
-    newCubes = modularGroup.children[i];
+      newCubes = modularGroup.children[i];
-    newCubes.rotation.x += 0.008;
+      newCubes.rotation.x += 0.008;
-    newCubes.rotation.y += 0.005;
+      newCubes.rotation.y += 0.005;
-    newCubes.rotation.z += 0.003;
+      newCubes.rotation.z += 0.003;
 
-    newCubes.position.x = Math.sin(time * newCubes.positionZ) * newCubes.positionY;
+      newCubes.position.x = Math.sin(time * newCubes.positionZ) * newCubes.positionY;
-    newCubes.position.y = Math.cos(time * newCubes.positionX) * newCubes.positionZ;
+      newCubes.position.y = Math.cos(time * newCubes.positionX) * newCubes.positionZ;
-    newCubes.position.z = Math.sin(time * newCubes.positionY) * newCubes.positionX;
+      newCubes.position.z = Math.sin(time * newCubes.positionY) * newCubes.positionX;
-    i += 1
+      i += 1
 
-  particularGroup.rotation.y += 0.005;
+    particularGroup.rotation.y += 0.005;
 
-  modularGroup.rotation.y -= ((mouse.x * 4) + modularGroup.rotation.y) * uSpeed;
+    modularGroup.rotation.y -= ((mouse.x * 4) + modularGroup.rotation.y) * uSpeed;
-  modularGroup.rotation.x -= ((-mouse.y * 4) + modularGroup.rotation.x) * uSpeed;
+    modularGroup.rotation.x -= ((-mouse.y * 4) + modularGroup.rotation.x) * uSpeed;
 
-  renderer.render( scene, camera )
+    renderer.render( scene, camera )
-  await asyncio.sleep(0.02)
+    await asyncio.sleep(0.02)
 
+    asyncio.ensure_future(main())
 
 </py-script>
 </body>

pyscriptjs/src/components/pyrepl.ts

@@ -150,7 +150,7 @@ export function make_PyRepl(runtime: Runtime) {
         /** Execute the python code written in the editor, and automatically
          *  display() the last evaluated expression
          */
-        async execute(): Promise<void> {
+        execute(): void {
             const pySrc = this.getPySrc();
 
             // determine the output element
@@ -166,7 +166,7 @@ export function make_PyRepl(runtime: Runtime) {
             outEl.innerHTML = '';
 
             // execute the python code
-            const pyResult = await pyExec(runtime, pySrc, outEl);
+            const pyResult = pyExec(runtime, pySrc, outEl);
 
             // display the value of the last evaluated expression (REPL-style)
             if (pyResult !== undefined) {

pyscriptjs/src/components/pyscript.ts

@@ -12,7 +12,7 @@ export function make_PyScript(runtime: Runtime) {
             ensureUniqueId(this);
             const pySrc = await this.getPySrc();
             this.innerHTML = '';
-            await pyExec(runtime, pySrc, this);
+            pyExec(runtime, pySrc, this);
         }
 
         async getPySrc(): Promise<string> {

pyscriptjs/src/pyexec.ts

@@ -1,18 +1,30 @@
 import { getLogger } from './logger';
-import { ensureUniqueId } from './utils';
+import { ensureUniqueId, ltrim } from './utils';
+import { UserError } from './exceptions';
 import type { Runtime } from './runtime';
 
 const logger = getLogger('pyexec');
 
-export async function pyExec(runtime: Runtime, pysrc: string, outElem: HTMLElement) {
+export function pyExec(runtime: Runtime, pysrc: string, outElem: HTMLElement) {
     // this is the python function defined in pyscript.py
     const set_current_display_target = runtime.globals.get('set_current_display_target');
     ensureUniqueId(outElem);
     set_current_display_target(outElem.id);
+    //This is the python function defined in pyscript.py
+    const usesTopLevelAwait = runtime.globals.get('uses_top_level_await')
     try {
         try {
-            return await runtime.run(pysrc);
+            if (usesTopLevelAwait(pysrc)){
-        } catch (err) {
+                throw new UserError(
+                'The use of top-level "await", "async for", and ' +
+                '"async with" is deprecated.' +
+                '\nPlease write a coroutine containing ' +
+                'your code and schedule it using asyncio.ensure_future() or similar.' +
+                '\nSee https://docs.pyscript.net/latest/guides/asyncio.html for more information.'
+                )
+       }
+       return runtime.run(pysrc);
+   } catch (err) {
             // XXX: currently we display exceptions in the same position as
             // the output. But we probably need a better way to do that,
             // e.g. allowing plugins to intercept exceptions and display them

pyscriptjs/src/pyodide.ts

@@ -75,8 +75,8 @@ export class PyodideRuntime extends Runtime {
         logger.info('pyodide loaded and initialized');
     }
 
-    async run(code: string): Promise<any> {
+    run(code: string) {
-        return await this.interpreter.runPythonAsync(code);
+        return this.interpreter.runPython(code);
     }
 
     registerJsModule(name: string, module: object): void {

pyscriptjs/src/python/pyscript.py

@@ -1,3 +1,4 @@
+import ast
 import asyncio
 import base64
 import html
@@ -404,4 +405,28 @@ class PyListTemplate:
         pass
 
 
+class TopLevelAsyncFinder(ast.NodeVisitor):
+    def is_source_top_level_await(self, source):
+        self.async_found = False
+        node = ast.parse(source)
+        self.generic_visit(node)
+        return self.async_found
+
+    def visit_Await(self, node):
+        self.async_found = True
+
+    def visit_AsyncFor(self, node):
+        self.async_found = True
+
+    def visit_AsyncWith(self, node):
+        self.async_found = True
+
+    def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef):
+        pass  # Do not visit children of async function defs
+
+
+def uses_top_level_await(source: str) -> bool:
+    return TopLevelAsyncFinder().is_source_top_level_await(source)
+
+
 pyscript = PyScript()

pyscriptjs/src/runtime.ts

@@ -55,7 +55,7 @@ export abstract class Runtime extends Object {
      * (asynchronously) which can call its own API behind the scenes.
      * Python exceptions are turned into JS exceptions.
      * */
-    abstract run(code: string): Promise<unknown>;
+    abstract run(code: string);
 
     /**
      * Same as run, but Python exceptions are not propagated: instead, they
@@ -64,11 +64,16 @@ export abstract class Runtime extends Object {
      * This is a bad API and should be killed/refactored/changed eventually,
      * but for now we have code which relies on it.
      * */
-    async runButDontRaise(code: string): Promise<unknown> {
+    runButDontRaise(code: string) {
-        return this.run(code).catch(err => {
+        let result
-            const error = err as Error;
+        try{
-            logger.error('Error:', error);
+            result = this.run(code)
-        });
+        }
+        catch (err){
+            const error = err as Error
+            logger.error('Error:', error)
+        }
+        return result
     }
 
     /**

pyscriptjs/tests/integration/test_03_async.py

@@ -2,56 +2,146 @@ from .support import PyScriptTest
 
 
 class TestAsync(PyScriptTest):
+    # ensure_future() and create_task() should behave similarly;
+    # we'll use the same source code to test both
+    coroutine_script = """
+        <py-script>
+        import js
+        import asyncio
+        js.console.log("first")
+        async def main():
+            await asyncio.sleep(1)
+            js.console.log("third")
+        asyncio.{func}(main())
+        js.console.log("second")
+        </py-script>
+        """
+
+    def test_asyncio_ensure_future(self):
+        self.pyscript_run(self.coroutine_script.format(func="ensure_future"))
+        self.wait_for_console("third")
+        assert self.console.log.lines == [self.PY_COMPLETE, "first", "second", "third"]
+
+    def test_asyncio_create_task(self):
+        self.pyscript_run(self.coroutine_script.format(func="create_task"))
+        self.wait_for_console("third")
+        assert self.console.log.lines == [self.PY_COMPLETE, "first", "second", "third"]
+
+    def test_asyncio_gather(self):
+        self.pyscript_run(
+            """
+            <py-script id="pys">
+            import asyncio
+            import js
+            from pyodide.ffi import to_js
+
+            async def coro(delay):
+                await asyncio.sleep(delay)
+                return(delay)
+
+            async def get_results():
+                results = await asyncio.gather(*[coro(d) for d in range(3,0,-1)])
+                js.console.log(to_js(results))
+                js.console.log("DONE")
+
+            asyncio.ensure_future(get_results())
+            </py-script>
+            """
+        )
+        self.wait_for_console("DONE")
+        assert self.console.log.lines[-2:] == ["[3,2,1]", "DONE"]
+
     def test_multiple_async(self):
         self.pyscript_run(
             """
         <py-script>
             import js
             import asyncio
-            for i in range(3):
+            async def a_func():
-                js.console.log('A', i)
+                for i in range(3):
-                await asyncio.sleep(0.1)
+                    js.console.log('A', i)
+                    await asyncio.sleep(0.1)
+            asyncio.ensure_future(a_func())
         </py-script>
 
         <py-script>
             import js
             import asyncio
-            for i in range(3):
+            async def b_func():
-                js.console.log('B', i)
+                for i in range(3):
-                await asyncio.sleep(0.1)
+                    js.console.log('B', i)
-            js.console.log("async tadone")
+                    await asyncio.sleep(0.1)
+                js.console.log('b func done')
+            asyncio.ensure_future(b_func())
         </py-script>
         """
         )
-        self.wait_for_console("async tadone")
+        self.wait_for_console("b func done")
         assert self.console.log.lines == [
-            "Python initialization complete",
+            self.PY_COMPLETE,
             "A 0",
             "B 0",
             "A 1",
             "B 1",
             "A 2",
             "B 2",
-            "async tadone",
+            "b func done",
         ]
 
-    def test_multiple_async_multiple_display(self):
+    def test_multiple_async_multiple_display_targetted(self):
+        self.pyscript_run(
+            """
+                <py-script id='pyA'>
+                    import js
+                    import asyncio
+
+                    async def a_func():
+                        for i in range(2):
+                            display(f'A{i}', target='pyA')
+                            await asyncio.sleep(0.1)
+                    asyncio.ensure_future(a_func())
+
+                </py-script>
+                <py-script id='pyB'>
+                    import js
+                    import asyncio
+
+                    async def a_func():
+                        for i in range(2):
+                            display(f'B{i}', target='pyB')
+                            await asyncio.sleep(0.1)
+                        js.console.log("B DONE")
+
+                    asyncio.ensure_future(a_func())
+                </py-script>
+            """
+        )
+        self.wait_for_console("B DONE")
+        inner_text = self.page.inner_text("html")
+        assert "A0\nA1\nB0\nB1" in inner_text
+
+    def test_async_display_untargetted(self):
         self.pyscript_run(
             """
                 <py-script id='pyA'>
                     import asyncio
-                    for i in range(2):
+                    import js
-                        display('A')
-                        await asyncio.sleep(0)
-                </py-script>
 
-                <py-script id='pyB'>
+                    async def a_func():
-                    import asyncio
+                        try:
-                    for i in range(2):
+                            display('A')
-                        display('B')
+                            await asyncio.sleep(0.1)
-                        await asyncio.sleep(0)
+                        except Exception as err:
+                            js.console.error(str(err))
+                        await asyncio.sleep(1)
+                        js.console.log("DONE")
+
+                    asyncio.ensure_future(a_func())
                 </py-script>
             """
         )
-        inner_text = self.page.inner_text("html")
+        self.wait_for_console("DONE")
-        assert "A\nB\nA\nB" in inner_text
+        assert (
+            self.console.error.lines[-1]
+            == "Implicit target not allowed here. Please use display(..., target=...)"
+        )

pyscriptjs/tests/integration/test_importmap.py

@@ -1,6 +1,9 @@
+import pytest
+
 from .support import PyScriptTest
 
 
+@pytest.mark.xfail(reason="See PR #938")
 class TestImportmap(PyScriptTest):
     def test_importmap(self):
         src = """

pyscriptjs/tests/integration/test_py_config.py

@@ -100,7 +100,7 @@ class TestConfig(PyScriptTest):
         """,
         )
 
-        assert self.console.log.lines == [self.PY_COMPLETE, "version 0.20.0"]
+        assert self.console.log.lines[-1] == "version 0.20.0"
         version = self.page.locator("py-script").inner_text()
         assert version == "0.20.0"
 

pyscriptjs/tests/integration/test_py_repl.py

@@ -63,13 +63,9 @@ class TestPyRepl(PyScriptTest):
             </py-repl>
             """
         )
+        self.page.wait_for_selector("#runButton")
         self.page.keyboard.press("Shift+Enter")
-
+        assert self.console.log.lines == [self.PY_COMPLETE, "hello world"]
-        # when we use locator('button').click() the message appears
-        # immediately, with keyboard.press we need to wait for it. I don't
-        # really know why it has a different behavior, I didn't investigate
-        # further.
-        self.wait_for_console("hello world")
 
     def test_display(self):
         self.pyscript_run(

pyscriptjs/tests/py-unit/test_pyscript.py

@@ -1,4 +1,5 @@
 import sys
+import textwrap
 from unittest.mock import Mock
 
 import pyscript
@@ -48,3 +49,71 @@ def test_format_mime_HTML():
     out, mime = pyscript.format_mime(obj)
     assert out == "<p>hello</p>"
     assert mime == "text/html"
+
+
+def test_uses_top_level_await():
+    # Basic Case
+    src = "x = 1"
+    assert pyscript.uses_top_level_await(src) is False
+
+    # Comments are not top-level await
+    src = textwrap.dedent(
+        """
+        #await async for async with asyncio
+        """
+    )
+
+    assert pyscript.uses_top_level_await(src) is False
+
+    # Top-level-await cases
+    src = textwrap.dedent(
+        """
+        async def foo():
+            pass
+        await foo
+        """
+    )
+    assert pyscript.uses_top_level_await(src) is True
+
+    src = textwrap.dedent(
+        """
+        async with object():
+            pass
+        """
+    )
+    assert pyscript.uses_top_level_await(src) is True
+
+    src = textwrap.dedent(
+        """
+        async for _ in range(10):
+            pass
+        """
+    )
+    assert pyscript.uses_top_level_await(src) is True
+
+    # Acceptable await/async for/async with cases
+    src = textwrap.dedent(
+        """
+        async def foo():
+            await foo()
+        """
+    )
+    assert pyscript.uses_top_level_await(src) is False
+
+    src = textwrap.dedent(
+        """
+        async def foo():
+            async with object():
+                pass
+        """
+    )
+    assert pyscript.uses_top_level_await(src) is False
+
+    src = textwrap.dedent(
+        """
+        async def foo():
+            async for _ in range(10):
+                pass
+        """
+    )
+    assert pyscript.uses_top_level_await(src) is False