Enable worker tests (#1757)

This PR re-enables tests on `worker`s. Highlights:
 
* by default, each test is run twice: the main thread version uses `<script type="py">`, the worker version automatically turn the tags into `<script type="py" worker>`

* you can tweak the settings per-class by using the `@with_execution_thread` decorator. In particular, `@with_execution_thread(None)` is for those tests which don't care about it (e.g., `test_py_config.py`)

* inside each class, there might be some test which should be run only in the main thread (because it doesn't make sense to test it in a worker). For those, I introduced the `@only_main` decorator

* we might introduce `@only_worker` in the future, if needed

* `@skip_worker` is for those tests which currently pass on main but not on workers. These are meant to be temporary, and eventually they should all be fixed
 
During the process, I tweaked/improved/fixed/deleted some of the existing tests. Some of them were at risk of being flaky and I made them more robust, others depended on some very precise implementation detail, and I made them more generic (for example, `test_image_renders_correctly` relied on pillow to render an image with a very specific string of bytes, and it broke due to the recent upgrade to pyodide 0.24.1)
 
I also renamed all the skip messages to start with `NEXT`, so that they are easier to grep.

pyscript.core/src/stdlib/pyscript/display.py

@@ -10,9 +10,9 @@ _MIME_METHODS = {
     "_repr_html_": "text/html",
     "_repr_markdown_": "text/markdown",
     "_repr_svg_": "image/svg+xml",
-    "_repr_png_": "image/png",
     "_repr_pdf_": "application/pdf",
     "_repr_jpeg_": "image/jpeg",
+    "_repr_png_": "image/png",
     "_repr_latex": "text/latex",
     "_repr_json_": "application/json",
     "_repr_javascript_": "application/javascript",

pyscript.core/tests/integration/support.py

@@ -104,6 +104,20 @@ def skip_worker(reason):
     return decorator
 
 
+def only_main(fn):
+    """
+    Decorator to mark a test which make sense only in the main thread
+    """
+
+    @functools.wraps(fn)
+    def decorated(self, *args):
+        if self.execution_thread == "worker":
+            return
+        return fn(self, *args)
+
+    return decorated
+
+
 def filter_inner_text(text, exclude=None):
     return "\n".join(filter_page_content(text.splitlines(), exclude=exclude))
 
@@ -126,7 +140,7 @@ def filter_page_content(lines, exclude=None):
 
 
 @pytest.mark.usefixtures("init")
-@with_execution_thread("main") # , "worker") # XXX re-enable workers eventually
+@with_execution_thread("main", "worker")
 class PyScriptTest:
     """
     Base class to write PyScript integration tests, based on playwright.
@@ -179,7 +193,7 @@ class PyScriptTest:
         # create a symlink to BUILD inside tmpdir
         tmpdir.join("build").mksymlinkto(BUILD)
         self.tmpdir.chdir()
-        self.tmpdir.join('favicon.ico').write("")
+        self.tmpdir.join("favicon.ico").write("")
         self.logger = logger
         self.execution_thread = execution_thread
         self.dev_server = None
@@ -376,7 +390,12 @@ class PyScriptTest:
         self.page.goto(url, timeout=0)
 
     def wait_for_console(
-        self, text, *, match_substring=False, timeout=None, check_js_errors=True
+        self,
+        text,
+        *,
+        match_substring=False,
+        timeout=None,
+        check_js_errors=True,
     ):
         """
         Wait until the given message appear in the console. If the message was
@@ -440,9 +459,15 @@ class PyScriptTest:
         If check_js_errors is True (the default), it also checks that no JS
         errors were raised during the waiting.
         """
-        # this is printed by interpreter.ts:Interpreter.initialize
+        scripts = (
+            self.page.locator("script[type=py]").all()
+            + self.page.locator("py-script").all()
+        )
+        n_scripts = len(scripts)
+
+        # this is printed by core.js:onAfterRun
         elapsed_ms = self.wait_for_console(
-            "[pyscript/main] PyScript Ready",
+            "---py:all-done---",
             timeout=timeout,
             check_js_errors=check_js_errors,
         )
@@ -453,54 +478,13 @@ class PyScriptTest:
         # events aren't being triggered in the tests.
         self.page.wait_for_timeout(100)
 
-    def _parse_py_config(self, doc):
-        configs = re.findall("<py-config>(.*?)</py-config>", doc, flags=re.DOTALL)
-        configs = [cfg.strip() for cfg in configs]
-        if len(configs) == 0:
-            return None
-        elif len(configs) == 1:
-            return toml.loads(configs[0])
-        else:
-            raise AssertionError("Too many <py-config>")
-
-    def _inject_execution_thread_config(self, snippet, execution_thread):
-        """
-        If snippet already contains a py-config, let's try to inject
-        execution_thread automatically. Note that this works only for plain
-        <py-config> with inline config: type="json" and src="..." are not
-        supported by this logic, which should remain simple.
-        """
-        cfg = self._parse_py_config(snippet)
-        if cfg is None:
-            # we don't have any <py-config>, let's add one
-            py_config_maybe = f"""
-            <py-config>
-                execution_thread = "{execution_thread}"
-            </py-config>
-            """
-        else:
-            cfg["execution_thread"] = execution_thread
-            dumped_cfg = toml.dumps(cfg)
-            new_py_config = f"""
-            <py-config>
-                {dumped_cfg}
-            </py-config>
-            """
-            snippet = re.sub(
-                "<py-config>.*</py-config>", new_py_config, snippet, flags=re.DOTALL
-            )
-            # no need for extra config, it's already in the snippet
-            py_config_maybe = ""
-        #
-        return snippet, py_config_maybe
+    SCRIPT_TAG_REGEX = re.compile('(<script type="py"|<py-script)')
 
     def _pyscript_format(self, snippet, *, execution_thread, extra_head=""):
-        if execution_thread is None:
-            py_config_maybe = ""
-        else:
-            snippet, py_config_maybe = self._inject_execution_thread_config(
-                snippet, execution_thread
-            )
+        if execution_thread == "worker":
+            # turn <script type="py"> into <script type="py" worker>, and
+            # similarly for <py-script>
+            snippet = self.SCRIPT_TAG_REGEX.sub(r"\1 worker", snippet)
 
         doc = f"""
         <html>
@@ -510,10 +494,19 @@ class PyScriptTest:
                     type="module"
                     src="{self.http_server_addr}/build/core.js"
                 ></script>
+              <script type="module">
+                addEventListener(
+                  'py:all-done',
+                  () => {{
+                    console.debug('---py:all-done---')
+                  }},
+                  {{ once: true }}
+                );
+              </script>
+
               {extra_head}
           </head>
           <body>
-            {py_config_maybe}
             {snippet}
           </body>
         </html>
@@ -578,7 +571,7 @@ class PyScriptTest:
         Ensure that there is an alert banner on the page with the given message.
         Currently it only handles a single.
         """
-        banner = self.page.wait_for_selector(".alert-banner")
+        banner = self.page.wait_for_selector(".py-error")
         banner_text = banner.inner_text()
 
         if expected_message not in banner_text:

pyscript.core/tests/integration/test_00_support.py

@@ -474,22 +474,3 @@ class TestSupport(PyScriptTest):
         assert [
             "Failed to load resource: the server responded with a status of 404 (Not Found)"
         ] == self.console.all.lines
-
-    def test__pyscript_format_inject_execution_thread(self):
-        """
-        This is slightly different than other tests: it doesn't use playwright, it
-        just tests that our own internal helper works
-        """
-        doc = self._pyscript_format("<b>Hello</b>", execution_thread="main")
-        cfg = self._parse_py_config(doc)
-        assert cfg == {"execution_thread": "main"}
-
-    def test__pyscript_format_modify_existing_py_config(self):
-        src = """
-        <py-config>
-            hello = 42
-        </py-config>
-        """
-        doc = self._pyscript_format(src, execution_thread="main")
-        cfg = self._parse_py_config(doc)
-        assert cfg == {"execution_thread": "main", "hello": 42}

pyscript.core/tests/integration/test_01_basic.py

@@ -2,7 +2,7 @@ import re
 
 import pytest
 
-from .support import PyScriptTest
+from .support import PyScriptTest, skip_worker, only_main
 
 
 class TestBasic(PyScriptTest):
@@ -11,40 +11,45 @@ class TestBasic(PyScriptTest):
             """
             <script type="py">
                 import js
-                js.console.log('hello from script py')
+                js.console.log('1. hello from script py')
             </script>
 
             <py-script>
                 import js
-                js.console.log('hello from py-script')
+                js.console.log('2. hello from py-script')
             </py-script>
             """
         )
+        if self.execution_thread == "main":
+            # in main, the order of execution is guaranteed
             assert self.console.log.lines == [
-            "hello from script py",
-            "hello from py-script",
+                "1. hello from script py",
+                "2. hello from py-script",
             ]
+        else:
+            # in workers, each tag is executed by its own worker, so they can
+            # come out of order
+            lines = sorted(self.console.log.lines)
+            assert lines == ["1. hello from script py", "2. hello from py-script"]
 
     def test_execution_thread(self):
         self.pyscript_run(
             """
-            <!-- we don't really need anything here, we just want to check that
-                 pyscript does not bootstrap -->
+            <script type="py">
+                import pyscript
+                import js
+                js.console.log("worker?", pyscript.RUNNING_IN_WORKER)
+            </script>
             """,
-            wait_for_pyscript=False,
         )
         assert self.execution_thread in ("main", "worker")
-        if self.execution_thread == "main":
-            pass
-        elif self.execution_thread == "worker":
-            pass
-        assert self.console.log.lines == []
+        in_worker = self.execution_thread == "worker"
+        in_worker = str(in_worker).lower()
+        assert self.console.log.lines[-1] == f"worker? {in_worker}"
 
     # TODO: if there's no py-script there are surely no plugins neither
     #       this test must be discussed or rewritten to make sense now
-    @pytest.mark.skip(
-        reason="FIXME: No banner and should also add a WARNING about CORS"
-    )
+    @pytest.mark.skip(reason="NEXT: No banner and should also add a WARNING about CORS")
     def test_no_cors_headers(self):
         self.disable_cors_headers()
         self.pyscript_run(
@@ -79,6 +84,7 @@ class TestBasic(PyScriptTest):
         )
         assert self.console.log.lines[-1] == "hello pyscript"
 
+    @skip_worker("NEXT: exceptions should be displayed in the DOM")
     def test_python_exception(self):
         self.pyscript_run(
             """
@@ -104,6 +110,7 @@ class TestBasic(PyScriptTest):
         assert tb_lines[0] == "Traceback (most recent call last):"
         assert tb_lines[-1] == "Exception: this is an error"
 
+    @skip_worker("NEXT: py-click doesn't work inside workers")
     def test_python_exception_in_event_handler(self):
         self.pyscript_run(
             """
@@ -131,6 +138,7 @@ class TestBasic(PyScriptTest):
         assert tb_lines[0] == "Traceback (most recent call last):"
         assert tb_lines[-1] == "Exception: this is an error inside handler"
 
+    @only_main
     def test_execution_in_order(self):
         """
         Check that they script py tags are executed in the same order they are
@@ -151,6 +159,7 @@ class TestBasic(PyScriptTest):
             "four",
         ]
 
+    @skip_worker("NEXT: something very weird happens here")
     def test_escaping_of_angle_brackets(self):
         """
         Check that script tags escape angle brackets
@@ -158,13 +167,16 @@ class TestBasic(PyScriptTest):
         self.pyscript_run(
             """
             <script type="py">import js; js.console.log("A", 1<2, 1>2)</script>
-            <script type="py">js.console.log("B <div></div>")</script>
+            <script type="py">import js; js.console.log("B <div></div>")</script>
             <py-script>import js; js.console.log("C", 1<2, 1>2)</py-script>
-            <py-script>js.console.log("D <div></div>")</py-script>
+            <py-script>import js; js.console.log("D <div></div>")</py-script>
 
         """
         )
-        assert self.console.log.lines[-4:] == [
+        # in workers the order of execution is not guaranteed, better to play
+        # safe
+        lines = sorted(self.console.log.lines[-4:])
+        assert lines == [
             "A true false",
             "B <div></div>",
             "C true false",
@@ -191,7 +203,7 @@ class TestBasic(PyScriptTest):
             "hello asciitree",  # printed by us
         ]
 
-    @pytest.mark.skip("FIXME: No banner")
+    @pytest.mark.skip("NEXT: No banner")
     def test_non_existent_package(self):
         self.pyscript_run(
             """
@@ -215,7 +227,7 @@ class TestBasic(PyScriptTest):
         assert expected_alert_banner_msg in alert_banner.inner_text()
         self.check_py_errors("Can't fetch metadata for 'i-dont-exist'")
 
-    @pytest.mark.skip("FIXME: No banner")
+    @pytest.mark.skip("NEXT: No banner")
     def test_no_python_wheel(self):
         self.pyscript_run(
             """
@@ -238,6 +250,7 @@ class TestBasic(PyScriptTest):
         assert expected_alert_banner_msg in alert_banner.inner_text()
         self.check_py_errors("Can't find a pure Python 3 wheel for 'opsdroid'")
 
+    @only_main
     def test_dynamically_add_py_script_tag(self):
         self.pyscript_run(
             """
@@ -265,6 +278,7 @@ class TestBasic(PyScriptTest):
         )
         assert self.console.log.lines[-1] == "hello from foo"
 
+    @skip_worker("NEXT: banner not shown")
     def test_py_script_src_not_found(self):
         self.pyscript_run(
             """
@@ -275,17 +289,12 @@ class TestBasic(PyScriptTest):
         assert "Failed to load resource" in self.console.error.lines[0]
 
         # TODO: we need to be sure errors make sense from both main and worker worlds
-        # expected_msg = "(PY0404): Fetching from URL foo.py failed with error 404"
-        # assert any((expected_msg in line) for line in self.console.js_error.lines)
-        # assert self.assert_banner_message(expected_msg)
-
-        # pyscript_tag = self.page.locator("script-py")
-        # assert pyscript_tag.inner_html() == ""
-
-        # self.check_js_errors(expected_msg)
+        expected_msg = "(PY0404): Fetching from URL foo.py failed with error 404"
+        assert any((expected_msg in line) for line in self.console.error.lines)
+        assert self.assert_banner_message(expected_msg)
 
     # TODO: ... and we shouldn't: it's a module and we better don't leak in global
-    @pytest.mark.skip("DIFFERENT BEHAVIOUR: we don't expose pyscript on window")
+    @pytest.mark.skip("NEXT: we don't expose pyscript on window")
     def test_js_version(self):
         self.pyscript_run(
             """
@@ -301,7 +310,7 @@ class TestBasic(PyScriptTest):
         )
 
     # TODO: ... and we shouldn't: it's a module and we better don't leak in global
-    @pytest.mark.skip("DIFFERENT BEHAVIOUR: we don't expose pyscript on window")
+    @pytest.mark.skip("NEXT: we don't expose pyscript on window")
     def test_python_version(self):
         self.pyscript_run(
             """
@@ -325,22 +334,7 @@ class TestBasic(PyScriptTest):
             is not None
         )
 
-    def test_assert_no_banners(self):
-        """
-        Test that the DOM doesn't contain error/warning banners
-        """
-        self.pyscript_run(
-            """
-            <script type="py">
-                import sys
-                print("hello world", file=sys.stderr)
-            </script>
-            """
-        )
-
-        assert self.page.locator(".py-error").inner_text() == "hello world"
-
-    @pytest.mark.skip("ERROR_SCRIPT: works with <py-script> not with <script>")
+    @pytest.mark.skip("NEXT: works with <py-script> not with <script>")
     def test_getPySrc_returns_source_code(self):
         self.pyscript_run(
             """
@@ -360,6 +354,7 @@ class TestBasic(PyScriptTest):
             == 'print("hello from script py")'
         )
 
+    @skip_worker("NEXT: py-click doesn't work inside workers")
     def test_py_attribute_without_id(self):
         self.pyscript_run(
             """
@@ -387,8 +382,5 @@ class TestBasic(PyScriptTest):
             </script>
             """
         )
-        btn = self.page.wait_for_selector("button")
-        btn.click()
-        self.wait_for_console("1")
-        assert self.console.log.lines[-1] == "2"
+        assert self.console.log.lines == ["1", "2"]
         assert self.console.error.lines == []

pyscript.core/tests/integration/test_02_display.py

@@ -1,3 +1,5 @@
+################################################################################
+
 import base64
 import io
 import os
@@ -13,9 +15,11 @@ from .support import (
     filter_inner_text,
     filter_page_content,
     wait_for_render,
+    skip_worker,
+    only_main,
 )
 
-DISPLAY_OUTPUT_ID_PATTERN = r'[id^="py-"]'
+DISPLAY_OUTPUT_ID_PATTERN = r'script-py[id^="py-"]'
 
 
 class TestDisplay(PyScriptTest):
@@ -68,6 +72,7 @@ class TestDisplay(PyScriptTest):
         mydiv = self.page.locator("#mydiv")
         assert mydiv.inner_text() == "hello world"
 
+    @skip_worker("NEXT: display(target=...) does not work")
     def test_tag_target_attribute(self):
         self.pyscript_run(
             """
@@ -87,6 +92,7 @@ class TestDisplay(PyScriptTest):
         goodbye = self.page.locator("#goodbye")
         assert goodbye.inner_text() == "goodbye world"
 
+    @skip_worker("NEXT: display target does not work properly")
     def test_target_script_py(self):
         self.pyscript_run(
             """
@@ -105,6 +111,7 @@ class TestDisplay(PyScriptTest):
         text = self.page.inner_text("body")
         assert text == "ONE\nTWO\nTHREE"
 
+    @skip_worker("NEXT: display target does not work properly")
     def test_consecutive_display_target(self):
         self.pyscript_run(
             """
@@ -142,6 +149,7 @@ class TestDisplay(PyScriptTest):
         lines = tag.inner_text().splitlines()
         assert lines == ["hello", "world"]
 
+    @only_main  # with workers, two tags are two separate interpreters
     def test_implicit_target_from_a_different_tag(self):
         self.pyscript_run(
             """
@@ -163,6 +171,7 @@ class TestDisplay(PyScriptTest):
         assert py0.inner_text() == ""
         assert py1.inner_text() == "hello"
 
+    @skip_worker("NEXT: py-click doesn't work")
     def test_no_explicit_target(self):
         self.pyscript_run(
             """
@@ -179,6 +188,7 @@ class TestDisplay(PyScriptTest):
         text = self.page.locator("script-py").text_content()
         assert "hello world" in text
 
+    @skip_worker("NEXT: display target does not work properly")
     def test_explicit_target_pyscript_tag(self):
         self.pyscript_run(
             """
@@ -195,6 +205,7 @@ class TestDisplay(PyScriptTest):
         text = self.page.locator("script-py").nth(1).inner_text()
         assert text == "hello"
 
+    @skip_worker("NEXT: display target does not work properly")
     def test_explicit_target_on_button_tag(self):
         self.pyscript_run(
             """
@@ -327,7 +338,7 @@ class TestDisplay(PyScriptTest):
         )
         out = self.page.locator("script-py > div")
         assert out.inner_html() == html.escape("<p>hello world</p>")
-        assert out.inner_text() == '<p>hello world</p>'
+        assert out.inner_text() == "<p>hello world</p>"
 
     def test_display_HTML(self):
         self.pyscript_run(
@@ -342,9 +353,7 @@ class TestDisplay(PyScriptTest):
         assert out.inner_html() == "<p>hello world</p>"
         assert out.inner_text() == "hello world"
 
-    # waiit_for_pyscript is broken: it waits until the python code is about to
-    # start, to until the python code has finished execution
-    @pytest.mark.skip("FIXME: wait_for_pyscript is broken")
+    @skip_worker("NEXT: matplotlib-pyodide backend does not work")
     def test_image_display(self):
         self.pyscript_run(
             """
@@ -357,7 +366,8 @@ class TestDisplay(PyScriptTest):
                     plt.plot(xpoints, ypoints)
                     display(plt)
                 </script>
-            """
+            """,
+            timeout=30 * 1000,
         )
         wait_for_render(self.page, "*", "<img src=['\"]data:image")
         test = self.page.wait_for_selector("img")
@@ -428,15 +438,12 @@ class TestDisplay(PyScriptTest):
         assert console_text.index("1print") == (console_text.index("2print") - 1)
         assert console_text.index("1console") == (console_text.index("2console") - 1)
 
+    @skip_worker("NEXT: display target does not work properly")
     def test_image_renders_correctly(self):
-        """This is just a sanity check to make sure that images are rendered correctly."""
-        buffer = io.BytesIO()
-        img = Image.new("RGB", (4, 4), color=(0, 0, 0))
-        img.save(buffer, format="PNG")
-
-        b64_img = base64.b64encode(buffer.getvalue()).decode("utf-8")
-        expected_img_src = f"data:image/png;charset=utf-8;base64,{b64_img}"
-
+        """
+        This is just a sanity check to make sure that images are rendered
+        in a reasonable way.
+        """
         self.pyscript_run(
             """
             <py-config>
@@ -453,5 +460,5 @@ class TestDisplay(PyScriptTest):
             """
         )
 
-        rendered_img_src = self.page.locator("img").get_attribute("src")
-        assert rendered_img_src == expected_img_src
+        img_src = self.page.locator("img").get_attribute("src")
+        assert img_src.startswith('data:image/png;charset=utf-8;base64')

pyscript.core/tests/integration/test_async.py

@@ -1,5 +1,5 @@
 import pytest
-from .support import PyScriptTest, filter_inner_text
+from .support import PyScriptTest, filter_inner_text, only_main
 
 
 class TestAsync(PyScriptTest):
@@ -52,6 +52,7 @@ class TestAsync(PyScriptTest):
         self.wait_for_console("DONE")
         assert self.console.log.lines[-2:] == ["[3, 2, 1]", "DONE"]
 
+    @only_main
     def test_multiple_async(self):
         self.pyscript_run(
             """
@@ -88,6 +89,7 @@ class TestAsync(PyScriptTest):
             "b func done",
         ]
 
+    @only_main
     def test_multiple_async_multiple_display_targeted(self):
         self.pyscript_run(
             """
@@ -145,6 +147,7 @@ class TestAsync(PyScriptTest):
         self.wait_for_console("DONE")
         assert self.page.locator("script-py").inner_text() == "A"
 
+    @only_main
     def test_sync_and_async_order(self):
         """
         The order of execution is defined as follows:

pyscript.core/tests/integration/test_interpreter.py

@@ -3,7 +3,7 @@ import pytest
 from .support import PyScriptTest
 
 pytest.skip(
-    reason="FIXME: pyscript API changed doesn't expose pyscript to window anymore",
+    reason="NEXT: pyscript API changed doesn't expose pyscript to window anymore",
     allow_module_level=True,
 )
 

pyscript.core/tests/integration/test_plugins.py

@@ -3,7 +3,7 @@ import pytest
 from .support import PyScriptTest, skip_worker
 
 pytest.skip(
-    reason="FIX LATER: pyscript NEXT doesn't support plugins yet",
+    reason="NEXT: plugins not supported",
     allow_module_level=True,
 )
 

pyscript.core/tests/integration/test_py_config.py

@@ -4,6 +4,7 @@ import pytest
 
 from .support import PyScriptTest, with_execution_thread
 
+
 # Disable the main/worker dual testing, for two reasons:
 #
 #   1. the <py-config> logic happens before we start the worker, so there is
@@ -30,7 +31,7 @@ class TestConfig(PyScriptTest):
         )
         assert self.console.log.lines[-1] == "config name: foobar"
 
-    @pytest.mark.skip("ERROR_SCRIPT: works with <py-script> not with <script>")
+    @pytest.mark.skip("NEXT: works with <py-script> not with <script>")
     def test_py_config_inline_scriptpy(self):
         self.pyscript_run(
             """
@@ -47,8 +48,7 @@ class TestConfig(PyScriptTest):
         )
         assert self.console.log.lines[-1] == "config name: foobar"
 
-
-    @pytest.mark.skip("ERROR_SCRIPT: works with <py-script> not with <script>")
+    @pytest.mark.skip("NEXT: works with <py-script> not with <script>")
     def test_py_config_external(self):
         pyconfig_toml = """
             name = "app with external config"
@@ -67,8 +67,7 @@ class TestConfig(PyScriptTest):
         )
         assert self.console.log.lines[-1] == "config name: app with external config"
 
-
-    @pytest.mark.skip("FIXME: We need to restore the banner.")
+    @pytest.mark.skip("NEXT: We need to restore the banner.")
     def test_invalid_json_config(self):
         # we need wait_for_pyscript=False because we bail out very soon,
         # before being able to write 'PyScript page fully initialized'
@@ -82,10 +81,7 @@ class TestConfig(PyScriptTest):
         )
         banner = self.page.wait_for_selector(".py-error")
         # assert "Unexpected end of JSON input" in self.console.error.text
-        expected = (
-            "(PY1000): Invalid JSON\n"
-            "Unexpected end of JSON input"
-        )
+        expected = "(PY1000): Invalid JSON\n" "Unexpected end of JSON input"
         assert banner.inner_text() == expected
 
     def test_invalid_toml_config(self):
@@ -108,7 +104,7 @@ class TestConfig(PyScriptTest):
         )
         assert banner.inner_text() == expected
 
-    @pytest.mark.skip("FIXME: emit a warning in case of multiple py-config")
+    @pytest.mark.skip("NEXT: emit a warning in case of multiple py-config")
     def test_multiple_py_config(self):
         self.pyscript_run(
             """
@@ -157,7 +153,7 @@ class TestConfig(PyScriptTest):
             "hello from B",
         ]
 
-    @pytest.mark.skip("FIXME: emit an error if fetch fails")
+    @pytest.mark.skip("NEXT: emit an error if fetch fails")
     def test_paths_that_do_not_exist(self):
         self.pyscript_run(
             """

pyscript.core/tests/integration/test_py_repl.py

@@ -5,7 +5,7 @@ import pytest
 from .support import PyScriptTest, skip_worker
 
 pytest.skip(
-    reason="FIX LATER: pyscript NEXT doesn't support the REPL yet",
+    reason="NEXT: pyscript NEXT doesn't support the REPL yet",
     allow_module_level=True,
 )
 

pyscript.core/tests/integration/test_script_type.py

@@ -1,8 +1,12 @@
 import pytest
 
-from .support import PyScriptTest
+from .support import PyScriptTest, with_execution_thread
 
 
+# these tests don't need to run in 'main' and 'worker' modes: the workers are
+# already tested explicitly by some of them (see e.g.
+# test_script_type_py_worker_attribute)
+@with_execution_thread(None)
 class TestScriptTypePyScript(PyScriptTest):
     def test_display_line_break(self):
         self.pyscript_run(
@@ -81,7 +85,6 @@ class TestScriptTypePyScript(PyScriptTest):
         )
         assert self.console.log.lines[-1] == "hello from foo"
 
-    @pytest.mark.skip("FIXME: wait_for_pyscript is broken")
     def test_script_type_py_worker_attribute(self):
         self.writefile("foo.py", "print('hello from foo')")
         self.pyscript_run(

pyscript.core/tests/integration/test_shadow_root.py

@@ -4,8 +4,7 @@ from .support import PyScriptTest
 
 
 class TestShadowRoot(PyScriptTest):
-    # @skip_worker("FIXME: js.document")
-    @pytest.mark.skip("FIXME: Element interface is gone. Replace with PyDom")
+    @pytest.mark.skip("NEXT: Element interface is gone. Replace with PyDom")
     def test_reachable_shadow_root(self):
         self.pyscript_run(
             r"""

pyscript.core/tests/integration/test_splashscreen.py

@@ -3,9 +3,7 @@ from playwright.sync_api import expect
 
 from .support import PyScriptTest, skip_worker
 
-pytest.skip(
-    reason="DECIDE: Should we remove the splashscreen?", allow_module_level=True
-)
+pytest.skip(reason="NEXT: Should we remove the splashscreen?", allow_module_level=True)
 
 
 class TestSplashscreen(PyScriptTest):

pyscript.core/tests/integration/test_stdio_handling.py

@@ -2,7 +2,7 @@ import pytest
 
 from .support import PyScriptTest, skip_worker
 
-pytest.skip(reason="FIXME: entire stdio should be reviewed", allow_module_level=True)
+pytest.skip(reason="NEXT: entire stdio should be reviewed", allow_module_level=True)
 
 
 class TestOutputHandling(PyScriptTest):

pyscript.core/tests/integration/test_style.py

@@ -1,9 +1,10 @@
 import pytest
 from playwright.sync_api import expect
 
-from .support import PyScriptTest, skip_worker
+from .support import PyScriptTest, with_execution_thread
 
 
+@with_execution_thread(None)
 class TestStyle(PyScriptTest):
     def test_pyscript_not_defined(self):
         """Test raw elements that are not defined for display:none"""

pyscript.core/tests/integration/test_warnings_and_banners.py

@@ -2,7 +2,7 @@ import pytest
 
 from .support import PyScriptTest
 
-pytest.skip(reason="FIXME: Restore the banner", allow_module_level=True)
+pytest.skip(reason="NEXT: Restore the banner", allow_module_level=True)
 
 
 class TestWarningsAndBanners(PyScriptTest):

pyscript.core/tests/integration/test_when.py

@@ -1,6 +1,6 @@
 import pytest
 
-from .support import PyScriptTest
+from .support import PyScriptTest, skip_worker
 
 
 class TestEventHandler(PyScriptTest):
@@ -15,14 +15,12 @@ class TestEventHandler(PyScriptTest):
                 from pyscript import when
                 @when("click", selector="#foo_id")
                 def foo(evt):
-                    print(f"I've clicked {evt.target} with id {evt.target.id}")
+                    print(f"clicked {evt.target.id}")
             </script>
         """
         )
         self.page.locator("text=foo_button").click()
-        console_text = self.console.all.lines
-        self.wait_for_console("I've clicked [object HTMLButtonElement] with id foo_id")
-        assert "I've clicked [object HTMLButtonElement] with id foo_id" in console_text
+        self.wait_for_console("clicked foo_id")
         self.assert_no_banners()
 
     def test_when_decorator_without_event(self):
@@ -42,7 +40,6 @@ class TestEventHandler(PyScriptTest):
         )
         self.page.locator("text=foo_button").click()
         self.wait_for_console("The button was clicked")
-        assert "The button was clicked" in self.console.log.lines
         self.assert_no_banners()
 
     def test_multiple_when_decorators_with_event(self):
@@ -53,23 +50,18 @@ class TestEventHandler(PyScriptTest):
             <script type="py">
                 from pyscript import when
                 @when("click", selector="#foo_id")
-                def foo(evt):
-                    print(f"I've clicked {evt.target} with id {evt.target.id}")
+                def foo_click(evt):
+                    print(f"foo_click! id={evt.target.id}")
                 @when("click", selector="#bar_id")
-                def foo(evt):
-                    print(f"I've clicked {evt.target} with id {evt.target.id}")
+                def bar_click(evt):
+                    print(f"bar_click! id={evt.target.id}")
             </script>
         """
         )
         self.page.locator("text=foo_button").click()
-        console_text = self.console.all.lines
-        self.wait_for_console("I've clicked [object HTMLButtonElement] with id foo_id")
-        assert "I've clicked [object HTMLButtonElement] with id foo_id" in console_text
-
+        self.wait_for_console("foo_click! id=foo_id")
         self.page.locator("text=bar_button").click()
-        console_text = self.console.all.lines
-        self.wait_for_console("I've clicked [object HTMLButtonElement] with id bar_id")
-        assert "I've clicked [object HTMLButtonElement] with id bar_id" in console_text
+        self.wait_for_console("bar_click! id=bar_id")
         self.assert_no_banners()
 
     def test_two_when_decorators(self):
@@ -83,15 +75,14 @@ class TestEventHandler(PyScriptTest):
                 @when("click", selector="#foo_id")
                 @when("mouseover", selector=".bar_class")
                 def foo(evt):
-                    print(f"An event of type {evt.type} happened")
+                    print(f"got event: {evt.type}")
             </script>
         """
         )
         self.page.locator("text=bar_button").hover()
+        self.wait_for_console("got event: mouseover")
         self.page.locator("text=foo_button").click()
-        self.wait_for_console("An event of type click happened")
-        assert "An event of type mouseover happened" in self.console.log.lines
-        assert "An event of type click happened" in self.console.log.lines
+        self.wait_for_console("got event: click")
         self.assert_no_banners()
 
     def test_two_when_decorators_same_element(self):
@@ -104,15 +95,14 @@ class TestEventHandler(PyScriptTest):
                 @when("click", selector="#foo_id")
                 @when("mouseover", selector="#foo_id")
                 def foo(evt):
-                    print(f"An event of type {evt.type} happened")
+                    print(f"got event: {evt.type}")
             </script>
         """
         )
         self.page.locator("text=foo_button").hover()
+        self.wait_for_console("got event: mouseover")
         self.page.locator("text=foo_button").click()
-        self.wait_for_console("An event of type click happened")
-        assert "An event of type mouseover happened" in self.console.log.lines
-        assert "An event of type click happened" in self.console.log.lines
+        self.wait_for_console("got event: click")
         self.assert_no_banners()
 
     def test_when_decorator_multiple_elements(self):
@@ -148,19 +138,18 @@ class TestEventHandler(PyScriptTest):
                 @when("click", selector="#foo_id")
                 @when("click", selector="#foo_id")
                 def foo(evt):
-                    print(f"I've clicked {evt.target} with id {evt.target.id}")
+                    foo.n += 1
+                    print(f"click {foo.n} on {evt.target.id}")
+                foo.n = 0
             </script>
         """
         )
         self.page.locator("text=foo_button").click()
-        console_text = self.console.all.lines
-        self.wait_for_console("I've clicked [object HTMLButtonElement] with id foo_id")
-        assert (
-            console_text.count("I've clicked [object HTMLButtonElement] with id foo_id")
-            == 2
-        )
+        self.wait_for_console("click 1 on foo_id")
+        self.wait_for_console("click 2 on foo_id")
         self.assert_no_banners()
 
+    @skip_worker("NEXT: error banner not shown")
     def test_when_decorator_invalid_selector(self):
         """When the selector parameter of @when is invalid, it should show an error"""
         self.pyscript_run(

pyscript.core/tests/integration/test_zzz_docs_snippets.py

@@ -1,305 +0,0 @@
-import re
-import pytest
-
-from .support import PyScriptTest, skip_worker
-
-
-@pytest.mark.skip(
-    reason="SKIPPING Docs: these should be reviewed ALL TOGETHER as we fix docs"
-)
-class TestDocsSnippets(PyScriptTest):
-    @skip_worker("FIXME: js.document")
-    def test_tutorials_py_click(self):
-        self.pyscript_run(
-            """
-            <button
-              py-click="current_time()"
-              id="get-time" class="py-button">
-              Get current time
-            </button>
-            <p id="current-time"></p>
-
-            <script type="py">
-                from pyscript import Element
-                import datetime
-
-                def current_time():
-                    now = datetime.datetime.now()
-
-                    # Get paragraph element by id
-                    paragraph = Element("current-time")
-
-                    # Add current time to the paragraph element
-                    paragraph.write(now.strftime("%Y-%m-%d %H:%M:%S"))
-            </script>
-            """
-        )
-
-        btn = self.page.wait_for_selector("#get-time")
-        btn.click()
-
-        current_time = self.page.wait_for_selector("#current-time")
-
-        pattern = "\\d+-\\d+-\\d+\\s\\d+:\\d+:\\d+"  # e.g. 08-09-2022 15:57:32
-        assert re.search(pattern, current_time.inner_text())
-        self.assert_no_banners()
-
-    def test_tutorials_requests(self):
-        self.pyscript_run(
-            """
-            <py-config>
-                packages = ["requests", "pyodide-http"]
-            </py-config>
-
-            <script type="py">
-                import requests
-                import pyodide_http
-
-                # Patch the Requests library so it works with Pyscript
-                pyodide_http.patch_all()
-
-                # Make a request to the JSON Placeholder API
-                response = requests.get("https://jsonplaceholder.typicode.com/todos")
-                print(response.json())
-            </script>
-            """
-        )
-
-        py_terminal = self.page.wait_for_selector("py-terminal")
-        # Just a small check to confirm that the response was received
-        assert "userId" in py_terminal.inner_text()
-        self.assert_no_banners()
-
-    @skip_worker("FIXME: js.document")
-    def test_tutorials_py_config_fetch(self):
-        # flake8: noqa
-        self.pyscript_run(
-            """
-                <py-config>
-                [[fetch]]
-                from = "https://pyscript.net/examples/"
-                files = ["utils.py"]
-                [[fetch]]
-                from = "https://gist.githubusercontent.com/FabioRosado/faba0b7f6ad4438b07c9ac567c73b864/raw/37603b76dc7ef7997bf36781ea0116150f727f44/"
-                files = ["todo.py"]
-                </py-config>
-                <script type="py">
-                    from todo import add_task, add_task_event
-                </script>
-                <section>
-                <div class="text-center w-full mb-8">
-                    <h1 class="text-3xl font-bold text-gray-800 uppercase tracking-tight">
-                        To Do List
-                    </h1>
-                </div>
-                <div>
-                    <input id="new-task-content" class="py-input" type="text">
-                    <button id="new-task-btn" class="py-button" type="submit" py-click="add_task()">
-                    Add task
-                    </button>
-                </div>
-                <div id="list-tasks-container" class="flex flex-col-reverse mt-4"></div>
-                <template id="task-template">
-                    <section class="task py-li-element">
-                    <label for="flex items-center p-2 ">
-                        <input class="mr-2" type="checkbox">
-                        <p class="m-0 inline"></p>
-                    </label>
-                    </section>
-                </template
-            """
-        )
-
-        todo_input = self.page.locator("input")
-        submit_task_button = self.page.locator("button")
-
-        todo_input.type("Fold laundry")
-        submit_task_button.click()
-
-        first_task = self.page.locator("#task-0")
-        assert "Fold laundry" in first_task.inner_text()
-
-        task_checkbox = first_task.locator("input")
-        # Confirm that the new task isn't checked
-        assert not task_checkbox.is_checked()
-
-        # Let's mark it as done now
-        task_checkbox.check()
-
-        # Basic check that the task has the line-through class
-        assert (
-            '<p class="m-0 inline line-through">Fold laundry</p>'
-            in first_task.inner_html()
-        )
-        self.assert_no_banners()
-
-    def test_tutorials_py_config_interpreter(self):
-        """Load a previous version of Pyodide"""
-        self.pyscript_run(
-            """
-            <py-config>
-                [[interpreters]]
-                    src = "https://cdn.jsdelivr.net/pyodide/v0.23.0/full/pyodide.js"
-                    name = "pyodide-0.23.0"
-                    lang = "python"
-            </py-config>
-            <script type="py">
-                import pyodide
-                print(pyodide.__version__)
-            </script>
-            """
-        )
-
-        py_terminal = self.page.wait_for_selector("py-terminal")
-        assert "0.23.0" in py_terminal.inner_text()
-        self.assert_no_banners()
-
-    @skip_worker("FIXME: display()")
-    def test_tutorials_writing_to_page(self):
-        self.pyscript_run(
-            """
-            <div id="manual-write"></div>
-            <button py-click="write_to_page()" id="manual">Say Hello</button>
-            <div id="display-write"></div>
-            <button py-click="display_to_div()" id="display">Say Things!</button>
-            <div>
-                <py-terminal>
-            </div>
-            <button py-click="print_to_page()" id="print">Print Things!</button>
-
-            <script type="py">
-            def write_to_page():
-                manual_div = Element("manual-write")
-                manual_div.element.innerHTML = "<p><b>Hello World</b></p>"
-
-            def display_to_div():
-                display("I display things!", target="display-write")
-
-            def print_to_page():
-                print("I print things!")
-            </script>
-            """
-        )
-        btn_manual = self.page.wait_for_selector("#manual")
-        btn_display = self.page.wait_for_selector("#display")
-        btn_print = self.page.wait_for_selector("#print")
-
-        btn_manual.click()
-        manual_write_div = self.page.wait_for_selector("#manual-write")
-        assert "<p><b>Hello World</b></p>" in manual_write_div.inner_html()
-
-        btn_display.click()
-        display_write_div = self.page.wait_for_selector("#display-write")
-        assert "I display things!" in display_write_div.inner_text()
-
-        btn_print.click()
-        py_terminal = self.page.wait_for_selector("py-terminal")
-        assert "I print things!" in py_terminal.inner_text()
-        self.assert_no_banners()
-
-    def test_guides_asyncio(self):
-        self.pyscript_run(
-            """
-            <script type="py">
-                import asyncio
-
-                async def main():
-                    for i in range(3):
-                        print(i)
-
-                asyncio.ensure_future(main())
-            </script>
-            """
-        )
-        py_terminal = self.page.wait_for_selector("py-terminal")
-
-        assert "0\n1\n2\n" in py_terminal.inner_text()
-
-    @skip_worker("FIXME: js.document")
-    def test_reference_pyterminal_xterm(self):
-        self.pyscript_run(
-            """
-            <py-config>
-                xterm = true
-            </py-config>
-            <script type="py">
-                print("HELLO!")
-                import js
-                import asyncio
-
-                async def adjust_term_size(columns, rows):
-                    xterm = await js.document.querySelector('py-terminal').xtermReady
-                    xterm.resize(columns, rows)
-                    print("test-done")
-
-                asyncio.ensure_future(adjust_term_size(40, 10))
-                </script>
-            """
-        )
-        self.page.get_by_text("test-done").wait_for()
-
-        py_terminal = self.page.locator("py-terminal")
-        print(dir(py_terminal))
-        print(type(py_terminal))
-        assert py_terminal.evaluate("el => el.xterm.cols") == 40
-        assert py_terminal.evaluate("el => el.xterm.rows") == 10
-
-    @skip_worker(reason="FIXME: js.document (@when decorator)")
-    def test_reference_when_simple(self):
-        self.pyscript_run(
-            """
-            <button id="my_btn">Click Me to Say Hi</button>
-            <script type="py">
-                from pyscript import when
-                @when("click", selector="#my_btn")
-                def say_hello():
-                    print(f"Hello, world!")
-            </script>
-            """
-        )
-        self.page.get_by_text("Click Me to Say Hi").click()
-        self.wait_for_console("Hello, world!")
-        assert ("Hello, world!") in self.console.log.lines
-
-    @skip_worker(reason="FIXME: js.document (@when decorator)")
-    def test_reference_when_complex(self):
-        self.pyscript_run(
-            """
-            <div id="container">
-                <button>First</button>
-                <button>Second</button>
-                <button>Third</button>
-            </div>
-            <script type="py">
-                from pyscript import when
-                import js
-
-                @when("click", selector="#container button")
-                def highlight(evt):
-                    #Set the clicked button's background to green
-                    evt.target.style.backgroundColor = 'green'
-
-                    #Set the background of all buttons to red
-                    other_buttons = (button for button in js.document.querySelectorAll('button') if button != evt.target)
-                    for button in other_buttons:
-                        button.style.backgroundColor = 'red'
-
-                    print("set") # Test Only
-            </script>
-            """
-        )
-
-        def getBackgroundColor(locator):
-            return locator.evaluate(
-                "(element) => getComputedStyle(element).getPropertyValue('background-color')"
-            )
-
-        first_button = self.page.get_by_text("First")
-        assert getBackgroundColor(first_button) == "rgb(239, 239, 239)"
-
-        first_button.click()
-        self.wait_for_console("set")
-
-        assert getBackgroundColor(first_button) == "rgb(0, 128, 0)"
-        assert getBackgroundColor(self.page.get_by_text("Second")) == "rgb(255, 0, 0)"
-        assert getBackgroundColor(self.page.get_by_text("Third")) == "rgb(255, 0, 0)"