Python Plugins (#961)

* add test and example files

* update config to include python plugins in build

* add markdown plugin

* remove full pyscript execution from pyodide

* move loading of pyscript.py from pyodide loagInterpreter to main setupVirtualEnv and add function to create python CE plugins

* add plugin class to pyscript.py

* add missing import

* fix plugin path

* add fetchPythonPlugins to PyScriptApp

* remove old comments

* fix test

* add support for python plugins beyond custom elements and add app to python namespace in main

* inject reference to PyScript app onto python plugins

* add example hook onto markdown plugin

* change plugin events logs

* remove unused PyPlugin

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* fix type import

* add docstring to fetchPythonPlugins

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* rename addPythonPlugin method

* address PR comment

* call python plugins on hooks after the interpreted is ready

* add test for event hooks and split the test in 2 separate plugins to isolte type of plugins tests

* change python plugins initialization and registration, to inject the app from app itself instead of on the plugins themselves

* handle case when plugin cannot load due to missing plugin attribute

* add test for fail scenario when a plugin module does not have a plugin attribute

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* add deprecation warning for pyscript objects loaded in global namespace

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* remove all from global scope

* remove create_custom_element from global scope

* rename create_custom_element to define_custom_element

* rename attributes in define_custom_element and add docstrings

* better handle connect event output

* add warning to py_markdown plugin

* remove debugging logs

* improve tests

* remove debugging log

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* remove unused import

* add executable shebang

* add pyodide mock module

* fmt and lint

* Update to pyodide.ffi.create_proxy per pyodide v21 api change

* Mock pyodide as package instead of mdoule

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* add __init__ to pyodide package

* Update pyscriptjs/src/plugin.ts

fix logger name

Co-authored-by: Antonio Cuni <anto.cuni@gmail.com>

* fix pyodide import but handling the diff in their API change

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* oops, conflict resolution blooper

* Fix failing integration tests

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Jeff Glass <glass.jeffrey@gmail.com>
Co-authored-by: Antonio Cuni <anto.cuni@gmail.com>
Co-authored-by: FabioRosado <fabiorosado@outlook.com>

pyscriptjs/tests/integration/test_01_basic.py

@@ -14,10 +14,8 @@ class TestBasic(PyScriptTest):
             </py-script>
             """
         )
-        assert self.console.log.lines == [
-            self.PY_COMPLETE,
-            "hello pyscript",
-        ]
+        assert self.console.log.lines[0] == self.PY_COMPLETE
+        assert self.console.log.lines[-1] == "hello pyscript"
 
     def test_python_exception(self):
         self.pyscript_run(
@@ -28,7 +26,8 @@ class TestBasic(PyScriptTest):
             </py-script>
         """
         )
-        assert self.console.log.lines == [self.PY_COMPLETE, "hello pyscript"]
+        assert self.console.log.lines[0] == self.PY_COMPLETE
+        assert "hello pyscript" in self.console.log.lines
         # check that we sent the traceback to the console
         tb_lines = self.console.error.lines[-1].splitlines()
         assert tb_lines[0] == "[pyexec] Python exception:"
@@ -57,8 +56,8 @@ class TestBasic(PyScriptTest):
             <py-script>js.console.log('four')</py-script>
         """
         )
-        assert self.console.log.lines == [
-            self.PY_COMPLETE,
+        assert self.console.log.lines[0] == self.PY_COMPLETE
+        assert self.console.log.lines[-4:] == [
             "one",
             "two",
             "three",
@@ -75,7 +74,9 @@ class TestBasic(PyScriptTest):
             <py-script>js.console.log("<div></div>")</py-script>
         """
         )
-        assert self.console.log.lines == [self.PY_COMPLETE, "true false", "<div></div>"]
+
+        assert self.console.log.lines[0] == self.PY_COMPLETE
+        assert self.console.log.lines[-2:] == ["true false", "<div></div>"]
 
     def test_packages(self):
         self.pyscript_run(
@@ -92,8 +93,9 @@ class TestBasic(PyScriptTest):
             </py-script>
             """
         )
-        assert self.console.log.lines == [
-            self.PY_COMPLETE,
+
+        assert self.console.log.lines[0] == self.PY_COMPLETE
+        assert self.console.log.lines[-3:] == [
             "Loading asciitree",  # printed by pyodide
             "Loaded asciitree",  # printed by pyodide
             "hello asciitree",  # printed by us
@@ -114,10 +116,9 @@ class TestBasic(PyScriptTest):
         )
         self.page.locator("button").click()
         self.page.locator("py-script")  # wait until <py-script> appears
-        assert self.console.log.lines == [
-            self.PY_COMPLETE,
-            "hello world",
-        ]
+
+        assert self.console.log.lines[0] == self.PY_COMPLETE
+        assert self.console.log.lines[-1] == "hello world"
 
     def test_py_script_src_attribute(self):
         self.writefile("foo.py", "print('hello from foo')")
@@ -126,10 +127,8 @@ class TestBasic(PyScriptTest):
             <py-script src="foo.py"></py-script>
             """
         )
-        assert self.console.log.lines == [
-            self.PY_COMPLETE,
-            "hello from foo",
-        ]
+        assert self.console.log.lines[0] == self.PY_COMPLETE
+        assert self.console.log.lines[-1] == "hello from foo"
 
     def test_py_script_src_not_found(self):
         self.pyscript_run(
@@ -137,9 +136,8 @@ class TestBasic(PyScriptTest):
             <py-script src="foo.py"></py-script>
             """
         )
-        assert self.console.log.lines == [
-            self.PY_COMPLETE,
-        ]
+        assert self.PY_COMPLETE in self.console.log.lines
+
         assert "Failed to load resource" in self.console.error.lines[0]
         with pytest.raises(JsErrors) as exc:
             self.check_js_errors()
@@ -192,3 +190,28 @@ class TestBasic(PyScriptTest):
             )
             is not None
         )
+
+    def test_python_modules_deprecated(self):
+        # GIVEN a py-script tag
+        self.pyscript_run(
+            """
+            <py-script>
+                print('hello pyscript')
+                raise Exception('this is an error')
+            </py-script>
+        """
+        )
+        # TODO: Adding a quick check that the deprecation warning is logged. Not spending
+        #       to much time to make it perfect since we'll remove this right after the
+        #       release. (Anyone wanting to improve it, please feel free to)
+        warning_msg = (
+            "[pyscript/main] DEPRECATION WARNING: 'micropip', 'Element', 'console', 'document' "
+            "and several other objects form the pyscript module (with the exception of 'display') "
+            "will be be removed from the Python global namespace in the following release. "
+            "To avoid errors in future releases use import from pyscript "
+            "instead. For instance: from pyscript import micropip, Element, "
+            "console, document"
+        )
+        # we EXPECTED to find a deprecation warning about what will be removed from the Python
+        # global namespace in the next releases
+        assert warning_msg in self.console.warning.lines

pyscriptjs/tests/integration/test_02_output.py

@@ -301,8 +301,8 @@ class TestOutput(PyScriptTest):
         )
         inner_text = self.page.inner_text("py-script")
         assert inner_text == "this goes to the DOM"
-        assert self.console.log.lines == [
-            self.PY_COMPLETE,
+        assert self.console.log.lines[0] == self.PY_COMPLETE
+        assert self.console.log.lines[-2:] == [
             "print from python",
             "print from js",
         ]

pyscriptjs/tests/integration/test_03_async.py

@@ -20,12 +20,14 @@ class TestAsync(PyScriptTest):
     def test_asyncio_ensure_future(self):
         self.pyscript_run(self.coroutine_script.format(func="ensure_future"))
         self.wait_for_console("third")
-        assert self.console.log.lines == [self.PY_COMPLETE, "first", "second", "third"]
+        assert self.console.log.lines[0] == self.PY_COMPLETE
+        assert self.console.log.lines[-3:] == ["first", "second", "third"]
 
     def test_asyncio_create_task(self):
         self.pyscript_run(self.coroutine_script.format(func="create_task"))
         self.wait_for_console("third")
-        assert self.console.log.lines == [self.PY_COMPLETE, "first", "second", "third"]
+        assert self.console.log.lines[0] == self.PY_COMPLETE
+        assert self.console.log.lines[-3:] == ["first", "second", "third"]
 
     def test_asyncio_gather(self):
         self.pyscript_run(
@@ -77,8 +79,10 @@ class TestAsync(PyScriptTest):
         """
         )
         self.wait_for_console("b func done")
-        assert self.console.log.lines == [
-            self.PY_COMPLETE,
+        assert self.console.log.lines[0] == self.PY_COMPLETE
+        # We are getting some deprecation warnings from pyodide, so we
+        # need to skip the first 2 lines
+        assert self.console.log.lines[3:] == [
             "A 0",
             "B 0",
             "A 1",

pyscriptjs/tests/integration/test_04_runtime.py

@@ -22,8 +22,8 @@ class TestRuntimeAccess(PyScriptTest):
         """
         )
 
-        assert self.console.log.lines == [
-            self.PY_COMPLETE,
+        assert self.console.log.lines[0] == self.PY_COMPLETE
+        assert self.console.log.lines[-2:] == [
             "x is 1",
             "py_func() returns 2",
         ]
@@ -38,5 +38,5 @@ class TestRuntimeAccess(PyScriptTest):
         interpreter.runPython('console.log("Interpreter Ran This")');
         """
         )
-
-        assert self.console.log.lines == [self.PY_COMPLETE, "Interpreter Ran This"]
+        assert self.console.log.lines[0] == self.PY_COMPLETE
+        assert self.console.log.lines[-1] == "Interpreter Ran This"

pyscriptjs/tests/integration/test_plugins.py

@@ -0,0 +1,184 @@
+from .support import PyScriptTest
+
+# Source code of a simple plugin that creates a Custom Element for testing purposes
+CE_PLUGIN_CODE = """
+from pyscript import Plugin, console
+
+plugin = Plugin('py-upper')
+
+console.log("py_upper Plugin loaded")
+
+@plugin.register_custom_element('py-up')
+class Upper:
+    def __init__(self, element):
+        self.element = element
+
+    def connect(self):
+        console.log("Upper plugin connected")
+        return self.element.originalInnerHTML.upper()
+"""
+
+# Source of a plugin hooks into the PyScript App lifecycle events
+HOOKS_PLUGIN_CODE = """
+from pyscript import Plugin, console
+
+class TestLogger(Plugin):
+    def configure(self, config):
+        console.log('configure called')
+
+    def beforeLaunch(self, config):
+        console.log('beforeLaunch called')
+
+    def afterSetup(self, config):
+        console.log('afterSetup called')
+
+    def afterStartup(self, config):
+        console.log('afterStartup called')
+
+    def onUserError(self, config):
+        console.log('onUserError called')
+
+
+plugin = TestLogger()
+"""
+
+# Source of a script that doesn't call define a `plugin` attribute
+NO_PLUGIN_CODE = """
+from pyscript import Plugin, console
+
+class TestLogger(Plugin):
+    pass
+"""
+
+# Source code of a simple plugin that creates a Custom Element for testing purposes
+CODE_CE_PLUGIN_BAD_RETURNS = """
+from pyscript import Plugin, console
+
+plugin = Plugin('py-broken')
+
+@plugin.register_custom_element('py-up')
+class Upper:
+    def __init__(self, element):
+        self.element = element
+
+    def connect(self):
+        # Just returning something... anything other than a string should be ignore
+        return Plugin
+"""
+HTML_TEMPLATE_WITH_TAG = """
+    <py-config>
+        plugins = [
+            "./{plugin_name}.py"
+        ]
+    </py-config>
+
+    <{tagname}>
+        {html}
+    </{tagname}>
+"""
+HTML_TEMPLATE_NO_TAG = """
+    <py-config>
+        plugins = [
+            "./{plugin_name}.py"
+        ]
+    </py-config>
+"""
+
+
+def prepare_test(
+    plugin_name, code, tagname="", html="", template=HTML_TEMPLATE_WITH_TAG
+):
+    """
+    Prepares the test by writing a new plugin file named `plugin_name`.py, with `code` as its
+    content and run `pyscript_run` on `template` formatted with the above inputs to create the
+    page HTML code.
+
+    For example:
+
+    >> @prepare_test('py-upper', CE_PLUGIN_CODE, tagname='py-up', html="Hello World")
+    >> def my_foo(...):
+    >>     ...
+
+    will:
+
+    * write a new `py-upper.py` file to the FS
+    * the contents of `py-upper.py` is equal to CE_PLUGIN_CODE
+    * call self.pyscript_run with the following string:
+        '''
+            <py-config>
+                plugins = [
+                    "./py-upper.py"
+                ]
+            </py-config>
+
+            <py-up>
+                {html}
+            </py-up>
+        '''
+    * call `my_foo` just like a normal decorator would
+
+    """
+
+    def dec(f):
+        def _inner(self, *args, **kws):
+            self.writefile(f"{plugin_name}.py", code)
+            page_html = template.format(
+                plugin_name=plugin_name, tagname=tagname, html=html
+            )
+            self.pyscript_run(page_html)
+            return f(self, *args, **kws)
+
+        return _inner
+
+    return dec
+
+
+class TestPlugin(PyScriptTest):
+    @prepare_test("py-upper", CE_PLUGIN_CODE, tagname="py-up", html="Hello World")
+    def test_py_plugin_inline(self):
+        """Test that a regular plugin that returns new HTML content from connected works"""
+        # GIVEN a plugin that returns the all caps version of the tag innerHTML and logs text
+        # during it's execution/hooks
+
+        # EXPECT the plugin logs to be present in the console logs
+        log_lines = self.console.log.lines
+        for log_line in ["py_upper Plugin loaded", "Upper plugin connected"]:
+            assert log_line in log_lines
+
+        # EXPECT the inner text of the Plugin CustomElement to be all caps
+        rendered_text = self.page.locator("py-up").inner_text()
+        assert rendered_text == "HELLO WORLD"
+
+    @prepare_test("hooks_logger", HOOKS_PLUGIN_CODE, template=HTML_TEMPLATE_NO_TAG)
+    def test_execution_hooks(self):
+        """Test that a Plugin that hooks into the PyScript App events, gets called
+        for each one of them"""
+        # GIVEN a plugin that logs specific strings for each app execution event
+        hooks_available = ["afterSetup", "afterStartup"]
+        hooks_unavailable = ["configure", "beforeLaunch"]
+
+        # EXPECT it to log the correct logs for the events it intercepts
+        log_lines = self.console.log.lines
+        for method in hooks_available:
+            assert f"{method} called" in log_lines
+
+        # EXPECT it to NOT be called (hence not log anything) the events that happen
+        # before it's ready, hence is not called
+        for method in hooks_unavailable:
+            assert f"{method} called" not in log_lines
+
+        # TODO: It'd be actually better to check that the events get called in order
+
+    @prepare_test("no_plugin", NO_PLUGIN_CODE)
+    def test_no_plugin_attribute_error(self):
+        """
+        Test a plugin that do not add the `plugin` attribute to its module
+        """
+        # GIVEN a Plugin NO `plugin` attribute in it's module
+        error_msg = (
+            "[pyscript/main] Cannot find plugin on Python module no_plugin! Python plugins "
+            'modules must contain a "plugin" attribute. For more information check the '
+            "plugins documentation."
+        )
+        # EXPECT an error for the missing attribute
+        assert error_msg in self.console.error.lines

pyscriptjs/tests/integration/test_py_button.py

@@ -12,10 +12,11 @@ class TestPyButton(PyScriptTest):
             </py-button>
         """
         )
-        assert self.console.log.lines == [self.PY_COMPLETE]
+        assert self.console.log.lines[0] == self.PY_COMPLETE
         self.page.locator("text=my button").click()
         self.page.locator("text=my button").click()
-        assert self.console.log.lines == [self.PY_COMPLETE, "clicked!", "clicked!"]
+
+        assert self.console.log.lines[-2:] == ["clicked!", "clicked!"]
 
     def test_deprecated_element(self):
         self.pyscript_run(

pyscriptjs/tests/integration/test_py_config.py

@@ -228,8 +228,8 @@ class TestConfig(PyScriptTest):
             </py-script>
             """
         )
-        assert self.console.log.lines == [
-            self.PY_COMPLETE,
+        assert self.console.log.lines[0] == self.PY_COMPLETE
+        assert self.console.log.lines[-2:] == [
             "hello from A",
             "hello from B",
         ]
@@ -279,7 +279,5 @@ class TestConfig(PyScriptTest):
             </py-script>
             """
         )
-        assert self.console.log.lines == [
-            self.PY_COMPLETE,
-            "hello from A",
-        ]
+        assert self.console.log.lines[0] == self.PY_COMPLETE
+        assert self.console.log.lines[-1] == "hello from A"

pyscriptjs/tests/integration/test_py_inputbox.py

@@ -13,13 +13,13 @@ class TestPyInputBox(PyScriptTest):
             </py-inputbox>
             """
         )
-        assert self.console.log.lines == [self.PY_COMPLETE]
+        assert self.console.log.lines[0] == self.PY_COMPLETE
         input = self.page.locator("input")
 
         input.type("Hello")
         input.press("Enter")
 
-        assert self.console.log.lines == [self.PY_COMPLETE, "Hello"]
+        assert self.console.log.lines[-1] == "Hello"
 
     def test_deprecated_element(self):
         self.pyscript_run(

pyscriptjs/tests/integration/test_py_repl.py

@@ -65,7 +65,9 @@ class TestPyRepl(PyScriptTest):
         )
         self.page.wait_for_selector("#runButton")
         self.page.keyboard.press("Shift+Enter")
-        assert self.console.log.lines == [self.PY_COMPLETE, "hello world"]
+
+        assert self.console.log.lines[0] == self.PY_COMPLETE
+        assert self.console.log.lines[-1] == "hello world"
 
     def test_display(self):
         self.pyscript_run(

pyscriptjs/tests/integration/test_py_terminal.py

@@ -32,8 +32,7 @@ class TestPyTerminal(PyScriptTest):
             "this goes to stderr",
             "this goes to stdout",
         ]
-        assert self.console.log.lines == [
-            self.PY_COMPLETE,
+        assert self.console.log.lines[-3:] == [
             "hello world",
             "this goes to stderr",
             "this goes to stdout",

pyscriptjs/tests/integration/test_splashscreen.py

@@ -34,11 +34,9 @@ class TestSplashscreen(PyScriptTest):
         # and now the splashscreen should have been removed
         expect(div).to_be_hidden()
         assert self.page.locator("py-locator").count() == 0
-        #
-        assert self.console.log.lines == [
-            self.PY_COMPLETE,
-            "hello pyscript",
-        ]
+
+        assert self.console.log.lines[0] == self.PY_COMPLETE
+        assert "hello pyscript" in self.console.log.lines
 
     def test_autoclose_false(self):
         self.pyscript_run(
@@ -56,10 +54,8 @@ class TestSplashscreen(PyScriptTest):
         expect(div).to_be_visible()
         expect(div).to_contain_text("Python startup...")
         expect(div).to_contain_text("Startup complete")
-        assert self.console.log.lines == [
-            self.PY_COMPLETE,
-            "hello pyscript",
-        ]
+        assert self.console.log.lines[0] == self.PY_COMPLETE
+        assert "hello pyscript" in self.console.log.lines
 
     def test_autoclose_loader_deprecated(self):
         self.pyscript_run(
@@ -75,12 +71,10 @@ class TestSplashscreen(PyScriptTest):
         warning = self.page.locator(".py-warning")
         inner_text = warning.inner_text()
         assert "The setting autoclose_loader is deprecated" in inner_text
-        #
+
         div = self.page.locator("py-splashscreen > div")
         expect(div).to_be_visible()
         expect(div).to_contain_text("Python startup...")
         expect(div).to_contain_text("Startup complete")
-        assert self.console.log.lines == [
-            self.PY_COMPLETE,
-            "hello pyscript",
-        ]
+        assert self.console.log.lines[0] == self.PY_COMPLETE
+        assert "hello pyscript" in self.console.log.lines

pyscriptjs/tests/py-unit/pyodide/ffi.py

@@ -0,0 +1,4 @@
+"""Mock module that emulates some of the pyodide js module features for the sake of tests"""
+from unittest.mock import Mock
+
+create_proxy = Mock()

pyscriptjs/tests/py-unit/pyodide/pyodide.py

@@ -0,0 +1,4 @@
+"""Mock module that emulates some of the pyodide js module features for the sake of tests"""
+from unittest.mock import Mock
+
+create_proxy = Mock()