jprdonnelly/pyscript
1
0
Fork 0
mirror of https://github.com/pyscript/pyscript.git synced 2026-02-17 10:01:09 -05:00
Code Issues Projects Releases Wiki Activity

Introduce PyScriptTest, an helper class to write integration tests (#663)

Browse Source 
This PR is about integration tests: they use playwright to load HTML pages in the browser and check that PyScript works as intended, as opposed to unit tests like the ones being introduced by #665 and #661.

The main goal of this PR is to introduce some machinery to make such tests easier to write, read and maintain, with some attention to capture enough information to produce useful error messages in case they fail in the CI.

In order to use the machinery, you need to subclass tests.support.PyScriptTest, which provides several useful API calls in the form self.xxx().

See the full description here:
https://github.com/pyscript/pyscript/pull/663

Co-authored-by: Mariana Meireles <marian.meireles@gmail.com>
Co-authored-by: mariana <marianameireles@protonmail.com>
This commit is contained in:
Antonio Cuni
2022-08-10 12:29:59 +02:00
committed by GitHub
parent bd7a20309b
commit 513dfe0b42
8 changed files with 684 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
pyscriptjs/src/components/pyconfig.ts
View File

@@ -119,6 +119,7 @@ export class PyodideRuntime extends Object {
for (const initializer of postInitializers_) {
await initializer();
}
console.log('===PyScript page fully initialized===');
}
}

16
pyscriptjs/tests/conftest.py
View File

@@ -5,6 +5,13 @@ from http.server import SimpleHTTPRequestHandler
import pytest
from .support import Logger
@pytest.fixture(scope="session")
def logger():
return Logger()
class HTTPServer(SuperHTTPServer):
"""
@@ -23,12 +30,17 @@ class HTTPServer(SuperHTTPServer):
@pytest.fixture(scope="session")
def http_server():
def http_server(logger):
class MyHTTPRequestHandler(SimpleHTTPRequestHandler):
def log_message(self, fmt, *args):
logger.log("http_server", fmt % args, color="blue")
host, port = "127.0.0.1", 8080
base_url = f"http://{host}:{port}"
# serve_Run forever under thread
server = HTTPServer((host, port), SimpleHTTPRequestHandler)
server = HTTPServer((host, port), MyHTTPRequestHandler)
thread = threading.Thread(None, server.run)
thread.start()

369
pyscriptjs/tests/support.py Normal file
View File

@@ -0,0 +1,369 @@
import time
import py
import pytest
ROOT = py.path.local(__file__).dirpath("..", "..")
BUILD = ROOT.join("pyscriptjs", "build")
@pytest.mark.usefixtures("init")
class PyScriptTest:
"""
Base class to write PyScript integration tests, based on playwright.
It provides a simple API to generate HTML files and load them in
playwright.
It also provides a Pythonic API on top of playwright for the most
common tasks; in particular:
- self.console collects all the JS console.* messages. Look at the doc
of ConsoleMessageCollection for more details.
- self.check_errors() checks that no JS errors have been thrown
- after each test, self.check_errors() is automatically run to ensure
that no JS error passes uncaught.
- self.wait_for_console waits until the specified message appears in the
console
- self.wait_for_pyscript waits until all the PyScript tags have been
evaluated
- self.pyscript_run is the main entry point for pyscript tests: it
creates an HTML page to run the specified snippet.
"""
@pytest.fixture()
def init(self, request, tmpdir, http_server, logger, page):
"""
Fixture to automatically initialize all the tests in this class and its
subclasses.
The magic is done by the decorator @pyest.mark.usefixtures("init"),
which tells pytest to automatically use this fixture for all the test
method of this class.
Using the standard pytest behavior, we can request more fixtures:
tmpdir, http_server and page; 'page' is a fixture provided by
pytest-playwright.
Then, we save these fixtures on the self and proceed with more
initialization. The end result is that the requested fixtures are
automatically made available as self.xxx in all methods.
"""
self.testname = request.function.__name__.replace("test_", "")
self.tmpdir = tmpdir
# create a symlink to BUILD inside tmpdir
tmpdir.join("build").mksymlinkto(BUILD)
self.tmpdir.chdir()
self.http_server = http_server
self.logger = logger
self.init_page(page)
#
# this extra print is useful when using pytest -s, else we start printing
# in the middle of the line
print()
def init_page(self, page):
self.page = page
self.console = ConsoleMessageCollection(self.logger)
self._page_errors = []
page.on("console", self.console.add_message)
page.on("pageerror", self._on_pageerror)
def teardown_method(self):
# we call check_errors on teardown: this means that if there are still
# non-cleared errors, the test will fail. If you expect errors in your
# page and they should not cause the test to fail, you should call
# self.check_errors() in the test itself.
self.check_errors()
def _on_pageerror(self, error):
self.logger.log("JS exception", error.stack, color="red")
self._page_errors.append(error)
def check_errors(self):
"""
Check whether JS errors were reported.
If it finds a single JS error, raise JsError.
If it finds multiple JS errors, raise JsMultipleErrors.
Upon return, all the errors are cleared, so a subsequent call to
check_errors will not raise, unless NEW JS errors have been reported
in the meantime.
"""
exc = None
if len(self._page_errors) == 1:
# if there is a single error, wrap it
exc = JsError(self._page_errors[0])
elif len(self._page_errors) >= 2:
exc = JsMultipleErrors(self._page_errors)
self._page_errors = []
if exc:
raise exc
def clear_errors(self):
"""
Clear all JS errors.
"""
self._page_errors = []
def writefile(self, filename, content):
"""
Very thin helper to write a file in the tmpdir
"""
f = self.tmpdir.join(filename)
f.write(content)
def goto(self, path):
self.logger.reset()
self.logger.log("page.goto", path, color="yellow")
url = f"{self.http_server}/{path}"
self.page.goto(url)
def wait_for_console(self, text, *, timeout=None, check_errors=True):
"""
Wait until the given message appear in the console.
Note: it must be the *exact* string as printed by e.g. console.log.
If you need more control on the predicate (e.g. if you want to match a
substring), use self.page.expect_console_message directly.
timeout is expressed in milliseconds. If it's None, it will use
playwright's own default value, which is 30 seconds).
If check_errors is True (the default), it also checks that no JS
errors were raised during the waiting.
"""
pred = lambda msg: msg.text == text
try:
with self.page.expect_console_message(pred, timeout=timeout):
pass
finally:
# raise JsError if there were any javascript exception. Note that
# this might happen also in case of a TimeoutError. In that case,
# the JsError will shadow the TimeoutError but this is correct,
# because it's very likely that the console message never appeared
# precisely because of the exception in JS.
if check_errors:
self.check_errors()
def wait_for_pyscript(self, *, timeout=None, check_errors=True):
"""
Wait until pyscript has been fully loaded.
Timeout is expressed in milliseconds. If it's None, it will use
playwright's own default value, which is 30 seconds).
If check_errors is True (the default), it also checks that no JS
errors were raised during the waiting.
"""
# this is printed by pyconfig.ts:PyodideRuntime.initialize
self.wait_for_console(
"===PyScript page fully initialized===",
timeout=timeout,
check_errors=check_errors,
)
def pyscript_run(self, snippet):
"""
Main entry point for pyscript tests.
snippet contains a fragment of HTML which will be put inside a full
HTML document. In particular, the <head> automatically contains the
correct <script> and <link> tags which are necessary to load pyscript
correctly.
This method does the following:
- write a full HTML file containing the snippet
- open a playwright page for it
- wait until pyscript has been fully loaded
"""
doc = f"""
<html>
<head>
<link rel="stylesheet" href="{self.http_server}/build/pyscript.css" />
<script defer src="{self.http_server}/build/pyscript.js"></script>
</head>
<body>
{snippet}
</body>
</html>
"""
filename = f"{self.testname}.html"
self.writefile(filename, doc)
self.goto(filename)
self.wait_for_pyscript()
# ============== Helpers and utility functions ==============
class JsError(Exception):
"""
Represent an exception which happened in JS.
It's a thin wrapper around playwright.sync_api.Error, with two important
differences:
1. it has a better name: if you see JsError in a traceback, it's
immediately obvious that it's a JS exception.
2. Show also the JS stacktrace by default, contrarily to
playwright.sync_api.Error
"""
def __init__(self, error):
super().__init__(self.format_playwright_error(error))
self.error = error
@staticmethod
def format_playwright_error(error):
# apparently, playwright Error.stack contains all the info that we
# want: exception name, message and stacktrace. The docs say that
# error.stack is optional, so fallback to the standard repr if it's
# unavailable.
return error.stack or str(error)
class JsMultipleErrors(Exception):
"""
This is raised in case we get multiple JS errors in the page
"""
def __init__(self, errors):
lines = ["Multiple JS errors found:"]
for err in errors:
lines.append(JsError.format_playwright_error(err))
msg = "\n".join(lines)
super().__init__(msg)
self.errors = errors
class ConsoleMessageCollection:
"""
Helper class to collect and expose ConsoleMessage in a Pythonic way.
Usage:
console.log.messages: list of ConsoleMessage with type=='log'
console.log.lines: list of strings
console.log.text: the whole text as single string
console.debug.* same as above, but with different types
console.info.*
console.error.*
console.warning.*
console.all.* same as above, but considering all messages, no filters
"""
class View:
"""
Filter console messages by the given msg_type
"""
def __init__(self, console, msg_type):
self.console = console
self.msg_type = msg_type
@property
def messages(self):
if self.msg_type is None:
return self.console._messages
else:
return [
msg for msg in self.console._messages if msg.type == self.msg_type
]
@property
def lines(self):
return [msg.text for msg in self.messages]
@property
def text(self):
return "\n".join(self.lines)
_COLORS = {
"error": "red",
"warning": "brown",
}
def __init__(self, logger):
self.logger = logger
self._messages = []
self.all = self.View(self, None)
self.log = self.View(self, "log")
self.debug = self.View(self, "debug")
self.info = self.View(self, "info")
self.error = self.View(self, "error")
self.warning = self.View(self, "warning")
def add_message(self, msg):
# log the message: pytest will capute the output and display the
# messages if the test fails.
category = f"console.{msg.type}"
color = self._COLORS.get(msg.type)
self.logger.log(category, msg.text, color=color)
self._messages.append(msg)
class Logger:
"""
Helper class to log messages to stdout.
Features:
- nice formatted category
- keep track of time passed since the last reset
- support colors
NOTE: the (lowercase) logger fixture is defined in conftest.py
"""
def __init__(self):
self.reset()
def reset(self):
self.start_time = time.time()
def log(self, category, text, *, color=None):
delta = time.time() - self.start_time
line = f"[{delta:6.2f} {category:15}] {text}"
if color:
line = Color.set(color, line)
print(line)
class Color:
"""
Helper method to print colored output using ANSI escape codes.
"""
black = "30"
darkred = "31"
darkgreen = "32"
brown = "33"
darkblue = "34"
purple = "35"
teal = "36"
lightgray = "37"
darkgray = "30;01"
red = "31;01"
green = "32;01"
yellow = "33;01"
blue = "34;01"
fuchsia = "35;01"
turquoise = "36;01"
white = "37;01"
@classmethod
def set(cls, color, string):
try:
color = getattr(cls, color)
except AttributeError:
pass
return f"\x1b[{color}m{string}\x1b[00m"

222
pyscriptjs/tests/test_00_support.py Normal file
View File

@@ -0,0 +1,222 @@
import textwrap
import pytest
from playwright import sync_api
from .support import JsError, JsMultipleErrors, PyScriptTest
class TestSupport(PyScriptTest):
"""
These are NOT tests about PyScript.
They test the PyScriptTest class, i.e. we want to ensure that all the
testing machinery that we have works correctly.
"""
def test_basic(self):
"""
Very basic test, just to check that we can write, serve and read a simple
HTML (no pyscript yet)
"""
doc = """
<html>
<body>
<h1>Hello world</h1>
</body>
</html>
"""
self.writefile("mytest.html", doc)
self.goto("mytest.html")
content = self.page.content()
assert "<h1>Hello world</h1>" in content
def test_console(self):
"""
Test that we capture console.log messages correctly.
"""
doc = """
<html>
<body>
<script>
console.log("my log 1");
console.debug("my debug");
console.info("my info");
console.error("my error");
console.warn("my warning");
console.log("my log 2");
</script>
</body>
</html>
"""
self.writefile("mytest.html", doc)
self.goto("mytest.html")
assert len(self.console.all.messages) == 6
assert self.console.all.lines == [
"my log 1",
"my debug",
"my info",
"my error",
"my warning",
"my log 2",
]
# fmt: off
assert self.console.all.text == textwrap.dedent("""
my log 1
my debug
my info
my error
my warning
my log 2
""").strip()
# fmt: on
assert self.console.log.lines == ["my log 1", "my log 2"]
assert self.console.debug.lines == ["my debug"]
def test_check_errors(self):
doc = """
<html>
<body>
<script>throw new Error('this is an error');</script>
</body>
</html>
"""
self.writefile("mytest.html", doc)
self.goto("mytest.html")
with pytest.raises(JsError) as exc:
self.check_errors()
# check that the exception message contains the error message and the
# stack trace
msg = str(exc.value)
assert "Error: this is an error" in msg
assert f"at {self.http_server}/mytest.html" in msg
#
# after a call to check_errors, the errors are cleared
self.check_errors()
def test_check_errors_multiple(self):
doc = """
<html>
<body>
<script>throw new Error('error 1');</script>
<script>throw new Error('error 2');</script>
</body>
</html>
"""
self.writefile("mytest.html", doc)
self.goto("mytest.html")
with pytest.raises(JsMultipleErrors) as exc:
self.check_errors()
assert "error 1" in str(exc.value)
assert "error 2" in str(exc.value)
#
# check that errors are cleared
self.check_errors()
def test_clear_errors(self):
doc = """
<html>
<body>
<script>throw new Error('this is an error');</script>
</body>
</html>
"""
self.writefile("mytest.html", doc)
self.goto("mytest.html")
self.clear_errors()
# self.check_errors does not raise, because the errors have been
# cleared
self.check_errors()
def test_wait_for_console(self):
"""
Test that self.wait_for_console actually waits.
If it's buggy, the test will try to read self.console.log BEFORE the
log has been written and it will fail.
"""
doc = """
<html>
<body>
<script>
setTimeout(function() {
console.log('Page loaded!');
}, 100);
</script>
</body>
</html>
"""
self.writefile("mytest.html", doc)
self.goto("mytest.html")
# we use a timeout of 500ms to give plenty of time to the page to
# actually run the setTimeout callback
self.wait_for_console("Page loaded!", timeout=200)
assert self.console.log.lines[-1] == "Page loaded!"
def test_wait_for_console_exception_1(self):
"""
Test that if a JS exception is raised while waiting for the console, we
report the exception and not the timeout.
There are two main cases:
1. there is an exception and the console message does not appear
2. there is an exception but the console message appears anyway
This test checks for case 1. Case 2 is tested by
test_wait_for_console_exception_2
"""
# case 1: there is an exception and the console message does not appear
doc = """
<html>
<body>
<script>throw new Error('this is an error');</script>
</body>
</html>
"""
self.writefile("mytest.html", doc)
# "Page loaded!" will never appear, of course.
self.goto("mytest.html")
with pytest.raises(JsError) as exc:
self.wait_for_console("Page loaded!", timeout=200)
assert "this is an error" in str(exc.value)
assert isinstance(exc.value.__context__, sync_api.TimeoutError)
#
# if we use check_errors=False, the error are ignored, but we get the
# Timeout anyway
self.goto("mytest.html")
with pytest.raises(sync_api.TimeoutError):
self.wait_for_console("Page loaded!", timeout=200, check_errors=False)
# we still got a JsError, so we need to manually clear it, else the
# test fails at teardown
self.clear_errors()
def test_wait_for_console_exception_2(self):
"""
See the description in test_wait_for_console_exception_1.
"""
# case 2: there is an exception, but the console message appears
doc = """
<html>
<body>
<script>
setTimeout(function() {
console.log('Page loaded!');
}, 100);
throw new Error('this is an error');
</script>
</body>
</html>
"""
self.writefile("mytest.html", doc)
self.goto("mytest.html")
with pytest.raises(JsError) as exc:
self.wait_for_console("Page loaded!", timeout=200)
assert "this is an error" in str(exc.value)
#
# with check_errors=False, the Error is ignored and the
# wait_for_console succeeds
self.goto("mytest.html")
self.wait_for_console("Page loaded!", timeout=200, check_errors=False)
# clear the errors, else the test fails at teardown
self.clear_errors()

38
pyscriptjs/tests/test_01_basic.py Normal file
View File

@@ -0,0 +1,38 @@
import re
from .support import PyScriptTest
class TestBasic(PyScriptTest):
def test_pyscript_hello(self):
self.pyscript_run(
"""
<py-script>
print('hello pyscript')
</py-script>
"""
)
# this is a very ugly way of checking the content of the DOM. If we
# find ourselves to write a lot of code in this style, we will
# probably want to write a nicer API for it.
inner_html = self.page.locator("py-script").inner_html()
pattern = r'<div id="py-.*">hello pyscript</div>'
assert re.search(pattern, inner_html)
def test_execution_in_order(self):
"""
Check that they py-script tags are executed in the same order they are
defined
"""
# NOTE: this test relies on the fact that pyscript does not write
# anything to console.info. If we start writing to info in the future,
# we will probably need to tweak this test.
self.pyscript_run(
"""
<py-script>import js; js.console.info('one')</py-script>
<py-script>js.console.info('two')</py-script>
<py-script>js.console.info('three')</py-script>
<py-script>js.console.info('four')</py-script>
"""
)
assert self.console.info.lines == ["one", "two", "three", "four"]

5
pyscriptjs/tests/test_examples.py
View File

@@ -17,6 +17,8 @@ from urllib.parse import urljoin
import pytest
from .support import ROOT
MAX_TEST_TIME = 30 # Number of seconds allowed for checking a testing condition
TEST_TIME_INCREMENT = 0.25 # 1/4 second, the length of each iteration
TEST_ITERATIONS = math.ceil(
@@ -188,6 +190,9 @@ def wait_for_render(page, selector, pattern):
@pytest.mark.parametrize("example", EXAMPLES)
def test_examples(example, http_server, page):
# make sure that the http server serves from the right directory
ROOT.join("pyscriptjs").chdir()
base_url = http_server
example_path = urljoin(base_url, TEST_PARAMS[example]["file"])

34
pyscriptjs/tests/test_py_button.py Normal file
View File

@@ -0,0 +1,34 @@
import pytest
from .support import PyScriptTest
class TestPyButton(PyScriptTest):
@pytest.mark.xfail
def test_on_click(self):
"""
currently this test fails for a bad reason which is unrelated to
py-button. During the page loading, the following JS exception occur,
in base.ts:BaseEvalElement.evaluate
[JS exception ] TypeError: Cannot use 'in' operator to search for 'runPythonAsync' in undefined
at http://127.0.0.1:8080/build/pyscript.js:305:38
at Object.subscribe (http://127.0.0.1:8080/build/pyscript.js:46:13)
at PyButton.runAfterRuntimeInitialized (http://127.0.0.1:8080/build/pyscript.js:304:27)
at PyButton.connectedCallback (http://127.0.0.1:8080/build/pyscript.js:26856:18)
at http://127.0.0.1:8080/build/pyscript.js:27075:20
at http://127.0.0.1:8080/build/pyscript.js:27093:3
""" # noqa: E501
self.pyscript_run(
"""
<py-button label="my button">
import js
def on_click(evt):
js.console.info('clicked!')
</py-button>
"""
)
assert self.console.info.lines == []
self.page.locator("text=my button").click()
self.page.locator("text=my button").click()
assert self.console.info.lines == ["clicked!", "clicked!"]

1
setup.cfg
View File

@@ -7,3 +7,4 @@ max-complexity = 10
max-line-length = 100
show-source = True
statistics = True
extend-ignore = E731