Update to Python 3.10 (#6991)

Updated from Python 3.8 to 3.10. Python 3.10 is the default for Ubuntu 22. This change necessitated upgrading to SQLAlchemy_Utils 0.38.3, and importing the sort_query function from an older version of SQLAlchemy_Utils because it was dropped in newer versions. Co-authored-by: Ezra Odio <eodio@starfishstorage.com>
2025-12-19 17:37:19 -05:00 · 2024-06-05 03:41:49 -04:00
parent 70dd05916f
commit bceaab0496
6 changed files with 414 additions and 15 deletions
--- a/2
+++ b/2
@@ -26,7 +26,7 @@ COPY --chown=redash client /frontend/client
 COPY --chown=redash webpack.config.js /frontend/
 RUN if [ "x$skip_frontend_build" = "x" ] ; then yarn build; else mkdir -p /frontend/client/dist && touch /frontend/client/dist/multi_org.html && touch /frontend/client/dist/index.html; fi

-FROM python:3.8-slim-bookworm
+FROM python:3.10-slim-bookworm

 EXPOSE 5000

--- a/poetry.lock
+++ b/poetry.lock
@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand.

 [[package]]
 name = "adal"
@@ -4611,31 +4611,29 @@ test = ["flake8 (>=2.4.0)", "isort (>=3.9.6)", "psycopg2 (>=2.4.6)", "pytest (>=

 [[package]]
 name = "sqlalchemy-utils"
-version = "0.34.2"
+version = "0.38.3"
 description = "Various utility functions for SQLAlchemy."
 optional = false
-python-versions = "*"
+python-versions = "~=3.6"
 files = [
-    {file = "SQLAlchemy-Utils-0.34.2.tar.gz", hash = "sha256:6689b29d7951c5c7c4d79fa6b8c95f9ff9ec708b07aa53f82060599bd14dcc88"},
+    {file = "SQLAlchemy-Utils-0.38.3.tar.gz", hash = "sha256:9f9afba607a40455cf703adfa9846584bf26168a0c5a60a70063b70d65051f4d"},
+    {file = "SQLAlchemy_Utils-0.38.3-py3-none-any.whl", hash = "sha256:5c13b5d08adfaa85f3d4e8ec09a75136216fad41346980d02974a70a77988bf9"},
 ]

 [package.dependencies]
-six = "*"
-SQLAlchemy = ">=1.0"
+SQLAlchemy = ">=1.3"

 [package.extras]
-anyjson = ["anyjson (>=0.3.3)"]
 arrow = ["arrow (>=0.3.4)"]
 babel = ["Babel (>=1.3)"]
 color = ["colour (>=0.0.4)"]
 encrypted = ["cryptography (>=0.6)"]
-enum = ["enum34"]
 intervals = ["intervals (>=0.7.1)"]
-ipaddress = ["ipaddr"]
 password = ["passlib (>=1.6,<2.0)"]
+pendulum = ["pendulum (>=2.0.5)"]
 phone = ["phonenumbers (>=5.9.2)"]
-test = ["Jinja2 (>=2.3)", "Pygments (>=1.2)", "docutils (>=0.10)", "flake8 (>=2.4.0)", "flexmock (>=0.9.7)", "isort (>=4.2.2)", "mock (==2.0.0)", "pg8000 (>=1.12.4)", "psycopg2 (>=2.5.1)", "pymysql", "pyodbc", "pytest (>=2.7.1)", "python-dateutil (>=2.6)", "pytz (>=2014.2)"]
-test-all = ["Babel (>=1.3)", "Jinja2 (>=2.3)", "Pygments (>=1.2)", "anyjson (>=0.3.3)", "arrow (>=0.3.4)", "colour (>=0.0.4)", "cryptography (>=0.6)", "docutils (>=0.10)", "enum34", "flake8 (>=2.4.0)", "flexmock (>=0.9.7)", "furl (>=0.4.1)", "intervals (>=0.7.1)", "ipaddr", "isort (>=4.2.2)", "mock (==2.0.0)", "passlib (>=1.6,<2.0)", "pg8000 (>=1.12.4)", "phonenumbers (>=5.9.2)", "psycopg2 (>=2.5.1)", "pymysql", "pyodbc", "pytest (>=2.7.1)", "python-dateutil", "python-dateutil (>=2.6)", "pytz (>=2014.2)"]
+test = ["Jinja2 (>=2.3)", "Pygments (>=1.2)", "backports.zoneinfo", "docutils (>=0.10)", "flake8 (>=2.4.0)", "flexmock (>=0.9.7)", "isort (>=4.2.2)", "pg8000 (>=1.12.4)", "psycopg2 (>=2.5.1)", "psycopg2cffi (>=2.8.1)", "pymysql", "pyodbc", "pytest (>=2.7.1)", "python-dateutil (>=2.6)", "pytz (>=2014.2)"]
+test-all = ["Babel (>=1.3)", "Jinja2 (>=2.3)", "Pygments (>=1.2)", "arrow (>=0.3.4)", "backports.zoneinfo", "colour (>=0.0.4)", "cryptography (>=0.6)", "docutils (>=0.10)", "flake8 (>=2.4.0)", "flexmock (>=0.9.7)", "furl (>=0.4.1)", "intervals (>=0.7.1)", "isort (>=4.2.2)", "passlib (>=1.6,<2.0)", "pendulum (>=2.0.5)", "pg8000 (>=1.12.4)", "phonenumbers (>=5.9.2)", "psycopg2 (>=2.5.1)", "psycopg2cffi (>=2.8.1)", "pymysql", "pyodbc", "pytest (>=2.7.1)", "python-dateutil", "python-dateutil (>=2.6)", "pytz (>=2014.2)"]
 timezone = ["python-dateutil"]
 url = ["furl (>=0.4.1)"]

@@ -5319,4 +5317,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"]
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.8,<3.11"
-content-hash = "3e7066c8bdd7bfe2365e49db094a2a6a705787bf56926df703457351482e5b6a"
+content-hash = "57adb580494581ed74afbf39281285a37212247e1a76090ab4a4f22920fdb94c"
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -72,7 +72,7 @@ semver = "2.8.1"
 sentry-sdk = "1.28.1"
 sqlalchemy = "1.3.24"
 sqlalchemy-searchable = "1.2.0"
-sqlalchemy-utils = "0.34.2"
+sqlalchemy-utils = "0.38.3"
 sqlparse = "0.5.0"
 sshtunnel = "0.1.5"
 statsd = "3.3.0"
--- a/redash/handlers/base.py
+++ b/redash/handlers/base.py
@@ -7,13 +7,13 @@ from flask_restful import Resource, abort
 from sqlalchemy import cast
 from sqlalchemy.dialects.postgresql import ARRAY
 from sqlalchemy.orm.exc import NoResultFound
-from sqlalchemy_utils.functions import sort_query

 from redash import settings
 from redash.authentication import current_org
 from redash.models import db
 from redash.tasks import record_event as record_event_task
 from redash.utils import json_dumps
+from redash.utils.query_order import sort_query

 routes = Blueprint("redash", __name__, template_folder=settings.fix_assets_path("templates"))

--- a/redash/utils/query_order.py
+++ b/redash/utils/query_order.py
@@ -0,0 +1,310 @@
+# Copyright (c) 2012, Konsta Vesterinen
+#
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+#
+# * The names of the contributors may not be used to endorse or promote products
+#   derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY DIRECT,
+# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from inspect import isclass
+
+import sqlalchemy as sa
+from sqlalchemy.orm import mapperlib
+from sqlalchemy.orm.properties import ColumnProperty
+from sqlalchemy.orm.query import _ColumnEntity
+from sqlalchemy.orm.util import AliasedInsp
+from sqlalchemy.sql.expression import asc, desc
+
+
+def get_query_descriptor(query, entity, attr):
+    if attr in query_labels(query):
+        return attr
+    else:
+        entity = get_query_entity_by_alias(query, entity)
+        if entity:
+            descriptor = get_descriptor(entity, attr)
+            if hasattr(descriptor, "property") and isinstance(descriptor.property, sa.orm.RelationshipProperty):
+                return
+            return descriptor
+
+
+def query_labels(query):
+    """
+    Return all labels for given SQLAlchemy query object.
+    Example::
+        query = session.query(
+            Category,
+            db.func.count(Article.id).label('articles')
+        )
+        query_labels(query)  # ['articles']
+    :param query: SQLAlchemy Query object
+    """
+    return [
+        entity._label_name for entity in query._entities if isinstance(entity, _ColumnEntity) and entity._label_name
+    ]
+
+
+def get_query_entity_by_alias(query, alias):
+    entities = get_query_entities(query)
+    if not alias:
+        return entities[0]
+    for entity in entities:
+        if isinstance(entity, sa.orm.util.AliasedClass):
+            name = sa.inspect(entity).name
+        else:
+            name = get_mapper(entity).tables[0].name
+        if name == alias:
+            return entity
+
+
+def get_query_entities(query):
+    """
+    Return a list of all entities present in given SQLAlchemy query object.
+    Examples::
+        from sqlalchemy_utils import get_query_entities
+        query = session.query(Category)
+        get_query_entities(query)  # [<Category>]
+        query = session.query(Category.id)
+        get_query_entities(query)  # [<Category>]
+    This function also supports queries with joins.
+    ::
+        query = session.query(Category).join(Article)
+        get_query_entities(query)  # [<Category>, <Article>]
+    .. versionchanged: 0.26.7
+        This function now returns a list instead of generator
+    :param query: SQLAlchemy Query object
+    """
+    exprs = [
+        d["expr"] if is_labeled_query(d["expr"]) or isinstance(d["expr"], sa.Column) else d["entity"]
+        for d in query.column_descriptions
+    ]
+    return [get_query_entity(expr) for expr in exprs] + [get_query_entity(entity) for entity in query._join_entities]
+
+
+def is_labeled_query(expr):
+    return isinstance(expr, sa.sql.elements.Label) and isinstance(
+        list(expr.base_columns)[0], (sa.sql.selectable.Select, sa.sql.selectable.ScalarSelect)
+    )
+
+
+def get_query_entity(expr):
+    if isinstance(expr, sa.orm.attributes.InstrumentedAttribute):
+        return expr.parent.class_
+    elif isinstance(expr, sa.Column):
+        return expr.table
+    elif isinstance(expr, AliasedInsp):
+        return expr.entity
+    return expr
+
+
+def get_mapper(mixed):
+    """
+    Return related SQLAlchemy Mapper for given SQLAlchemy object.
+    :param mixed: SQLAlchemy Table / Alias / Mapper / declarative model object
+    ::
+        from sqlalchemy_utils import get_mapper
+        get_mapper(User)
+        get_mapper(User())
+        get_mapper(User.__table__)
+        get_mapper(User.__mapper__)
+        get_mapper(sa.orm.aliased(User))
+        get_mapper(sa.orm.aliased(User.__table__))
+    Raises:
+        ValueError: if multiple mappers were found for given argument
+    .. versionadded: 0.26.1
+    """
+    if isinstance(mixed, sa.orm.query._MapperEntity):
+        mixed = mixed.expr
+    elif isinstance(mixed, sa.Column):
+        mixed = mixed.table
+    elif isinstance(mixed, sa.orm.query._ColumnEntity):
+        mixed = mixed.expr
+    if isinstance(mixed, sa.orm.Mapper):
+        return mixed
+    if isinstance(mixed, sa.orm.util.AliasedClass):
+        return sa.inspect(mixed).mapper
+    if isinstance(mixed, sa.sql.selectable.Alias):
+        mixed = mixed.element
+    if isinstance(mixed, AliasedInsp):
+        return mixed.mapper
+    if isinstance(mixed, sa.orm.attributes.InstrumentedAttribute):
+        mixed = mixed.class_
+    if isinstance(mixed, sa.Table):
+        mappers = [mapper for mapper in mapperlib._mapper_registry if mixed in mapper.tables]
+        if len(mappers) > 1:
+            raise ValueError("Multiple mappers found for table '%s'." % mixed.name)
+        elif not mappers:
+            raise ValueError("Could not get mapper for table '%s'." % mixed.name)
+        else:
+            return mappers[0]
+    if not isclass(mixed):
+        mixed = type(mixed)
+    return sa.inspect(mixed)
+
+
+def get_polymorphic_mappers(mixed):
+    if isinstance(mixed, AliasedInsp):
+        return mixed.with_polymorphic_mappers
+    else:
+        return mixed.polymorphic_map.values()
+
+
+def get_descriptor(entity, attr):
+    mapper = sa.inspect(entity)
+    for key, descriptor in get_all_descriptors(mapper).items():
+        if attr == key:
+            prop = descriptor.property if hasattr(descriptor, "property") else None
+            if isinstance(prop, ColumnProperty):
+                if isinstance(entity, sa.orm.util.AliasedClass):
+                    for c in mapper.selectable.c:
+                        if c.key == attr:
+                            return c
+                else:
+                    # If the property belongs to a class that uses
+                    # polymorphic inheritance we have to take into account
+                    # situations where the attribute exists in child class
+                    # but not in parent class.
+                    return getattr(prop.parent.class_, attr)
+            else:
+                # Handle synonyms, relationship properties and hybrid
+                # properties
+                if isinstance(entity, sa.orm.util.AliasedClass):
+                    return getattr(entity, attr)
+                try:
+                    return getattr(mapper.class_, attr)
+                except AttributeError:
+                    pass
+
+
+def get_all_descriptors(expr):
+    if isinstance(expr, sa.sql.selectable.Selectable):
+        return expr.c
+    insp = sa.inspect(expr)
+    try:
+        polymorphic_mappers = get_polymorphic_mappers(insp)
+    except sa.exc.NoInspectionAvailable:
+        return get_mapper(expr).all_orm_descriptors
+    else:
+        attrs = dict(get_mapper(expr).all_orm_descriptors)
+        for submapper in polymorphic_mappers:
+            for key, descriptor in submapper.all_orm_descriptors.items():
+                if key not in attrs:
+                    attrs[key] = descriptor
+        return attrs
+
+
+class QuerySorterException(Exception):
+    pass
+
+
+class QuerySorter:
+    def __init__(self, silent=True, separator="-"):
+        self.separator = separator
+        self.silent = silent
+
+    def assign_order_by(self, entity, attr, func):
+        expr = get_query_descriptor(self.query, entity, attr)
+        if expr is not None:
+            return self.query.order_by(func(expr))
+        if not self.silent:
+            raise QuerySorterException("Could not sort query with expression '%s'" % attr)
+        return self.query
+
+    def parse_sort_arg(self, arg):
+        if arg[0] == self.separator:
+            func = desc
+            arg = arg[1:]
+        else:
+            func = asc
+        parts = arg.split(self.separator)
+        return {
+            "entity": parts[0] if len(parts) > 1 else None,
+            "attr": parts[1] if len(parts) > 1 else arg,
+            "func": func,
+        }
+
+    def __call__(self, query, *args):
+        self.query = query
+        for sort in args:
+            if not sort:
+                continue
+            self.query = self.assign_order_by(**self.parse_sort_arg(sort))
+        return self.query
+
+
+def sort_query(query, *args, **kwargs):
+    """
+    Applies an sql ORDER BY for given query. This function can be easily used
+    with user-defined sorting.
+    The examples use the following model definition:
+    ::
+        import sqlalchemy as sa
+        from sqlalchemy import create_engine
+        from sqlalchemy.orm import sessionmaker
+        from sqlalchemy.ext.declarative import declarative_base
+        from sqlalchemy_utils import sort_query
+        engine = create_engine(
+            'sqlite:///'
+        )
+        Base = declarative_base()
+        Session = sessionmaker(bind=engine)
+        session = Session()
+        class Category(Base):
+            __tablename__ = 'category'
+            id = sa.Column(sa.Integer, primary_key=True)
+            name = sa.Column(sa.Unicode(255))
+        class Article(Base):
+            __tablename__ = 'article'
+            id = sa.Column(sa.Integer, primary_key=True)
+            name = sa.Column(sa.Unicode(255))
+            category_id = sa.Column(sa.Integer, sa.ForeignKey(Category.id))
+            category = sa.orm.relationship(
+                Category, primaryjoin=category_id == Category.id
+            )
+    1. Applying simple ascending sort
+    ::
+        query = session.query(Article)
+        query = sort_query(query, 'name')
+    2. Applying descending sort
+    ::
+        query = sort_query(query, '-name')
+    3. Applying sort to custom calculated label
+    ::
+        query = session.query(
+            Category, sa.func.count(Article.id).label('articles')
+        )
+        query = sort_query(query, 'articles')
+    4. Applying sort to joined table column
+    ::
+        query = session.query(Article).join(Article.category)
+        query = sort_query(query, 'category-name')
+    :param query:
+        query to be modified
+    :param sort:
+        string that defines the label or column to sort the query by
+    :param silent:
+        Whether or not to raise exceptions if unknown sort column
+        is passed. By default this is `True` indicating that no errors should
+        be raised for unknown columns.
+    """
+    return QuerySorter(**kwargs)(query, *args)
--- a/tests/handlers/test_order_results.py
+++ b/tests/handlers/test_order_results.py
@@ -0,0 +1,91 @@
+from redash import models
+from redash.handlers.base import order_results
+from redash.models import db
+from tests import BaseTestCase
+
+
+class TestOrderResults(BaseTestCase):
+    def setUp(self):
+        super().setUp()
+
+        user1 = self.factory.create_user(name="Charlie")
+        user2 = self.factory.create_user(name="Bravo")
+        user3 = self.factory.create_user(name="Alpha")
+
+        q1 = self.factory.create_query(name="a", user=user1)
+        q2 = self.factory.create_query(name="b", user=user2)
+        q3 = self.factory.create_query(name="c", user=user3)
+
+        db.session.add(user1)
+        db.session.add(user2)
+        db.session.add(user3)
+
+        db.session.add(q1)
+        db.session.add(q2)
+        db.session.add(q3)
+        db.session.commit()
+
+        self.results = db.session.query(models.Query)
+        self.results = self.results.join(models.User, models.Query.user_id == models.User.id)
+
+        self.allowed_orders = {
+            "name": "name",
+            "-name": "-name",
+            "users-name": "users-name",
+            "-users-name": "-users-name",
+        }
+        self.default_order = "-name"
+
+    def test_no_order_no_fallback(self):
+        with self.app.test_request_context("/items?order="):
+            ordered_results = order_results(self.results, self.default_order, self.allowed_orders, fallback=False)
+            self.assertEqual(self.results, ordered_results)
+
+    def test_no_order_yes_fallback(self):
+        with self.app.test_request_context("/items?order="):
+            ordered_results = order_results(self.results, self.default_order, self.allowed_orders, fallback=True)
+            ordered_results = [entry.name for entry in ordered_results]
+            self.assertEqual(ordered_results, ["c", "b", "a"])
+
+    def test_invalid_order_no_fallback(self):
+        with self.app.test_request_context("/items?order=some_invalid_order"):
+            ordered_results = order_results(self.results, self.default_order, self.allowed_orders, fallback=False)
+            ordered_results = [entry.name for entry in ordered_results]
+            self.assertEqual(ordered_results, [entry.name for entry in self.results])
+
+    def test_invalid_order_yes_fallback(self):
+        with self.app.test_request_context("/items?order=some_invalid_order"):
+            ordered_results = order_results(self.results, self.default_order, self.allowed_orders, fallback=True)
+            ordered_results = [entry.name for entry in ordered_results]
+            self.assertEqual(ordered_results, ["c", "b", "a"])
+
+    def test_valid_requested_order_no_fallback(self):
+        with self.app.test_request_context("/items?order=name"):
+            ordered_results = order_results(self.results, self.default_order, self.allowed_orders, fallback=False)
+            ordered_results = [entry.name for entry in ordered_results]
+            self.assertEqual(ordered_results, ["a", "b", "c"])
+
+    def test_valid_requested_order_yes_fallback(self):
+        with self.app.test_request_context("/items?order=name"):
+            ordered_results = order_results(self.results, self.default_order, self.allowed_orders, fallback=True)
+            ordered_results = [entry.name for entry in ordered_results]
+            self.assertEqual(ordered_results, ["a", "b", "c"])
+
+    def test_requested_entity_no_fallback(self):
+        with self.app.test_request_context("/items?order=users-name"):
+            ordered_results = order_results(self.results, self.default_order, self.allowed_orders, fallback=False)
+            ordered_results = [entry.name for entry in ordered_results]
+            self.assertEqual(ordered_results, ["c", "b", "a"])
+
+    def test_requested_entity_yes_fallback(self):
+        with self.app.test_request_context("/items?order=-users-name"):
+            ordered_results = order_results(self.results, self.default_order, self.allowed_orders, fallback=True)
+            ordered_results = [entry.name for entry in ordered_results]
+            self.assertEqual(ordered_results, ["a", "b", "c"])
+
+    def test_order_by_attached(self):
+        self.results = self.results.order_by(models.Query.name)
+        with self.app.test_request_context("/items?order=-name"):
+            ordered_results = order_results(self.results, self.default_order, self.allowed_orders, fallback=False)
+            ordered_results = [entry.name for entry in ordered_results]
+            self.assertEqual(ordered_results, ["c", "b", "a"])