Convert project to poetry (#6423)

* poetry init

* add all dependencies

* update pyproject.toml

* add poetry.lock

* remove requirements txt fixer

* add ldap3 group and make all groups optional

* remove requirements files

* convert dockerfile to use poetry instead of pip

* fix ldap 3 dependency group typo

* update ldap3 inline error

* update cypress to install only main poetry group

* convert test_all_deps

* update redis/rq

* don't create virtualenv

* add -dev suffix and make version snapshot work

* fix typo in snapshot commit message

* remove importlib-resources

.ci/docker-compose.cypress.yml

@@ -3,8 +3,7 @@ x-redash-service: &redash-service
   build:
     context: ../
     args:
-      skip_dev_deps: "true"
-      skip_ds_deps: "true"
+      install_groups: "main"
       code_coverage: ${CODE_COVERAGE}
 x-redash-environment: &redash-environment
   REDASH_LOG_LEVEL: "INFO"

.ci/docker_build

@@ -30,7 +30,7 @@ DOCKERHUB_REPO="redash/redash"
 DOCKER_TAGS="-t redash/redash:preview -t redash/preview:${VERSION_TAG}"
 
 # Build the docker container
-docker build --build-arg test_all_deps=true ${DOCKER_TAGS} .
+docker build --build-arg install_groups="main,all_ds,dev" ${DOCKER_TAGS} .
 
 # Push the container to the preview build locations
 docker push "${DOCKERHUB_REPO}:preview"

.github/workflows/ci.yml

@@ -36,7 +36,7 @@ jobs:
       - name: Build Docker Images
         run: |
           set -x
-          docker-compose build --build-arg test_all_deps=true --build-arg skip_frontend_build=true
+          docker-compose build --build-arg install_groups="main,all_ds,dev" --build-arg skip_frontend_build=true
           docker-compose up -d
           sleep 10
       - name: Create Test Database

.github/workflows/periodic-snapshot.yml

@@ -17,10 +17,11 @@ jobs:
           date="$(date +%y.%m).0-dev"
           gawk -i inplace -F: -v q=\" -v tag=$date '/^  "version": / { print $1 FS, q tag q ","; next} { print }' package.json
           gawk -i inplace -F= -v q=\" -v tag=$date '/^__version__ =/ { print $1 FS, q tag q; next} { print }' redash/__init__.py
+          gawk -i inplace -F= -v q=\" -v tag=$date '/^version =/ { print $1 FS, q tag q; next} { print }' pyproject.toml
           git config user.name github-actions
           git config user.email github-actions@github.com
-          git add package.json redash/__init__.py
-          git commit -m "Shapshot: ${date}"
+          git add package.json redash/__init__.py pyproject.toml
+          git commit -m "Snapshot: ${date}"
           git push origin
           git tag $date
           git push origin $date

.pre-commit-config.yaml

@@ -13,7 +13,3 @@ repos:
     hooks:
       - id: flake8
         exclude: "migration/.*|.git|viz-lib|node_modules|migrations|bin/upgrade"
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
-    hooks:
-      - id: requirements-txt-fixer

Dockerfile

@@ -29,13 +29,6 @@ FROM python:3.8-slim-buster
 
 EXPOSE 5000
 
-# Controls whether to install extra dependencies needed for all data sources.
-ARG skip_ds_deps
-# Controls whether to install dev dependencies.
-ARG skip_dev_deps
-# Controls whether to install all dependencies for testing.
-ARG test_all_deps
-
 RUN useradd --create-home redash
 
 # Ubuntu packages
@@ -85,24 +78,18 @@ RUN if [ "$TARGETPLATFORM" = "linux/amd64" ]; then \
 
 WORKDIR /app
 
-# Disable PIP Cache and Version Check
-ENV PIP_DISABLE_PIP_VERSION_CHECK=1
-ENV PIP_NO_CACHE_DIR=1
+ENV POETRY_VERSION=1.6.1
+ENV POETRY_HOME=/etc/poetry
+ENV POETRY_VIRTUALENVS_CREATE=false
+RUN curl -sSL https://install.python-poetry.org | python3 -
 
-RUN pip install pip==23.1.2;
+COPY pyproject.toml poetry.lock ./
 
-# We first copy only the requirements file, to avoid rebuilding on every file change.
-COPY requirements_all_ds.txt ./
-RUN if [ "x$skip_ds_deps" = "x" ] ; then cat requirements_all_ds.txt | sed -e '/^\s*#.*$/d' -e '/^\s*$/d' | xargs -n 1 pip install || true ; else echo "Skipping pip install -r requirements_all_ds.txt" ; fi
-
-
-COPY requirements_dev.txt ./
-RUN if [ "x$skip_dev_deps" = "x" ] ; then pip install -r requirements_dev.txt ; fi
-
-COPY requirements.txt ./
-RUN pip install -r requirements.txt
-
-RUN if [ "x$test_all_deps" != "x" ] ; then pip3 install -r requirements.txt -r requirements_dev.txt -r requirements_all_ds.txt ; fi
+ARG POETRY_OPTIONS="--no-root --no-interaction --no-ansi"
+# for LDAP authentication, install with `ldap3` group
+# disabled by default due to GPL license conflict
+ARG install_groups="main,all_ds,dev"
+RUN /etc/poetry/bin/poetry install --only $install_groups $POETRY_OPTIONS
 
 COPY --chown=redash . /app
 COPY --from=frontend-builder --chown=redash /frontend/client/dist /app/client/dist

pyproject.toml

@@ -16,3 +16,151 @@ multi_line_output = 3
 include_trailing_comma = true
 use_parentheses = true
 skip = "migrations"
+
+[tool.poetry]
+name = "redash"
+version = "23.09.0-dev"
+description = "Make Your Company Data Driven. Connect to any data source, easily visualize, dashboard and share your data."
+authors = ["Your Name <you@example.com>"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = ">=3.8,<3.9"
+advocate = "1.0.0"
+aniso8601 = "8.0.0"
+authlib = "0.15.5"
+blinker = "1.6.2"
+click = "8.1.3"
+cryptography = "41.0.3"
+disposable-email-domains = ">=0.0.52"
+flask = "2.3.2"
+flask-limiter = "3.3.1"
+flask-login = "0.6.0"
+flask-mail = "0.9.1"
+flask-migrate = "2.5.2"
+flask-restful = "0.3.10"
+flask-sqlalchemy = "2.5.1"
+flask-talisman = "0.7.0"
+flask-wtf = "1.1.1"
+funcy = "1.13"
+gevent = "22.10.1"
+greenlet = "1.1.3"
+gunicorn = "20.0.4"
+httplib2 = "0.19.0"
+itsdangerous = "2.1.2"
+jinja2 = "3.1.2"
+jsonschema = "3.1.1"
+markupsafe = "2.1.1"
+maxminddb-geolite2 = "2018.703"
+parsedatetime = "2.4"
+passlib = "1.7.3"
+psycopg2-binary = "2.9.6"
+pyjwt = "2.4.0"
+pyopenssl = "23.2.0"
+pypd = "1.1.0"
+pysaml2 = "7.3.1"
+pystache = "0.6.0"
+python-dateutil = "2.8.0"
+python-dotenv = "0.19.2"
+pytz = ">=2019.3"
+pyyaml = "6.0.1"
+redis = "4.6.0"
+regex = "2023.8.8"
+requests = "2.31.0"
+restrictedpython = "6.2"
+rq = "1.9.0"
+rq-scheduler = "0.11.0"
+semver = "2.8.1"
+sentry-sdk = "1.28.1"
+simplejson = "3.16.0"
+sqlalchemy = "1.3.24"
+sqlalchemy-searchable = "1.2.0"
+sqlalchemy-utils = "0.34.2"
+sqlparse = "0.4.4"
+sshtunnel = "0.1.5"
+statsd = "3.3.0"
+supervisor = "4.1.0"
+supervisor-checks = "0.8.1"
+ua-parser = "0.18.0"
+urllib3 = "1.26.16"
+user-agents = "2.0"
+werkzeug = "2.3.6"
+wtforms = "2.2.1"
+xlsxwriter = "1.2.2"
+
+[tool.poetry.group.all_ds]
+optional = true
+
+[tool.poetry.group.all_ds.dependencies]
+atsd-client = "3.0.5"
+azure-kusto-data = "0.0.35"
+boto3 = "1.28.8"
+botocore = "1.31.8"
+cassandra-driver = "3.21.0"
+certifi = ">=2019.9.11"
+cmem-cmempy = "21.2.3"
+databend-py = "0.4.6"
+databend-sqlalchemy = "0.2.4"
+google-api-python-client = "1.7.11"
+gspread = "3.1.0"
+impyla = "0.16.0"
+influxdb = "5.2.3"
+memsql = "3.2.0"
+mysqlclient = "2.1.1"
+nzalchemy = "^11.0.2"
+nzpy = ">=1.15"
+oauth2client = "4.1.3"
+openpyxl = "3.0.7"
+oracledb = "1.4.0"
+pandas = "1.3.4"
+phoenixdb = "0.7"
+pinotdb = ">=0.4.5"
+protobuf = "3.18.3"
+pyathena = ">=1.5.0,<=1.11.5"
+pydgraph = "2.0.2"
+pydruid = "0.5.7"
+pyexasol = "0.12.0"
+pygridgain = "1.4.0"
+pyhive = "0.6.1"
+pyignite = "0.6.1"
+pymongo = {version = "4.3.3", extras = ["srv", "tls"]}
+pymssql = "2.2.8"
+pyodbc = "4.0.28"
+python-arango = "6.1.0"
+python-rapidjson = "1.1.0"
+qds-sdk = ">=1.9.6"
+requests-aws-sign = "0.1.5"
+sasl = ">=0.1.3"
+simple-salesforce = "0.74.3"
+snowflake-connector-python = "3.1.0"
+td-client = "1.0.0"
+thrift = ">=0.8.0"
+thrift-sasl = ">=0.1.0"
+trino = ">=0.305,<1.0"
+vertica-python = "1.1.1"
+xlrd = "2.0.1"
+
+[tool.poetry.group.ldap3]
+optional = true
+
+[tool.poetry.group.ldap3.dependencies]
+ldap3 = "2.2.4"
+
+[tool.poetry.group.dev]
+optional = true
+
+[tool.poetry.group.dev.dependencies]
+pytest = "7.4.0"
+coverage = "7.2.7"
+freezegun = "1.2.1"
+jwcrypto = "1.5.0"
+mock = "5.0.2"
+pre-commit = "3.3.3"
+ptpython = "3.0.23"
+ptvsd = "4.3.2"
+pytest-cov = "4.1.0"
+watchdog = "3.0.0"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

redash/authentication/ldap_auth.py

@@ -11,7 +11,7 @@ try:
 except ImportError:
     if settings.LDAP_LOGIN_ENABLED:
         sys.exit(
-            "The ldap3 library was not found. This is required to use LDAP authentication (see requirements.txt)."
+            "The ldap3 library was not found. This is required to use LDAP authentication. Rebuild the Docker image installing the `ldap3` poetry dependency group."
         )
 
 from redash.authentication import (

requirements.txt

@@ -1,64 +0,0 @@
-advocate==1.0.0
-aniso8601==8.0.0
-# Uncomment the requirement for ldap3 if using ldap.
-# It is not included by default because of the GPL license conflict.
-# ldap3==2.2.4
-Authlib==0.15.5
-blinker==1.6.2
-click==8.1.3
-cryptography==41.0.3
-disposable-email-domains>=0.0.52
-Flask==2.3.2
-Flask-Limiter==3.3.1
-Flask-Login==0.6.0
-flask-mail==0.9.1
-Flask-Migrate==2.5.2
-Flask-RESTful==0.3.10
-Flask-SQLAlchemy==2.5.1
-flask-talisman==0.7.0
-Flask-WTF==1.1.1
-funcy==1.13
-gevent==22.10.1
-greenlet==1.1.3
-gunicorn==20.0.4
-httplib2==0.19.0
-itsdangerous==2.1.2
-Jinja2==3.1.2
-jsonschema==3.1.1
-MarkupSafe==2.1.1
-maxminddb-geolite2==2018.703
-parsedatetime==2.4
-passlib==1.7.3
-psycopg2-binary==2.9.6
-PyJWT==2.4.0
-pyOpenSSL==23.2.0
-pypd==1.1.0
-pysaml2==7.3.1
-pystache==0.6.0
-python-dateutil==2.8.0
-python-dotenv==0.19.2
-pytz>=2019.3
-PyYAML==6.0.1
-redis==4.6.0
-regex==2023.8.8
-requests==2.31.0
-RestrictedPython==6.2
-rq==1.9.0
-rq-scheduler==0.11.0
-semver==2.8.1
-sentry-sdk==1.28.1
-simplejson==3.16.0
-SQLAlchemy==1.3.24
-SQLAlchemy-Searchable==1.2.0
-SQLAlchemy-Utils==0.34.2
-sqlparse==0.4.4
-sshtunnel==0.1.5
-statsd==3.3.0
-supervisor==4.1.0
-supervisor_checks==0.8.1
-ua-parser==0.18.0
-urllib3==1.26.16
-user-agents==2.0
-werkzeug==2.3.6
-wtforms==2.2.1
-xlsxwriter==1.2.2

requirements_all_ds.txt

@@ -1,52 +0,0 @@
-atsd_client==3.0.5
-azure-kusto-data==0.0.35
-boto3==1.28.8
-botocore==1.31.8
-cassandra-driver==3.21.0
-# certifi is needed to support MongoDB and SSL:
-certifi>=2019.9.11
-cmem-cmempy==21.2.3
-databend-py==0.4.6
-databend-sqlalchemy==0.2.4
-google-api-python-client==1.7.11
-google-auth>=2.22.0
-gspread==5.1.0
-impyla==0.16.0
-influxdb==5.2.3
-memsql==3.2.0
-mysqlclient==2.1.1
-nzalchemy
-nzpy>=1.15
-openpyxl==3.0.7
-# python-oracledb supports Oracle 12c and above without needing the Oracle
-# client libraries installed.  To support Oracle 9.x and above as well,
-# then the Oracle client libraries will need installing first.
-oracledb==1.4.0
-pandas==1.3.4
-phoenixdb==0.7
-pinotdb>=0.4.5
-protobuf==3.18.3
-PyAthena>=1.5.0,<=1.11.5
-pydgraph==2.0.2
-# ibm-db>=2.0.9
-pydruid==0.5.7
-pyexasol==0.12.0
-pygridgain==1.4.0
-pyhive==0.6.1
-pyignite==0.6.1
-pymongo[tls,srv]==4.3.3
-pymssql==2.2.8
-pyodbc==4.0.28
-python-arango==6.1.0
-python-rapidjson==1.1.0
-qds-sdk>=1.9.6
-requests_aws_sign==0.1.5
-sasl>=0.1.3
-simple_salesforce==0.74.3
-snowflake-connector-python==3.1.0
-td-client==1.0.0
-thrift>=0.8.0
-thrift_sasl>=0.1.0
-trino~=0.305
-vertica-python==1.1.1
-xlrd==2.0.1

requirements_dev.txt

@@ -1,10 +0,0 @@
-coverage==7.2.7
-freezegun==1.2.1
-jwcrypto==1.5.0
-mock==5.0.2
-pre-commit==3.3.3
-ptpython==3.0.23
-ptvsd==4.3.2
-pytest==7.4.0
-pytest-cov==4.1.0
-watchdog==3.0.0