Celery doesn't auto reload in development (#3898)

* pick up *.py file changes and restart scheduler

* only watch /redash in order to avoid reloading on other file changes (e.g. tests)

* add dev_scheduler entrypoint

* use exec

* Update bin/docker-entrypoint

* rename dev_scheduler to dev_worker

* use same defaults as worker

bin/docker-entrypoint

@@ -20,6 +20,16 @@ scheduler() {
   exec /usr/local/bin/celery worker --app=redash.worker --beat -s$SCHEDULE_DB -c$WORKERS_COUNT -Q$QUEUES -linfo --max-tasks-per-child=10 -Ofair
 }
 
+dev_worker() {
+  WORKERS_COUNT=${WORKERS_COUNT:-2}
+  QUEUES=${QUEUES:-queries,scheduled_queries,celery,schemas}
+  SCHEDULE_DB=${SCHEDULE_DB:-celerybeat-schedule}
+
+  echo "Starting dev scheduler and $WORKERS_COUNT workers for queues: $QUEUES..."
+
+  exec watchmedo auto-restart --directory=./redash/ --pattern=*.py --recursive -- /usr/local/bin/celery worker --app=redash.worker --beat -s$SCHEDULE_DB -c$WORKERS_COUNT -Q$QUEUES -linfo --max-tasks-per-child=10 -Ofair
+}
+
 server() {
   exec /usr/local/bin/gunicorn -b 0.0.0.0:5000 --name redash -w${REDASH_WEB_WORKERS:-4} redash.wsgi:app
 }
@@ -41,6 +51,7 @@ help() {
   echo "server -- start Redash server (with gunicorn)"
   echo "worker -- start Celery worker"
   echo "scheduler -- start Celery worker with a beat (scheduler) process"
+  echo "dev_worker  -- start Celery worker with a beat (scheduler) process which picks up code changes and reloads"
   echo "celery_healthcheck -- runs a Celery healthcheck. Useful for Docker's HEALTHCHECK mechanism."
   echo ""
   echo "shell -- open shell"
@@ -75,6 +86,10 @@ case "$1" in
     shift
     scheduler
     ;;
+  dev_worker)
+    shift
+    dev_worker
+    ;;
   dev_server)
     export FLASK_DEBUG=1
     exec /app/manage.py runserver --debugger --reload -h 0.0.0.0

docker-compose.yml

@@ -1,6 +1,6 @@
 # This configuration file is for the **development** setup.
 # For a production example please refer to setup/docker-compose.yml.
-version: '3'
+version: '3.7'
 services:
   server:
     build: .
@@ -21,9 +21,11 @@ services:
       REDASH_RATELIMIT_ENABLED: "false"
   worker:
     build: .
-    command: scheduler
+    command: dev_worker
     volumes:
-      - ".:/app"
+      - type: bind
+        source: .
+        target: /app
     depends_on:
       - server
     environment:

requirements_dev.txt

@@ -10,3 +10,4 @@ botocore==1.12.115
 PyAthena>=1.5.0
 ptvsd==4.2.3
 freezegun==0.3.11
+watchdog==0.9.0