Merge pull request #253 from EverythingMe/146-filter-sync

rd_ui: sync filters with location.search [closes #146]

.gitignore

@@ -4,10 +4,17 @@
 .coverage
 rd_ui/dist
 .DS_Store
+celerybeat-schedule*
+.#*
+\#*#
+*~
 
 # Vagrant related
 .vagrant
 Berksfile.lock
 redash/dump.rdb
 .env
-.ruby-version
+.ruby-version
+venv
+
+dump.rdb

Makefile

@@ -1,19 +1,22 @@
 NAME=redash
 VERSION=`python ./manage.py version`
 FULL_VERSION=$(VERSION)+b$(CIRCLE_BUILD_NUM)
-FILENAME=$(CIRCLE_ARTIFACTS)/$(NAME).$(FULL_VERSION).tar.gz
+# VERSION gets evaluated every time it's referenced, therefore we need to use VERSION here instead of FULL_VERSION.
+FILENAME=$(CIRCLE_ARTIFACTS)/$(NAME).$(VERSION).tar.gz
 
 deps:
 	cd rd_ui && npm install
-	cd rd_ui && npm install grunt-cli bower
+	cd rd_ui && npm install -g bower grunt-cli
 	cd rd_ui && bower install
 	cd rd_ui && grunt build
 
 pack:
+	sed -ri "s/^__version__ = '([0-9.]*)'/__version__ = '$(FULL_VERSION)'/" redash/__init__.py
 	tar -zcv -f $(FILENAME) --exclude=".git*" --exclude="*.pyc" --exclude="*.pyo" --exclude="venv" --exclude="rd_ui/node_modules" --exclude="rd_ui/dist/bower_components" --exclude="rd_ui/app" *
 
 upload:
-	python bin/upload_version.py $(FULL_VERSION) $(FILENAME)
+	python bin/upload_version.py $(VERSION) $(FILENAME)
 
 test:
 	nosetests --with-coverage --cover-package=redash tests/*.py
+	cd rd_ui && grunt test

Procfile.dev

@@ -1,2 +1,2 @@
 web: ./manage.py runserver -p $PORT
-worker: ./manage.py runworkers
+worker: ./bin/run celery worker --app=redash.worker --beat -Qqueries,celery,scheduled_queries

Procfile.heroku

@@ -1,2 +1,2 @@
 web: ./manage.py runserver -p $PORT --host 0.0.0.0 -d -r
-worker: ./manage.py runworkers
+worker: ./bin/run celery worker --app=redash.worker --beat -Qqueries,celery,scheduled_queries

bin/latest_release.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+import sys
+import requests
+
+if __name__ == '__main__':
+    response = requests.get('https://api.github.com/repos/EverythingMe/redash/releases')
+
+    if response.status_code != 200:
+        exit("Failed getting releases (status code: %s)." % response.status_code)
+
+    sorted_releases = sorted(response.json(), key=lambda release: release['id'], reverse=True)
+
+    latest_release = sorted_releases[0]
+    asset_url = latest_release['assets'][0]['url']
+    filename = latest_release['assets'][0]['name']
+
+    wget_command = 'wget --header="Accept: application/octet-stream" %s -O %s' % (asset_url, filename)
+
+    if '--url-only' in sys.argv:
+        print asset_url
+    elif '--wget' in sys.argv:
+        print wget_command
+    else:
+        print "Latest release: %s" % latest_release['tag_name']
+        print latest_release['body']
+
+        print "\nTarball URL: %s" % asset_url
+        print 'wget: %s' % (wget_command)
+
+

bin/run

@@ -1,3 +1,10 @@
 #!/usr/bin/env bash
-source .env
-"$@"
+
+# Ideally I would use stdin with source, but in older bash versions this
+# wasn't supported properly.
+TEMP_ENV_FILE=`mktemp /tmp/redash_env.XXXXXX`
+sed 's/^REDASH/export REDASH/' .env > $TEMP_ENV_FILE
+source $TEMP_ENV_FILE
+rm $TEMP_ENV_FILE
+
+exec "$@"

bin/upload_version.py

@@ -3,30 +3,44 @@ import os
 import sys
 import json
 import requests
+import subprocess
+
+
+def capture_output(command):
+    proc = subprocess.Popen(command, stdout=subprocess.PIPE)
+    return proc.stdout.read()
+
 
 if __name__ == '__main__':
-  version = sys.argv[1]
-  filepath = sys.argv[2]
-  filename = filepath.split('/')[-1]
-  github_token = os.environ['GITHUB_TOKEN']
-  auth = (github_token, 'x-oauth-basic')
-  commit_sha = os.environ['CIRCLE_SHA1']
+    version = sys.argv[1]
+    filepath = sys.argv[2]
+    filename = filepath.split('/')[-1]
+    github_token = os.environ['GITHUB_TOKEN']
+    auth = (github_token, 'x-oauth-basic')
+    commit_sha = os.environ['CIRCLE_SHA1']
 
-  params = json.dumps({
-    'tag_name': 'v{0}'.format(version),
-    'name': 're:dash v{0}'.format(version),
-    'target_commitish': commit_sha,
-    'prerelease': True
-  })
+    commit_body = capture_output(["git", "log", "--format=%b", "-n", "1", commit_sha])
+    file_md5_checksum = capture_output(["md5sum", filepath]).split()[0]
+    file_sha256_checksum = capture_output(["sha256sum", filepath]).split()[0]
+    version_body = "%s\n\nMD5: %s\nSHA256: %s" % (commit_body, file_md5_checksum, file_sha256_checksum)
 
-  response = requests.post('https://api.github.com/repos/everythingme/redash/releases',
-                            data=params,
-                            auth=auth)
+    params = json.dumps({
+        'tag_name': 'v{0}'.format(version),
+        'name': 're:dash v{0}'.format(version),
+        'body': version_body,
+        'target_commitish': commit_sha,
+        'prerelease': True
+    })
 
-  upload_url = response.json()['upload_url']
-  upload_url = upload_url.replace('{?name}', '')
+    response = requests.post('https://api.github.com/repos/everythingme/redash/releases',
+                             data=params,
+                             auth=auth)
 
-  with open(filepath) as file_content:
-    headers = {'Content-Type': 'application/gzip'}
-    response = requests.post(upload_url, file_content, params={'name': filename}, auth=auth, headers=headers, verify=False)
+    upload_url = response.json()['upload_url']
+    upload_url = upload_url.replace('{?name}', '')
+
+    with open(filepath) as file_content:
+        headers = {'Content-Type': 'application/gzip'}
+        response = requests.post(upload_url, file_content, params={'name': filename}, auth=auth,
+                                 headers=headers, verify=False)
 

circle.yml

@@ -1,7 +1,7 @@
 machine:
   node:
     version:
-      0.10.22
+      0.10.24
   python:
     version:
       2.7.3
@@ -17,7 +17,7 @@ test:
   override:
     - make test
   post:
-    - make pack 
+    - make pack
 deployment:
   github:
     branch: master

manage.py

@@ -2,20 +2,17 @@
 """
 CLI to manage redash.
 """
-import atfork
-atfork.monkeypatch_os_fork_functions()
-import atfork.stdlib_fixer
-atfork.stdlib_fixer.fix_logging_module()
-
-import logging
-import time
-from redash import settings, app, db, models, data_manager, __version__
-from redash.import_export import import_manager
+import datetime
 from flask.ext.script import Manager, prompt_pass
 
+from redash import settings, models, __version__
+from redash.wsgi import app
+from redash.import_export import import_manager
+
 manager = Manager(app)
 database_manager = Manager(help="Manages the database (create/drop tables).")
 users_manager = Manager(help="Users management commands.")
+data_sources_manager = Manager(help="Data sources management commands.")
 
 @manager.command
 def version():
@@ -25,31 +22,13 @@ def version():
 
 @manager.command
 def runworkers():
-    """Starts the re:dash query executors/workers."""
+    """Prints deprecation warning."""
+    print "** This command is deprecated. Please use Celery's CLI to control the workers. **"
 
-    try:
-        old_workers = data_manager.redis_connection.smembers('workers')
-        data_manager.redis_connection.delete('workers')
-
-        logging.info("Cleaning old workers: %s", old_workers)
-
-        data_manager.start_workers(settings.WORKERS_COUNT)
-        logging.info("Workers started.")
-
-        while True:
-            try:
-                data_manager.refresh_queries()
-                data_manager.report_status()
-            except Exception as e:
-                logging.error("Something went wrong with refreshing queries...")
-                logging.exception(e)
-            time.sleep(60)
-    except KeyboardInterrupt:
-        logging.warning("Exiting; waiting for threads")
-        data_manager.stop_workers()
 
 @manager.shell
 def make_shell_context():
+    from redash.models import db
     return dict(app=app, db=db, models=models)
 
 @manager.command
@@ -61,12 +40,56 @@ def check_settings():
         if not callable(item) and not name.startswith("__") and not isinstance(item, ModuleType):
             print "{} = {}".format(name, item)
 
+@manager.command
+def import_events(events_file):
+    import json
+    from collections import Counter
+
+    count = Counter()
+
+    with open(events_file) as f:
+        for line in f:
+            try:
+                event = json.loads(line)
+
+                user = event.pop('user_id')
+                action = event.pop('action')
+                object_type = event.pop('object_type')
+                object_id = event.pop('object_id', None)
+
+                if object_id == 'dashboard' and object_type == 'dashboard':
+                    count['bad dashboard id'] += 1
+                    continue
+
+                created_at = datetime.datetime.utcfromtimestamp(event.pop('timestamp'))
+                additional_properties = json.dumps(event)
+
+                models.Event.create(user=user, action=action, object_type=object_type, object_id=object_id,
+                                    additional_properties=additional_properties, created_at=created_at)
+
+                count['imported'] += 1
+
+            except Exception as ex:
+                print "Failed importing line:"
+                print line
+                print ex.message
+                count[ex.message] += 1
+                count['failed'] += 1
+
+                models.db.close_db(None)
+
+    for k, v in count.iteritems():
+        print k
+        print v
+
+
 @database_manager.command
 def create_tables():
     """Creates the database tables."""
-    from redash.models import create_db
+    from redash.models import create_db, init_db
 
     create_db(True, False)
+    init_db()
 
 @database_manager.command
 def drop_tables():
@@ -81,19 +104,19 @@ def drop_tables():
 @users_manager.option('--admin', dest='is_admin', action="store_true", default=False, help="set user as admin")
 @users_manager.option('--google', dest='google_auth', action="store_true", default=False, help="user uses Google Auth to login")
 @users_manager.option('--password', dest='password', default=None, help="Password for users who don't use Google Auth (leave blank for prompt).")
-@users_manager.option('--permissions', dest='permissions', default=models.User.DEFAULT_PERMISSIONS, help="Comma seperated list of permissions (leave blank for default).")
-def create(email, name, permissions, is_admin=False, google_auth=False, password=None):
+@users_manager.option('--groups', dest='groups', default=models.User.DEFAULT_GROUPS, help="Comma seperated list of groups (leave blank for default).")
+def create(email, name, groups, is_admin=False, google_auth=False, password=None):
     print "Creating user (%s, %s)..." % (email, name)
     print "Admin: %r" % is_admin
     print "Login with Google Auth: %r\n" % google_auth
-    if isinstance(permissions, basestring):
-        permissions = permissions.split(',')
-        permissions.remove('') # in case it was empty string
+    if isinstance(groups, basestring):
+        groups= groups.split(',')
+        groups.remove('') # in case it was empty string
 
     if is_admin:
-        permissions += ['admin']
+        groups += ['admin']
 
-    user = models.User(email=email, name=name, permissions=permissions)
+    user = models.User(email=email, name=name, groups=groups)
     if not google_auth:
         password = password or prompt_pass("Password")
         user.hash_password(password)
@@ -109,9 +132,38 @@ def delete(email):
     deleted_count = models.User.delete().where(models.User.email == email).execute()
     print "Deleted %d users." % deleted_count
 
+@data_sources_manager.command
+def import_from_settings(name=None):
+    """Import data source from settings (env variables)."""
+    name = name or "Default"
+    data_source = models.DataSource.create(name=name,
+                                           type=settings.CONNECTION_ADAPTER,
+                                           options=settings.CONNECTION_STRING)
+
+    print "Imported data source from settings (id={}).".format(data_source.id)
+
+
+@data_sources_manager.command
+def list():
+    """List currently configured data sources"""
+    for ds in models.DataSource.select():
+        print "Name: {}\nType: {}\nOptions: {}".format(ds.name, ds.type, ds.options)
+
+@data_sources_manager.command
+def new(name, type, options):
+    """Create new data source"""
+    # TODO: validate it's a valid type and in the future, validate the options.
+    print "Creating {} data source ({}) with options:\n{}".format(type, name, options)
+    data_source = models.DataSource.create(name=name,
+                                           type=type,
+                                           options=options)
+    print "Id: {}".format(data_source.id)
+
+
 manager.add_command("database", database_manager)
 manager.add_command("users", users_manager)
 manager.add_command("import", import_manager)
+manager.add_command("ds", data_sources_manager)
 
 if __name__ == '__main__':
     manager.run()

migrations/add_global_filters_to_dashboard.py

@@ -0,0 +1,12 @@
+from playhouse.migrate import Migrator
+from redash import models
+from redash.models import db
+
+
+if __name__ == '__main__':
+    db.connect_db()
+    migrator = Migrator(db.database)
+    with db.database.transaction():
+        migrator.add_column(models.Dashboard, models.Dashboard.dashboard_filters_enabled, 'dashboard_filters_enabled')
+
+    db.close_db(None)

migrations/add_queue_name_to_data_source.py

@@ -0,0 +1,13 @@
+from playhouse.migrate import Migrator
+from redash.models import db
+from redash import models
+
+
+if __name__ == '__main__':
+    db.connect_db()
+    migrator = Migrator(db.database)
+    with db.database.transaction():
+        migrator.add_column(models.DataSource, models.DataSource.queue_name, 'queue_name')
+        migrator.add_column(models.DataSource, models.DataSource.scheduled_queue_name, 'scheduled_queue_name')
+
+    db.close_db(None)

migrations/add_text_to_widgets.py

@@ -0,0 +1,13 @@
+from playhouse.migrate import Migrator
+from redash.models import db
+from redash import models
+
+
+if __name__ == '__main__':
+    db.connect_db()
+    migrator = Migrator(db.database)
+    with db.database.transaction():
+        migrator.add_column(models.Widget, models.Widget.text, 'text')
+        migrator.set_nullable(models.Widget, models.Widget.visualization, True)
+
+    db.close_db(None)

migrations/create_events.py

@@ -0,0 +1,12 @@
+from redash.models import db
+from redash import models
+
+
+if __name__ == '__main__':
+    db.connect_db()
+
+    if not models.Event.table_exists():
+        print "Creating events table..."
+        models.Event.create_table()
+
+    db.close_db(None)

migrations/permissions_migration.py

@@ -0,0 +1,29 @@
+import peewee
+from playhouse.migrate import Migrator
+from redash import models
+from redash.models import db
+
+
+if __name__ == '__main__':
+    db.connect_db()
+    migrator = Migrator(db.database)
+    
+    if not models.Group.table_exists():
+        print "Creating groups table..."
+        models.Group.create_table()
+    
+    with db.database.transaction():
+        models.Group.insert(name='admin', permissions=['admin'], tables=['*']).execute()
+        models.Group.insert(name='api', permissions=['view_query'], tables=['*']).execute()
+        models.Group.insert(name='default', permissions=models.Group.DEFAULT_PERMISSIONS, tables=['*']).execute()
+
+        migrator.add_column(models.User, models.User.groups, 'groups')
+        
+        models.User.update(groups=['admin', 'default']).where(peewee.SQL("is_admin = true")).execute()
+        models.User.update(groups=['admin', 'default']).where(peewee.SQL("'admin' = any(permissions)")).execute()
+        models.User.update(groups=['default']).where(peewee.SQL("is_admin = false")).execute()
+
+        migrator.drop_column(models.User, 'permissions')
+        migrator.drop_column(models.User, 'is_admin')
+
+    db.close_db(None)

rd_ui/.travis.yml

@@ -1,6 +1,5 @@
 language: node_js
 node_js:
-  - '0.8'
   - '0.10'
 before_script:
   - 'npm install -g bower grunt-cli'

rd_ui/Gruntfile.js

@@ -1,10 +1,5 @@
-// Generated on 2013-08-25 using generator-angular 0.4.0
+// Generated on 2014-07-30 using generator-angular 0.9.2
 'use strict';
-var LIVERELOAD_PORT = 35729;
-var lrSnippet = require('connect-livereload')({ port: LIVERELOAD_PORT });
-var mountFolder = function (connect, dir) {
-  return connect.static(require('path').resolve(dir));
-};
 
 // # Globbing
 // for performance reasons we're only matching one level down:
@@ -13,48 +8,148 @@ var mountFolder = function (connect, dir) {
 // 'test/spec/**/*.js'
 
 module.exports = function (grunt) {
+
+  // Load grunt tasks automatically
   require('load-grunt-tasks')(grunt);
+
+  // Time how long tasks take. Can help when optimizing build times
   require('time-grunt')(grunt);
 
-  // configurable paths
-  var yeomanConfig = {
-    app: 'app',
+  // Configurable paths for the application
+  var appConfig = {
+    app: require('./bower.json').appPath || 'app',
     dist: 'dist'
   };
 
-  try {
-    yeomanConfig.app = require('./bower.json').appPath || yeomanConfig.app;
-  } catch (e) {}
-
+  // Define the configuration for all the tasks
   grunt.initConfig({
-    yeoman: yeomanConfig,
+
+    // Project settings
+    yeoman: appConfig,
+
+    // Watches files for changes and runs tasks based on the changed files
     watch: {
-      coffee: {
-        files: ['<%= yeoman.app %>/scripts/{,*/}*.coffee'],
-        tasks: ['coffee:dist']
+      bower: {
+        files: ['bower.json'],
+        tasks: ['wiredep']
       },
-      coffeeTest: {
-        files: ['test/spec/{,*/}*.coffee'],
-        tasks: ['coffee:test']
+      js: {
+        files: ['<%= yeoman.app %>/scripts/{,*/}*.js'],
+        tasks: ['newer:jshint:all'],
+        options: {
+          livereload: '<%= connect.options.livereload %>'
+        }
+      },
+      jsTest: {
+        files: ['test/spec/{,*/}*.js'],
+        tasks: ['newer:jshint:test', 'karma']
       },
       styles: {
         files: ['<%= yeoman.app %>/styles/{,*/}*.css'],
-        tasks: ['copy:styles', 'autoprefixer']
+        tasks: ['newer:copy:styles', 'autoprefixer']
+      },
+      gruntfile: {
+        files: ['Gruntfile.js']
       },
       livereload: {
         options: {
-          livereload: LIVERELOAD_PORT
+          livereload: '<%= connect.options.livereload %>'
         },
         files: [
           '<%= yeoman.app %>/{,*/}*.html',
           '.tmp/styles/{,*/}*.css',
-          '{.tmp,<%= yeoman.app %>}/scripts/{,*/}*.js',
           '<%= yeoman.app %>/images/{,*/}*.{png,jpg,jpeg,gif,webp,svg}'
         ]
       }
     },
+
+    // The actual grunt server settings
+    connect: {
+      options: {
+        port: 9000,
+        // Change this to '0.0.0.0' to access the server from outside.
+        hostname: 'localhost',
+        livereload: 35729
+      },
+      livereload: {
+        options: {
+          open: true,
+          middleware: function (connect) {
+            return [
+              connect.static('.tmp'),
+              connect().use(
+                '/bower_components',
+                connect.static('./bower_components')
+              ),
+              connect.static(appConfig.app)
+            ];
+          }
+        }
+      },
+      test: {
+        options: {
+          port: 9001,
+          middleware: function (connect) {
+            return [
+              connect.static('.tmp'),
+              connect.static('test'),
+              connect().use(
+                '/bower_components',
+                connect.static('./bower_components')
+              ),
+              connect.static(appConfig.app)
+            ];
+          }
+        }
+      },
+      dist: {
+        options: {
+          open: true,
+          base: '<%= yeoman.dist %>'
+        }
+      }
+    },
+
+    // Make sure code styles are up to par and there are no obvious mistakes
+    jshint: {
+      options: {
+        jshintrc: '.jshintrc',
+        reporter: require('jshint-stylish')
+      },
+      all: {
+        src: [
+          'Gruntfile.js',
+          '<%= yeoman.app %>/scripts/{,*/}*.js'
+        ]
+      },
+      test: {
+        options: {
+          jshintrc: 'test/.jshintrc'
+        },
+        src: ['test/spec/{,*/}*.js']
+      }
+    },
+
+    // Empties folders to start fresh
+    clean: {
+      dist: {
+        files: [{
+          dot: true,
+          src: [
+            '.tmp',
+            '<%= yeoman.dist %>/{,*/}*',
+            '!<%= yeoman.dist %>/.git*'
+          ]
+        }]
+      },
+      server: '.tmp'
+    },
+
+    // Add vendor prefixed styles
     autoprefixer: {
-      options: ['last 1 version'],
+      options: {
+        browsers: ['last 1 version']
+      },
       dist: {
         files: [{
           expand: true,
@@ -64,134 +159,95 @@ module.exports = function (grunt) {
         }]
       }
     },
-    connect: {
+
+    // Automatically inject Bower components into the app
+    wiredep: {
       options: {
-        port: 9000,
-        // Change this to '0.0.0.0' to access the server from outside.
-        hostname: 'localhost'
+        cwd: '<%= yeoman.app %>'
       },
-      livereload: {
-        options: {
-          middleware: function (connect) {
-            return [
-              lrSnippet,
-              mountFolder(connect, '.tmp'),
-              mountFolder(connect, yeomanConfig.app)
-            ];
-          }
-        }
-      },
-      test: {
-        options: {
-          middleware: function (connect) {
-            return [
-              mountFolder(connect, '.tmp'),
-              mountFolder(connect, 'test')
-            ];
-          }
-        }
-      },
-      dist: {
-        options: {
-          middleware: function (connect) {
-            return [
-              mountFolder(connect, yeomanConfig.dist)
-            ];
-          }
-        }
+      app: {
+        src: ['<%= yeoman.app %>/index.html'],
+        ignorePath:  /\.\.\//
       }
     },
-    open: {
-      server: {
-        url: 'http://localhost:<%= connect.options.port %>'
-      }
-    },
-    clean: {
-      dist: {
-        files: [{
-          dot: true,
-          src: [
-            '.tmp',
-            '<%= yeoman.dist %>/*',
-            '!<%= yeoman.dist %>/.git*'
-          ]
-        }]
-      },
-      server: '.tmp'
-    },
-    jshint: {
-      options: {
-        jshintrc: '.jshintrc'
-      },
-      all: [
-        'Gruntfile.js',
-        '<%= yeoman.app %>/scripts/{,*/}*.js'
-      ]
-    },
-    coffee: {
-      options: {
-        sourceMap: true,
-        sourceRoot: ''
-      },
-      dist: {
-        files: [{
-          expand: true,
-          cwd: '<%= yeoman.app %>/scripts',
-          src: '{,*/}*.coffee',
-          dest: '.tmp/scripts',
-          ext: '.js'
-        }]
-      },
-      test: {
-        files: [{
-          expand: true,
-          cwd: 'test/spec',
-          src: '{,*/}*.coffee',
-          dest: '.tmp/spec',
-          ext: '.js'
-        }]
-      }
-    },
-    // not used since Uglify task does concat,
-    // but still available if needed
-    /*concat: {
-      dist: {}
-    },*/
-    rev: {
-      dist: {
-        files: {
-          src: [
-            '<%= yeoman.dist %>/scripts/{,*/}*.js',
-            '<%= yeoman.dist %>/styles/{,*/}*.css',
-            '<%= yeoman.dist %>/images/{,*/}*.{png,jpg,jpeg,gif,webp,svg}',
-            '<%= yeoman.dist %>/styles/fonts/*'
-          ]
-        }
+
+    // Renames files for browser caching purposes
+    filerev: {
+      dist: {
+        src: [
+          '<%= yeoman.dist %>/scripts/{,*/}*.js',
+          '<%= yeoman.dist %>/styles/{,*/}*.css',
+          '<%= yeoman.dist %>/images/{,*/}*.{png,jpg,jpeg,gif,webp,svg}',
+          '<%= yeoman.dist %>/styles/fonts/*'
+        ]
       }
     },
+
+    // Reads HTML for usemin blocks to enable smart builds that automatically
+    // concat, minify and revision files. Creates configurations in memory so
+    // additional tasks can operate on them
     useminPrepare: {
-      html: ['<%= yeoman.app %>/index.html', '<%= yeoman.app %>/login.html'],
+      html: '<%= yeoman.app %>/index.html',
       options: {
-        dest: '<%= yeoman.dist %>'
+        dest: '<%= yeoman.dist %>',
+        flow: {
+          html: {
+            steps: {
+              js: ['concat', 'uglifyjs'],
+              css: ['cssmin']
+            },
+            post: {}
+          }
+        }
       }
     },
+
+    // Performs rewrites based on filerev and the useminPrepare configuration
     usemin: {
       html: ['<%= yeoman.dist %>/{,*/}*.html'],
       css: ['<%= yeoman.dist %>/styles/{,*/}*.css'],
       options: {
-        dirs: ['<%= yeoman.dist %>']
+        assetsDirs: ['<%= yeoman.dist %>','<%= yeoman.dist %>/images']
       }
     },
+
+    // The following *-min tasks will produce minified files in the dist folder
+    // By default, your `index.html`'s <!-- Usemin block --> will take care of
+    // minification. These next options are pre-configured if you do not wish
+    // to use the Usemin blocks.
+    // cssmin: {
+    //   dist: {
+    //     files: {
+    //       '<%= yeoman.dist %>/styles/main.css': [
+    //         '.tmp/styles/{,*/}*.css'
+    //       ]
+    //     }
+    //   }
+    // },
+    // uglify: {
+    //   dist: {
+    //     files: {
+    //       '<%= yeoman.dist %>/scripts/scripts.js': [
+    //         '<%= yeoman.dist %>/scripts/scripts.js'
+    //       ]
+    //     }
+    //   }
+    // },
+    // concat: {
+    //   dist: {}
+    // },
+
     imagemin: {
       dist: {
         files: [{
           expand: true,
           cwd: '<%= yeoman.app %>/images',
-          src: '{,*/}*.{png,jpg,jpeg}',
+          src: '{,*/}*.{png,jpg,jpeg,gif}',
           dest: '<%= yeoman.dist %>/images'
         }]
       }
     },
+
     svgmin: {
       dist: {
         files: [{
@@ -202,41 +258,47 @@ module.exports = function (grunt) {
         }]
       }
     },
-    cssmin: {
-      // By default, your `index.html` <!-- Usemin Block --> will take care of
-      // minification. This option is pre-configured if you do not wish to use
-      // Usemin blocks.
-      // dist: {
-      //   files: {
-      //     '<%= yeoman.dist %>/styles/main.css': [
-      //       '.tmp/styles/{,*/}*.css',
-      //       '<%= yeoman.app %>/styles/{,*/}*.css'
-      //     ]
-      //   }
-      // }
-    },
+
     htmlmin: {
       dist: {
         options: {
-          /*removeCommentsFromCDATA: true,
-          // https://github.com/yeoman/grunt-usemin/issues/44
-          //collapseWhitespace: true,
+          collapseWhitespace: true,
+          conservativeCollapse: true,
           collapseBooleanAttributes: true,
-          removeAttributeQuotes: true,
-          removeRedundantAttributes: true,
-          useShortDoctype: true,
-          removeEmptyAttributes: true,
-          removeOptionalTags: true*/
+          removeCommentsFromCDATA: true,
+          removeOptionalTags: true
         },
         files: [{
           expand: true,
-          cwd: '<%= yeoman.app %>',
-          src: ['*.html', 'views/**/*.html'],
+          cwd: '<%= yeoman.dist %>',
+          src: ['*.html', 'views/{,*/}*.html'],
           dest: '<%= yeoman.dist %>'
         }]
       }
     },
-    // Put files not handled in other tasks here
+
+    // ngmin tries to make the code safe for minification automatically by
+    // using the Angular long form for dependency injection. It doesn't work on
+    // things like resolve or inject so those have to be done manually.
+    ngmin: {
+      dist: {
+        files: [{
+          expand: true,
+          cwd: '.tmp/concat/scripts',
+          src: '*.js',
+          dest: '.tmp/concat/scripts'
+        }]
+      }
+    },
+
+    // Replace Google CDN references
+    cdnify: {
+      dist: {
+        html: ['<%= yeoman.dist %>/*.html']
+      }
+    },
+
+    // Copies remaining files to places other tasks can use
     copy: {
       dist: {
         files: [{
@@ -247,17 +309,21 @@ module.exports = function (grunt) {
           src: [
             '*.{ico,png,txt}',
             '.htaccess',
-            'bower_components/**/*',
-            'images/{,*/}*.{gif,webp}',
+            '*.html',
+            'views/{,*/}*.html',
+            'images/{,*/}*.{webp}',
             'fonts/*'
           ]
         }, {
           expand: true,
           cwd: '.tmp/images',
           dest: '<%= yeoman.dist %>/images',
-          src: [
-            'generated/*'
-          ]
+          src: ['generated/*']
+        }, {
+          expand: true,
+          cwd: 'bower_components/bootstrap/dist',
+          src: 'fonts/*',
+          dest: '<%= yeoman.dist %>'
         }]
       },
       styles: {
@@ -267,70 +333,52 @@ module.exports = function (grunt) {
         src: '{,*/}*.css'
       }
     },
+
+    // Run some tasks in parallel to speed up the build process
     concurrent: {
       server: [
-        'coffee:dist',
         'copy:styles'
       ],
       test: [
-        'coffee',
         'copy:styles'
       ],
       dist: [
-        'coffee',
         'copy:styles',
         'imagemin',
-        'svgmin',
-        'htmlmin'
+        'svgmin'
       ]
     },
+
+    // Test settings
     karma: {
       unit: {
-        configFile: 'karma.conf.js',
+        configFile: 'test/karma.conf.js',
         singleRun: true
       }
-    },
-    cdnify: {
-      dist: {
-        html: ['<%= yeoman.dist %>/*.html']
-      }
-    },
-    ngmin: {
-      dist: {
-        files: [{
-          expand: true,
-          cwd: '<%= yeoman.dist %>/scripts',
-          src: '*.js',
-          dest: '<%= yeoman.dist %>/scripts'
-        }]
-      }
-    },
-    uglify: {
-      dist: {
-        files: {
-          '<%= yeoman.dist %>/scripts/scripts.js': [
-            '<%= yeoman.dist %>/scripts/scripts.js'
-          ]
-        }
-      }
     }
   });
 
-  grunt.registerTask('server', function (target) {
+
+  grunt.registerTask('serve', 'Compile then start a connect web server', function (target) {
     if (target === 'dist') {
-      return grunt.task.run(['build', 'open', 'connect:dist:keepalive']);
+      return grunt.task.run(['build', 'connect:dist:keepalive']);
     }
 
     grunt.task.run([
       'clean:server',
+      'wiredep',
       'concurrent:server',
       'autoprefixer',
       'connect:livereload',
-      'open',
       'watch'
     ]);
   });
 
+  grunt.registerTask('server', 'DEPRECATED TASK. Use the "serve" task instead', function (target) {
+    grunt.log.warn('The `server` task has been deprecated. Use `grunt serve` to start a server.');
+    grunt.task.run(['serve:' + target]);
+  });
+
   grunt.registerTask('test', [
     'clean:server',
     'concurrent:test',
@@ -341,21 +389,23 @@ module.exports = function (grunt) {
 
   grunt.registerTask('build', [
     'clean:dist',
+    'wiredep',
     'useminPrepare',
     'concurrent:dist',
     'autoprefixer',
     'concat',
+    'ngmin',
     'copy:dist',
     'cdnify',
-    'ngmin',
     'cssmin',
     'uglify',
-    'rev',
-    'usemin'
+    'filerev',
+    'usemin',
+    'htmlmin'
   ]);
 
   grunt.registerTask('default', [
-    'jshint',
+    'newer:jshint',
     'test',
     'build'
   ]);

rd_ui/app/index.html

@@ -12,8 +12,10 @@
     <link rel="stylesheet" href="/bower_components/bootstrap/dist/css/bootstrap.css">
     <link rel="stylesheet" href="/bower_components/codemirror/lib/codemirror.css">
     <link rel="stylesheet" href="/bower_components/gridster/dist/jquery.gridster.css">
-    <link rel="stylesheet" href="/bower_components/pivottable/examples/pivot.css">
+    <link rel="stylesheet" href="/bower_components/pivottable/dist/pivot.css">
     <link rel="stylesheet" href="/bower_components/cornelius/src/cornelius.css">
+    <link rel="stylesheet" href="/bower_components/select2/select2.css">
+    <link rel="stylesheet" href="/bower_components/pace/themes/pace-theme-minimal.css">
     <link rel="stylesheet" href="/styles/redash.css">
     <!-- endbuild -->
 </head>
@@ -35,7 +37,7 @@
     <div class="collapse navbar-collapse navbar-ex1-collapse">
         <ul class="nav navbar-nav">
             <li class="active" ng-show="pageTitle"><a class="page-title" ng-bind="pageTitle"></a></li>
-            <li class="dropdown">
+            <li class="dropdown" ng-show="groupedDashboards.length > 0 || otherDashboards.length > 0 || currentUser.hasPermission('create_dashboard')">
                 <a href="#" class="dropdown-toggle" data-toggle="dropdown"><span class="glyphicon glyphicon-th-large"></span> <b class="caret"></b></a>
                 <ul class="dropdown-menu">
                     <span ng-repeat="(name, group) in groupedDashboards">
@@ -51,7 +53,7 @@
                     <li ng-repeat="dashboard in otherDashboards">
                         <a role="menu-item" ng-href="/dashboard/{{dashboard.slug}}" ng-bind="dashboard.name"></a>
                     </li>
-                    <li class="divider" ng-show="currentUser.hasPermission('create_dashboard')"></li>
+                    <li class="divider" ng-show="currentUser.hasPermission('create_dashboard') && (groupedDashboards.length > 0 || otherDashboards.length > 0)"></li>
                     <li><a data-toggle="modal" href="#new_dashboard_dialog" ng-show="currentUser.hasPermission('create_dashboard')">New Dashboard</a></li>
                 </ul>
             </li>
@@ -102,15 +104,19 @@
 <script src="/bower_components/highcharts/modules/exporting.js"></script>
 <script src="/bower_components/gridster/dist/jquery.gridster.js"></script>
 <script src="/bower_components/angular-growl/build/angular-growl.js"></script>
-<script src="/bower_components/pivottable/examples/pivot.js"></script>
+<script src="/bower_components/pivottable/dist/pivot.js"></script>
 <script src="/bower_components/cornelius/src/cornelius.js"></script>
 <script src="/bower_components/mousetrap/mousetrap.js"></script>
 <script src="/bower_components/mousetrap/plugins/global-bind/mousetrap-global-bind.js"></script>
-
+<script src="/bower_components/select2/select2.js"></script>
+<script src="/bower_components/angular-ui-select2/src/select2.js"></script>
+<script src="/bower_components/underscore.string/lib/underscore.string.js"></script>
+<script src="/bower_components/marked/lib/marked.js"></script>
 <script src="/scripts/ng_highchart.js"></script>
 <script src="/scripts/ng_smart_table.js"></script>
 <script src="/scripts/ui-bootstrap-tpls-0.5.0.min.js"></script>
-
+<script src="/bower_components/bucky/bucky.js"></script>
+<script src="/bower_components/pace/pace.js"></script>
 <!-- endbuild -->
 
 <!-- build:js({.tmp,app}) /scripts/scripts.js -->
@@ -136,6 +142,8 @@
 <!-- endbuild -->
 
 <script>
+    // TODO: move currentUser & features to be an Angular service
+    var featureFlags = {{ features|safe }};
     var currentUser = {{ user|safe }};
 
     currentUser.canEdit = function(object) {
@@ -151,4 +159,4 @@
 </script>
 
 </body>
-</html>
+</html>

rd_ui/app/scripts/app.js

@@ -8,71 +8,85 @@ angular.module('redash', [
     'redash.visualization',
     'ui.codemirror',
     'highchart',
+    'ui.select2',
     'angular-growl',
     'angularMoment',
     'ui.bootstrap',
     'smartTable.table',
     'ngResource',
     'ngRoute'
-]).config(['$routeProvider', '$locationProvider', '$compileProvider', 'growlProvider',
-    function($routeProvider, $locationProvider, $compileProvider, growlProvider) {
+  ]).config(['$routeProvider', '$locationProvider', '$compileProvider', 'growlProvider',
+    function ($routeProvider, $locationProvider, $compileProvider, growlProvider) {
+      if (featureFlags.clientSideMetrics) {
+        Bucky.setOptions({
+          host: '/api/metrics'
+        });
 
-        function getQuery(Query, $route) {
-            var query = Query.get({'id': $route.current.params.queryId });
-            return query.$promise;
-        };
+        Bucky.requests.monitor('ajax_requsts');
+        Bucky.requests.transforms.enable('dashboards', /dashboard\/[\w-]+/ig, '/dashboard');
+      }
 
-        $compileProvider.aHrefSanitizationWhitelist(/^\s*(https?|http|data):/);
-        $locationProvider.html5Mode(true);
-        growlProvider.globalTimeToLive(2000);
+      function getQuery(Query, $route) {
+        var query = Query.get({'id': $route.current.params.queryId });
+        return query.$promise;
+      };
 
-        $routeProvider.when('/dashboard/:dashboardSlug', {
-            templateUrl: '/views/dashboard.html',
-            controller: 'DashboardCtrl'
-        });
-        $routeProvider.when('/queries', {
-            templateUrl: '/views/queries.html',
-            controller: 'QueriesCtrl',
-            reloadOnSearch: false
-        });
-        $routeProvider.when('/queries/new', {
-            templateUrl: '/views/query.html',
-            controller: 'QuerySourceCtrl',
-            reloadOnSearch: false,
-            resolve: {
-                'query': ['Query', function newQuery(Query) {
-                    return Query.newQuery();
-                }]
-            }
-        });
-        $routeProvider.when('/queries/:queryId', {
-            templateUrl: '/views/query.html',
-            controller: 'QueryViewCtrl',
-            reloadOnSearch: false,
-            resolve: {
-                'query': ['Query', '$route', getQuery]
-            }
-        });
-        $routeProvider.when('/queries/:queryId/source', {
-            templateUrl: '/views/query.html',
-            controller: 'QuerySourceCtrl',
-            reloadOnSearch: false,
-            resolve: {
-                'query': ['Query', '$route', getQuery]
-            }
-        });
-        $routeProvider.when('/admin/status', {
-            templateUrl: '/views/admin_status.html',
-            controller: 'AdminStatusCtrl'
-        });
-        $routeProvider.when('/', {
-            templateUrl: '/views/index.html',
-            controller: 'IndexCtrl'
-        });
-        $routeProvider.otherwise({
-            redirectTo: '/'
-        });
+      $compileProvider.aHrefSanitizationWhitelist(/^\s*(https?|http|data):/);
+      $locationProvider.html5Mode(true);
+      growlProvider.globalTimeToLive(2000);
+
+      $routeProvider.when('/dashboard/:dashboardSlug', {
+        templateUrl: '/views/dashboard.html',
+        controller: 'DashboardCtrl'
+      });
+      $routeProvider.when('/queries', {
+        templateUrl: '/views/queries.html',
+        controller: 'QueriesCtrl',
+        reloadOnSearch: false
+      });
+      $routeProvider.when('/queries/new', {
+        templateUrl: '/views/query.html',
+        controller: 'QuerySourceCtrl',
+        reloadOnSearch: false,
+        resolve: {
+          'query': ['Query', function newQuery(Query) {
+            return Query.newQuery();
+          }]
+        }
+      });
+      $routeProvider.when('/queries/:queryId', {
+        templateUrl: '/views/query.html',
+        controller: 'QueryViewCtrl',
+        reloadOnSearch: false,
+        resolve: {
+          'query': ['Query', '$route', getQuery]
+        }
+      });
+      $routeProvider.when('/queries/:queryId/source', {
+        templateUrl: '/views/query.html',
+        controller: 'QuerySourceCtrl',
+        reloadOnSearch: false,
+        resolve: {
+          'query': ['Query', '$route', getQuery]
+        }
+      });
+      $routeProvider.when('/admin/status', {
+        templateUrl: '/views/admin_status.html',
+        controller: 'AdminStatusCtrl'
+      });
+      $routeProvider.when('/admin/workers', {
+        templateUrl: '/views/admin_workers.html',
+        controller: 'AdminWorkersCtrl'
+      });
+
+      $routeProvider.when('/', {
+        templateUrl: '/views/index.html',
+        controller: 'IndexCtrl'
+      });
+      $routeProvider.otherwise({
+        redirectTo: '/'
+      });
 
 
     }
-]);
+  ]);

rd_ui/app/scripts/controllers/admin_controllers.js

@@ -1,23 +1,31 @@
 (function () {
-    var AdminStatusCtrl = function ($scope, $http, $timeout) {
-        $scope.$parent.pageTitle = "System Status";
+  var AdminStatusCtrl = function ($scope, Events, $http, $timeout) {
+    Events.record(currentUser, "view", "page", "admin/status");
+    $scope.$parent.pageTitle = "System Status";
 
-        var refresh = function () {
-            $scope.refresh_time = moment().add('minutes', 1);
-            $http.get('/status.json').success(function (data) {
-                $scope.workers = data.workers;
-                delete data.workers;
-                $scope.manager = data.manager;
-                delete data.manager;
-                $scope.status = data;
-            });
+    var refresh = function () {
+      $scope.refresh_time = moment().add('minutes', 1);
+      $http.get('/status.json').success(function (data) {
+        $scope.workers = data.workers;
+        delete data.workers;
+        $scope.manager = data.manager;
+        delete data.manager;
+        $scope.status = data;
+      });
 
-            $timeout(refresh, 59 * 1000);
-        };
+      $timeout(refresh, 59 * 1000);
+    };
 
-        refresh();
-    }
+    $scope.flowerUrl = featureFlags.flowerUrl;
 
-    angular.module('redash.admin_controllers', [])
-        .controller('AdminStatusCtrl', ['$scope', '$http', '$timeout', AdminStatusCtrl])
+    refresh();
+  }
+
+  var AdminWorkersCtrl = function ($scope, $sce) {
+    $scope.flowerUrl = $sce.trustAsResourceUrl(featureFlags.flowerUrl);
+  };
+
+  angular.module('redash.admin_controllers', [])
+         .controller('AdminStatusCtrl', ['$scope', 'Events', '$http', '$timeout', AdminStatusCtrl])
+         .controller('AdminWorkersCtrl', ['$scope', '$sce', AdminWorkersCtrl])
 })();

rd_ui/app/scripts/controllers/controllers.js

@@ -1,155 +1,169 @@
 (function () {
-    var QueriesCtrl = function($scope, $http, $location, $filter, Query) {
-        $scope.$parent.pageTitle = "All Queries";
-        $scope.gridConfig = {
-            isPaginationEnabled: true,
-            itemsByPage: 50,
-            maxSize: 8,
-            isGlobalSearchActivated: true
-        }
-
-        $scope.allQueries = [];
-        $scope.queries = [];
-
-        var dateFormatter = function (value) {
-            if (!value) return "-";
-            return value.format("DD/MM/YY HH:mm");
-        }
-
-        var filterQueries = function() {
-            $scope.queries = _.filter($scope.allQueries, function(query) {
-                if (!$scope.selectedTab) {
-                    return false;
-                }
-
-                if ($scope.selectedTab.key == 'my') {
-                    return query.user.id == currentUser.id && query.name != 'New Query';
-                } else if ($scope.selectedTab.key == 'drafts') {
-                    return query.user.id == currentUser.id && query.name == 'New Query';
-                }
-
-                return query.name != 'New Query';
-            });
-        }
-
-        Query.query(function(queries) {
-            $scope.allQueries = _.map(queries, function(query) {
-                query.created_at = moment(query.created_at);
-                query.last_retrieved_at = moment(query.last_retrieved_at);
-                return query;
-            });
-
-            filterQueries();
-        });
-
-        $scope.gridColumns = [
-            {
-                "label": "Name",
-                "map": "name",
-                "cellTemplateUrl": "/views/queries_query_name_cell.html"
-            },
-            {
-                'label': 'Created By',
-                'map': 'user.name'
-            },
-            {
-                'label': 'Created At',
-                'map': 'created_at',
-                'formatFunction': dateFormatter
-            },
-            {
-                'label': 'Runtime (avg)',
-                'map': 'avg_runtime',
-                'formatFunction': function(value) {
-                    return $filter('durationHumanize')(value);
-                }
-            },
-            {
-                'label': 'Runtime (min)',
-                'map': 'min_runtime',
-                'formatFunction': function(value) {
-                    return $filter('durationHumanize')(value);
-                }
-            },
-            {
-                'label': 'Runtime (max)',
-                'map': 'max_runtime',
-                'formatFunction': function(value) {
-                    return $filter('durationHumanize')(value);
-                }
-            },
-            {
-                'label': 'Last Executed At',
-                'map': 'last_retrieved_at',
-                'formatFunction': dateFormatter
-            },
-            {
-                'label': 'Times Executed',
-                'map': 'times_retrieved'
-            },
-            {
-                'label': 'Update Schedule',
-                'map': 'ttl',
-                'formatFunction': function(value) {
-                    return $filter('refreshRateHumanize')(value);
-                }
-            }
-        ]
-        $scope.tabs = [{"name": "My Queries", "key": "my"}, {"key": "all", "name": "All Queries"}, {"key": "drafts", "name": "Drafts"}];
-
-        $scope.$watch('selectedTab', function(tab) {
-            if (tab) {
-                $scope.$parent.pageTitle =  tab.name;
-            }
-
-            filterQueries();
-        });
+  var QueriesCtrl = function ($scope, $http, $location, $filter, Query) {
+    $scope.$parent.pageTitle = "All Queries";
+    $scope.gridConfig = {
+      isPaginationEnabled: true,
+      itemsByPage: 50,
+      maxSize: 8,
+      isGlobalSearchActivated: true
     }
 
-    var MainCtrl = function ($scope, Dashboard, notifications) {
-        $scope.dashboards = [];
-        $scope.reloadDashboards = function() {
-            Dashboard.query(function (dashboards) {
-                $scope.dashboards = _.sortBy(dashboards, "name");
-                $scope.allDashboards = _.groupBy($scope.dashboards, function(d) {
-                    parts = d.name.split(":");
-                    if (parts.length == 1) {
-                        return "Other";
-                    }
-                    return parts[0];
-                });
-                $scope.otherDashboards = $scope.allDashboards['Other'] || [];
-                $scope.groupedDashboards = _.omit($scope.allDashboards, 'Other');
-            });
+    $scope.allQueries = [];
+    $scope.queries = [];
+
+    var dateFormatter = function (value) {
+      if (!value) return "-";
+      return value.format("DD/MM/YY HH:mm");
+    }
+
+    var filterQueries = function () {
+      $scope.queries = _.filter($scope.allQueries, function (query) {
+        if (!$scope.selectedTab) {
+          return false;
         }
 
-        $scope.reloadDashboards();
-
-        $scope.currentUser = currentUser;
-        $scope.newDashboard = {
-            'name': null,
-            'layout': null
+        if ($scope.selectedTab.key == 'my') {
+          return query.user.id == currentUser.id && query.name != 'New Query';
+        } else if ($scope.selectedTab.key == 'drafts') {
+          return query.user.id == currentUser.id && query.name == 'New Query';
         }
 
-        $(window).click(function () {
-            notifications.getPermissions();
+        return query.name != 'New Query';
+      });
+    }
+
+    Query.query(function (queries) {
+      $scope.allQueries = _.map(queries, function (query) {
+        query.created_at = moment(query.created_at);
+        query.last_retrieved_at = moment(query.last_retrieved_at);
+        return query;
+      });
+
+      filterQueries();
+    });
+
+    $scope.gridColumns = [
+      {
+        "label": "Name",
+        "map": "name",
+        "cellTemplateUrl": "/views/queries_query_name_cell.html"
+      },
+      {
+        'label': 'Created By',
+        'map': 'user.name'
+      },
+      {
+        'label': 'Created At',
+        'map': 'created_at',
+        'formatFunction': dateFormatter
+      },
+      {
+        'label': 'Runtime (avg)',
+        'map': 'avg_runtime',
+        'formatFunction': function (value) {
+          return $filter('durationHumanize')(value);
+        }
+      },
+      {
+        'label': 'Runtime (min)',
+        'map': 'min_runtime',
+        'formatFunction': function (value) {
+          return $filter('durationHumanize')(value);
+        }
+      },
+      {
+        'label': 'Runtime (max)',
+        'map': 'max_runtime',
+        'formatFunction': function (value) {
+          return $filter('durationHumanize')(value);
+        }
+      },
+      {
+        'label': 'Last Executed At',
+        'map': 'last_retrieved_at',
+        'formatFunction': dateFormatter
+      },
+      {
+        'label': 'Times Executed',
+        'map': 'times_retrieved'
+      },
+      {
+        'label': 'Update Schedule',
+        'map': 'ttl',
+        'formatFunction': function (value) {
+          return $filter('refreshRateHumanize')(value);
+        }
+      }
+    ]
+    $scope.tabs = [
+      {"name": "My Queries", "key": "my"},
+      {"key": "all", "name": "All Queries"},
+      {"key": "drafts", "name": "Drafts"}
+    ];
+
+    $scope.$watch('selectedTab', function (tab) {
+      if (tab) {
+        $scope.$parent.pageTitle = tab.name;
+      }
+
+      filterQueries();
+    });
+  }
+
+  var MainCtrl = function ($scope, Dashboard, notifications) {
+    if (featureFlags.clientSideMetrics) {
+      $scope.$on('$locationChangeSuccess', function(event, newLocation, oldLocation) {
+        // This will be called once per actual page load.
+        Bucky.sendPagePerformance();
+      });
+    }
+
+
+    $scope.dashboards = [];
+    $scope.reloadDashboards = function () {
+      Dashboard.query(function (dashboards) {
+        $scope.dashboards = _.sortBy(dashboards, "name");
+        $scope.allDashboards = _.groupBy($scope.dashboards, function (d) {
+          parts = d.name.split(":");
+          if (parts.length == 1) {
+            return "Other";
+          }
+          return parts[0];
         });
+        $scope.otherDashboards = $scope.allDashboards['Other'] || [];
+        $scope.groupedDashboards = _.omit($scope.allDashboards, 'Other');
+      });
     }
 
-    var IndexCtrl = function($scope, Dashboard) {
-        $scope.$parent.pageTitle = "Home";
+    $scope.reloadDashboards();
 
-        $scope.archiveDashboard = function(dashboard) {
-            if (confirm('Are you sure you want to delete "' + dashboard.name + '" dashboard?')) {
-                dashboard.$delete(function() {
-                    $scope.$parent.reloadDashboards();
-                });
-            }
-        }
+    $scope.currentUser = currentUser;
+    $scope.newDashboard = {
+      'name': null,
+      'layout': null
     }
 
-    angular.module('redash.controllers', [])
-        .controller('QueriesCtrl', ['$scope', '$http', '$location', '$filter', 'Query', QueriesCtrl])
-        .controller('IndexCtrl', ['$scope', 'Dashboard', IndexCtrl])
-        .controller('MainCtrl', ['$scope', 'Dashboard', 'notifications', MainCtrl]);
+    $(window).click(function () {
+      notifications.getPermissions();
+    });
+  }
+
+  var IndexCtrl = function ($scope, Events, Dashboard) {
+    Events.record(currentUser, "view", "page", "homepage");
+    $scope.$parent.pageTitle = "Home";
+
+    $scope.archiveDashboard = function (dashboard) {
+      if (confirm('Are you sure you want to delete "' + dashboard.name + '" dashboard?')) {
+        Events.record(currentUser, "archive", "dashboard", dashboard.id);
+        dashboard.$delete(function () {
+          $scope.$parent.reloadDashboards();
+        });
+      }
+    }
+  }
+
+  angular.module('redash.controllers', [])
+    .controller('QueriesCtrl', ['$scope', '$http', '$location', '$filter', 'Query', QueriesCtrl])
+    .controller('IndexCtrl', ['$scope', 'Events', 'Dashboard', IndexCtrl])
+    .controller('MainCtrl', ['$scope', 'Dashboard', 'notifications', MainCtrl]);
 })();

rd_ui/app/scripts/controllers/dashboard.js

@@ -1,12 +1,66 @@
 (function() {
-  var DashboardCtrl = function($scope, $routeParams, $http, $timeout, Dashboard) {
+  var DashboardCtrl = function($scope, Events, Widget, $routeParams, $http, $timeout, $q, Dashboard) {
     $scope.refreshEnabled = false;
     $scope.refreshRate = 60;
-    $scope.dashboard = Dashboard.get({
-      slug: $routeParams.dashboardSlug
-    }, function(dashboard) {
-      $scope.$parent.pageTitle = dashboard.name;
-    });
+
+    var loadDashboard = _.throttle(function() {
+      $scope.dashboard = Dashboard.get({ slug: $routeParams.dashboardSlug }, function (dashboard) {
+        Events.record(currentUser, "view", "dashboard", dashboard.id);
+
+        $scope.$parent.pageTitle = dashboard.name;
+
+        var promises = [];
+
+        $scope.dashboard.widgets = _.map($scope.dashboard.widgets, function (row) {
+          return _.map(row, function (widget) {
+            var w = new Widget(widget);
+
+            if (w.visualization && dashboard.dashboard_filters_enabled) {
+              promises.push(w.getQuery().getQueryResultPromise());
+            }
+
+            return w;
+          });
+        });
+
+        $q.all(promises).then(function(queryResults) {
+          var filters = {};
+          _.each(queryResults, function(queryResult) {
+            var queryFilters = queryResult.getFilters();
+            _.each(queryFilters, function (filter) {
+              if (!_.has(filters, filter.name)) {
+                // TODO: first object should be a copy, otherwise one of the chart filters behaves different than the others.
+                filters[filter.name] = filter;
+                filters[filter.name].originFilters = [];
+
+                $scope.$watch(function () { return filter.current }, function (value) {
+                  _.each(filter.originFilters, function (originFilter) {
+                    originFilter.current = value;
+                  });
+                });
+              };
+
+              // TODO: merge values.
+              filters[filter.name].originFilters.push(filter);
+            });
+          });
+
+          if (dashboard.dashboard_filters_enabled) {
+            $scope.filters = _.values(filters);
+          }
+        });
+
+
+      }, function () {
+        // error...
+        // try again. we wrap loadDashboard with throttle so it doesn't happen too often.\
+        // we might want to consider exponential backoff and also move this as a general solution in $http/$resource for
+        // all AJAX calls.
+        loadDashboard();
+      });
+    }, 1000);
+
+    loadDashboard();
 
     var autoRefresh = function() {
       if ($scope.refreshEnabled) {
@@ -35,6 +89,8 @@
     $scope.triggerRefresh = function() {
       $scope.refreshEnabled = !$scope.refreshEnabled;
 
+      Events.record(currentUser, "autorefresh", "dashboard", dashboard.id, {'enable': $scope.refreshEnabled});
+
       if ($scope.refreshEnabled) {
         var refreshRate = _.min(_.flatten($scope.dashboard.widgets), function(widget) {
           return widget.visualization.query.ttl;
@@ -47,32 +103,41 @@
     };
   };
 
-  var WidgetCtrl = function($scope, $http, $location, Query) {
+  var WidgetCtrl = function($scope, Events, Query) {
     $scope.deleteWidget = function() {
-      if (!confirm('Are you sure you want to remove "' + $scope.widget.visualization.name + '" from the dashboard?')) {
+      if (!confirm('Are you sure you want to remove "' + $scope.widget.getName() + '" from the dashboard?')) {
         return;
       }
 
-      $http.delete('/api/widgets/' + $scope.widget.id).success(function() {
+      Events.record(currentUser, "delete", "widget", $scope.widget.id);
+
+      $scope.widget.$delete(function() {
         $scope.dashboard.widgets = _.map($scope.dashboard.widgets, function(row) {
           return _.filter(row, function(widget) {
-            return widget.id != $scope.widget.id;
+            return widget.id != undefined;
           })
         });
       });
     };
 
-    $scope.query = new Query($scope.widget.visualization.query);
-    $scope.queryResult = $scope.query.getQueryResult();
+    Events.record(currentUser, "view", "widget", $scope.widget.id);
 
-    $scope.updateTime = (new Date($scope.queryResult.getUpdatedAt())).toISOString();
-    $scope.nextUpdateTime = moment(new Date(($scope.query.updated_at + $scope.query.ttl + $scope.query.runtime + 300) * 1000)).fromNow();
+    if ($scope.widget.visualization) {
+      Events.record(currentUser, "view", "query", $scope.widget.visualization.query.id);
+      Events.record(currentUser, "view", "visualization", $scope.widget.visualization.id);
 
-    $scope.updateTime = '';
+      $scope.query = $scope.widget.getQuery();
+      $scope.queryResult = $scope.query.getQueryResult();
+      $scope.nextUpdateTime = moment(new Date(($scope.query.updated_at + $scope.query.ttl + $scope.query.runtime + 300) * 1000)).fromNow();
+
+      $scope.type = 'visualization';
+    } else {
+      $scope.type = 'textbox';
+    }
   };
 
   angular.module('redash.controllers')
-    .controller('DashboardCtrl', ['$scope', '$routeParams', '$http', '$timeout', 'Dashboard', DashboardCtrl])
-    .controller('WidgetCtrl', ['$scope', '$http', '$location', 'Query', WidgetCtrl])
+    .controller('DashboardCtrl', ['$scope', 'Events', 'Widget', '$routeParams', '$http', '$timeout', '$q', 'Dashboard', DashboardCtrl])
+    .controller('WidgetCtrl', ['$scope', 'Events', 'Query', WidgetCtrl])
 
 })();

rd_ui/app/scripts/controllers/query_source.js

@@ -1,7 +1,7 @@
 (function() {
   'use strict';
 
-  function QuerySourceCtrl($controller, $scope, $location, Query, Visualization, KeyboardShortcuts) {
+  function QuerySourceCtrl(Events, growl, $controller, $scope, $location, Query, Visualization, KeyboardShortcuts) {
     // extends QueryViewCtrl
     $controller('QueryViewCtrl', {$scope: $scope});
     // TODO:
@@ -9,6 +9,8 @@
     // Obviously it shouldn't be repeated, but we got bigger fish to fry.
     var DEFAULT_TAB = 'table';
 
+    Events.record(currentUser, 'view_source', 'query', $scope.query.id);
+
     var isNewQuery = !$scope.query.id,
         queryText = $scope.query.query,
         // ref to QueryViewCtrl.saveQuery
@@ -18,6 +20,9 @@
             if ($scope.canEdit) {
               $scope.saveQuery();
             }
+          },
+          'meta+enter': function () {
+            $scope.executeQuery();
           }
         };
 
@@ -27,6 +32,14 @@
 
     $scope.newVisualization = undefined;
 
+    // @override
+    Object.defineProperty($scope, 'showDataset', {
+      get: function() {
+        return $scope.queryResult && $scope.queryResult.getStatus() == 'done';
+      }
+    });
+
+
     KeyboardShortcuts.bind(shortcuts);
 
     // @override
@@ -47,6 +60,7 @@
     };
 
     $scope.duplicateQuery = function() {
+      Events.record(currentUser, 'fork', 'query', $scope.query.id);
       $scope.query.id = null;
       $scope.query.ttl = -1;
 
@@ -62,15 +76,21 @@
     $scope.deleteVisualization = function($e, vis) {
       $e.preventDefault();
       if (confirm('Are you sure you want to delete ' + vis.name + ' ?')) {
-        Visualization.delete(vis);
-        if ($scope.selectedTab == vis.id) {
-          $scope.selectedTab = DEFAULT_TAB;
-          $location.hash($scope.selectedTab);
-        }
-        $scope.query.visualizations =
-          $scope.query.visualizations.filter(function(v) {
-            return vis.id !== v.id;
-          });
+        Events.record(currentUser, 'delete', 'visualization', vis.id);
+
+        Visualization.delete(vis, function() {
+          if ($scope.selectedTab == vis.id) {
+            $scope.selectedTab = DEFAULT_TAB;
+            $location.hash($scope.selectedTab);
+          }
+          $scope.query.visualizations =
+            $scope.query.visualizations.filter(function (v) {
+              return vis.id !== v.id;
+            });
+        }, function () {
+          growl.addErrorMessage("Error deleting visualization. Maybe it's used in a dashboard?");
+        });
+
       }
     };
 
@@ -94,7 +114,7 @@
   }
 
   angular.module('redash.controllers').controller('QuerySourceCtrl', [
-    '$controller', '$scope', '$location', 'Query',
+    'Events', 'growl', '$controller', '$scope', '$location', 'Query',
     'Visualization', 'KeyboardShortcuts', QuerySourceCtrl
     ]);
-})();
+})();

rd_ui/app/scripts/controllers/query_view.js

@@ -1,10 +1,11 @@
 (function() {
   'use strict';
 
-  function QueryViewCtrl($scope, $route, $location, notifications, growl, Query, DataSource) {
+  function QueryViewCtrl($scope, Events, $route, $location, notifications, growl, Query, DataSource) {
     var DEFAULT_TAB = 'table';
 
     $scope.query = $route.current.locals.query;
+    Events.record(currentUser, 'view', 'query', $scope.query.id);
     $scope.queryResult = $scope.query.getQueryResult();
     $scope.queryExecuting = false;
 
@@ -15,6 +16,10 @@
       $scope.query.data_source_id = $scope.query.data_source_id || dataSources[0].id;
     });
 
+    // in view mode, latest dataset is always visible
+    // source mode changes this behavior
+    $scope.showDataset = true;
+
     $scope.lockButton = function(lock) {
       $scope.queryExecuting = lock;
     };
@@ -23,7 +28,7 @@
       if (data) {
         data.id = $scope.query.id;
       } else {
-        data = $scope.query;
+        data = _.clone($scope.query);
       }
 
       options = _.extend({}, {
@@ -31,21 +36,23 @@
         errorMessage: 'Query could not be saved'
       }, options);
 
-      delete $scope.query.latest_query_data;
+      delete data.latest_query_data;
+      delete data.queryResult;
 
       return Query.save(data, function() {
         growl.addSuccessMessage(options.successMessage);
       }, function(httpResponse) {
         growl.addErrorMessage(options.errorMessage);
-      })
-      .$promise;
+      }).$promise;
     }
 
     $scope.saveDescription = function() {
+      Events.record(currentUser, 'edit_description', 'query', $scope.query.id);
       $scope.saveQuery(undefined, {'description': $scope.query.description});
     };
 
     $scope.saveName = function() {
+      Events.record(currentUser, 'edit_name', 'query', $scope.query.id);
       $scope.saveQuery(undefined, {'name': $scope.query.name});
     };
 
@@ -53,23 +60,30 @@
       $scope.queryResult = $scope.query.getQueryResult(0);
       $scope.lockButton(true);
       $scope.cancelling = false;
+      Events.record(currentUser, 'execute', 'query', $scope.query.id);
     };
 
     $scope.cancelExecution = function() {
       $scope.cancelling = true;
       $scope.queryResult.cancelExecution();
+      Events.record(currentUser, 'cancel_execute', 'query', $scope.query.id);
     };
 
     $scope.updateDataSource = function() {
-        $scope.query.latest_query_data = null;
-        $scope.query.latest_query_data_id = null;
-        Query.save({
-            'id': $scope.query.id,
-            'data_source_id': $scope.query.data_source_id,
-            'latest_query_data_id': null
-        });
+      Events.record(currentUser, 'update_data_source', 'query', $scope.query.id);
 
-        $scope.executeQuery();
+      $scope.query.latest_query_data = null;
+      $scope.query.latest_query_data_id = null;
+
+      if ($scope.query.id) {
+        Query.save({
+          'id': $scope.query.id,
+          'data_source_id': $scope.query.data_source_id,
+          'latest_query_data_id': null
+        });
+      }
+
+      $scope.executeQuery();
     };
 
     $scope.setVisualizationTab = function (visualization) {
@@ -81,45 +95,20 @@
       $scope.$parent.pageTitle = $scope.query.name;
     });
 
-    $scope.$watch('queryResult && queryResult.getError()',
-      function(newError, oldError) {
-        if (newError == undefined) {
-          return;
-        }
+    $scope.$watch('queryResult && queryResult.getData()', function(data, oldData) {
+      if (!data) {
+        return;
+      }
 
-        if (oldError == undefined && newError != undefined) {
-          $scope.lockButton(false);
-        }
-      });
-
-    $scope.$watch('queryResult && queryResult.getData()',
-      function(data, oldData) {
-        if (!data) {
-          return;
-        }
-
-        $scope.filters = $scope.queryResult.getFilters();
-
-        if ($scope.queryResult.getId() == null) {
-          $scope.dataUri = "";
-        } else {
-          $scope.dataUri =
-            '/api/queries/' + $scope.query.id + '/results/' +
-            $scope.queryResult.getId() + '.csv';
-
-          $scope.dataFilename =
-            $scope.query.name.replace(" ", "_") +
-            moment($scope.queryResult.getUpdatedAt()).format("_YYYY_MM_DD") +
-            ".csv";
-        }
-      });
+      $scope.filters = $scope.queryResult.getFilters();
+    });
 
     $scope.$watch("queryResult && queryResult.getStatus()", function(status) {
       if (!status) {
         return;
       }
 
-      if (status == "done") {
+      if (status == 'done') {
         if ($scope.query.id &&
           $scope.query.latest_query_data_id != $scope.queryResult.getId() &&
           $scope.query.query_hash == $scope.queryResult.query_result.query_hash) {
@@ -129,9 +118,12 @@
           })
         }
         $scope.query.latest_query_data_id = $scope.queryResult.getId();
+        $scope.query.queryResult = $scope.queryResult;
 
         notifications.showNotification("re:dash", $scope.query.name + " updated.");
+      }
 
+      if (status === 'done' || status === 'failed') {
         $scope.lockButton(false);
       }
     });
@@ -139,11 +131,14 @@
     $scope.$watch(function() {
       return $location.hash()
     }, function(hash) {
+      if (hash == 'pivot') {
+        Events.record(currentUser, 'pivot', 'query', $scope.query && $scope.query.id);
+      }
       $scope.selectedTab = hash || DEFAULT_TAB;
     });
   };
 
   angular.module('redash.controllers')
     .controller('QueryViewCtrl',
-      ['$scope', '$route', '$location', 'notifications', 'growl', 'Query', 'DataSource', QueryViewCtrl]);
-})();
+      ['$scope', 'Events', '$route', '$location', 'notifications', 'growl', 'Query', 'DataSource', QueryViewCtrl]);
+})();

rd_ui/app/scripts/directives/dashboard_directives.js

@@ -3,8 +3,8 @@
 
   var directives = angular.module('redash.directives');
 
-  directives.directive('editDashboardForm', ['$http', '$location', '$timeout', 'Dashboard',
-    function($http, $location, $timeout, Dashboard) {
+  directives.directive('editDashboardForm', ['Events', '$http', '$location', '$timeout', 'Dashboard',
+    function(Events, $http, $location, $timeout, Dashboard) {
       return {
         restrict: 'E',
         scope: {
@@ -46,7 +46,7 @@
                       row: rowIndex + 1,
                       ySize: 1,
                       xSize: widget.width,
-                      name: widget.visualization.query.name
+                      name: widget.getName()//visualization.query.name
                     });
                   });
                 });
@@ -54,7 +54,6 @@
                 _.each(layout, function(item) {
                   var el = gsItemTemplate.replace('{id}', item.id).replace('{name}', item.name);
                   gridster.add_widget(el, item.xSize, item.ySize, item.col, item.row);
-
                 });
               }
             });
@@ -89,14 +88,22 @@
                 $scope.dashboard = new Dashboard(response);
                 $scope.saveInProgress = false;
                 $(element).modal('hide');
-              })
+              });
+              Events.record(currentUser, 'edit', 'dashboard', $scope.dashboard.id);
             } else {
+
               $http.post('/api/dashboards', {
                 'name': $scope.dashboard.name
               }).success(function(response) {
                 $(element).modal('hide');
+                $scope.dashboard = {
+                  'name': null,
+                  'layout': null
+                };
+                $scope.saveInProgress = false;
                 $location.path('/dashboard/' + response.slug).replace();
-              })
+              });
+              Events.record(currentUser, 'create', 'dashboard');
             }
           }
 
@@ -123,25 +130,37 @@
             value: 2
           }];
 
+          $scope.type = 'visualization';
+
+          $scope.isVisualization = function () {
+            return $scope.type == 'visualization';
+          };
+
+          $scope.isTextBox = function () {
+            return $scope.type == 'textbox';
+          };
+
+          $scope.setType = function (type) {
+            $scope.type = type;
+          };
+
           var reset = function() {
             $scope.saveInProgress = false;
             $scope.widgetSize = 1;
             $scope.queryId = null;
             $scope.selectedVis = null;
             $scope.query = null;
-
-          }
+            $scope.text = "";
+          };
 
           reset();
 
-          $scope.loadVisualizations = function() {
+          $scope.loadVisualizations = function () {
             if (!$scope.queryId) {
               return;
             }
 
-            Query.get({
-              id: $scope.queryId
-            }, function(query) {
+            Query.get({ id: $scope.queryId }, function(query) {
               if (query) {
                 $scope.query = query;
                 if (query.visualizations.length) {
@@ -155,19 +174,21 @@
             $scope.saveInProgress = true;
 
             var widget = new Widget({
-              'visualization_id': $scope.selectedVis.id,
+              'visualization_id': $scope.selectedVis && $scope.selectedVis.id,
               'dashboard_id': $scope.dashboard.id,
               'options': {},
-              'width': $scope.widgetSize
+              'width': $scope.widgetSize,
+              'text': $scope.text
             });
 
             widget.$save().then(function(response) {
               // update dashboard layout
               $scope.dashboard.layout = response['layout'];
+              var newWidget = new Widget(response['widget']);
               if (response['new_row']) {
-                $scope.dashboard.widgets.push([response['widget']]);
+                $scope.dashboard.widgets.push([newWidget]);
               } else {
-                $scope.dashboard.widgets[$scope.dashboard.widgets.length - 1].push(response['widget']);
+                $scope.dashboard.widgets[$scope.dashboard.widgets.length - 1].push(newWidget);
               }
 
               // close the dialog

rd_ui/app/scripts/directives/directives.js

@@ -1,219 +1,227 @@
-(function() {
-    'use strict';
+(function () {
+  'use strict';
 
-    var directives = angular.module('redash.directives', []);
+  var directives = angular.module('redash.directives', []);
 
-    directives.directive('alertUnsavedChanges', ['$window', function($window) {
-        return {
-            restrict: 'E',
-            replace: true,
-            scope: {
-                'isDirty': '='
-            },
-            link: function($scope) {
-                var
+  directives.directive('alertUnsavedChanges', ['$window', function ($window) {
+    return {
+      restrict: 'E',
+      replace: true,
+      scope: {
+        'isDirty': '='
+      },
+      link: function ($scope) {
+        var
 
-                unloadMessage = "You will lose your changes if you leave",
-                confirmMessage = unloadMessage + "\n\nAre you sure you want to leave this page?",
+          unloadMessage = "You will lose your changes if you leave",
+          confirmMessage = unloadMessage + "\n\nAre you sure you want to leave this page?",
 
-                // store original handler (if any)
-                _onbeforeunload = $window.onbeforeunload;
+        // store original handler (if any)
+          _onbeforeunload = $window.onbeforeunload;
 
-                $window.onbeforeunload = function() {
-                    return $scope.isDirty ? unloadMessage : null;
-                }
-
-                $scope.$on('$locationChangeStart', function(event, next, current) {
-                  if (next.split("#")[0] == current.split("#")[0]) {
-                    return;
-                  }
-
-                  if ($scope.isDirty && !confirm(confirmMessage)) {
-                    event.preventDefault();
-                  }
-                });
-
-                $scope.$on('$destroy', function() {
-                    $window.onbeforeunload = _onbeforeunload;
-                });
-            }
+        $window.onbeforeunload = function () {
+          return $scope.isDirty ? unloadMessage : null;
         }
-    }]);
 
-    directives.directive('rdTab', function() {
-        return {
-            restrict: 'E',
-            scope: {
-                'tabId': '@',
-                'name': '@'
-            },
-            transclude: true,
-            template: '<li class="rd-tab" ng-class="{active: tabId==selectedTab}"><a href="#{{tabId}}">{{name}}<span ng-transclude></span></a></li>',
-            replace: true,
-            link: function(scope) {
-                scope.$watch(function(){return scope.$parent.selectedTab}, function(tab) {
-                    scope.selectedTab = tab;
-                });
-            }
+        $scope.$on('$locationChangeStart', function (event, next, current) {
+          if (next.split("#")[0] == current.split("#")[0]) {
+            return;
+          }
+
+          if ($scope.isDirty && !confirm(confirmMessage)) {
+            event.preventDefault();
+          }
+        });
+
+        $scope.$on('$destroy', function () {
+          $window.onbeforeunload = _onbeforeunload;
+        });
+      }
+    }
+  }]);
+
+  directives.directive('rdTab', function () {
+    return {
+      restrict: 'E',
+      scope: {
+        'tabId': '@',
+        'name': '@'
+      },
+      transclude: true,
+      template: '<li class="rd-tab" ng-class="{active: tabId==selectedTab}"><a href="#{{tabId}}">{{name}}<span ng-transclude></span></a></li>',
+      replace: true,
+      link: function (scope) {
+        scope.$watch(function () {
+          return scope.$parent.selectedTab
+        }, function (tab) {
+          scope.selectedTab = tab;
+        });
+      }
+    }
+  });
+
+  directives.directive('rdTabs', ['$location', function ($location) {
+    return {
+      restrict: 'E',
+      scope: {
+        tabsCollection: '=',
+        selectedTab: '='
+      },
+      template: '<ul class="nav nav-tabs"><li ng-class="{active: tab==selectedTab}" ng-repeat="tab in tabsCollection"><a href="#{{tab.key}}">{{tab.name}}</a></li></ul>',
+      replace: true,
+      link: function ($scope, element, attrs) {
+        $scope.selectTab = function (tabKey) {
+          $scope.selectedTab = _.find($scope.tabsCollection, function (tab) {
+            return tab.key == tabKey;
+          });
         }
-    });
 
-    directives.directive('rdTabs', ['$location', function($location) {
-        return {
-            restrict: 'E',
-            scope: {
-                tabsCollection: '=',
-                selectedTab: '='
-            },
-            template: '<ul class="nav nav-tabs"><li ng-class="{active: tab==selectedTab}" ng-repeat="tab in tabsCollection"><a href="#{{tab.key}}">{{tab.name}}</a></li></ul>',
-            replace: true,
-            link: function($scope, element, attrs) {
-                $scope.selectTab = function(tabKey) {
-                    $scope.selectedTab  = _.find($scope.tabsCollection, function(tab) { return tab.key == tabKey; });
-                }
+        $scope.$watch(function () {
+          return $location.hash()
+        }, function (hash) {
+          if (hash) {
+            $scope.selectTab($location.hash());
+          } else {
+            $scope.selectTab($scope.tabsCollection[0].key);
+          }
+        });
+      }
+    }
+  }]);
 
-                $scope.$watch(function() { return $location.hash()}, function(hash) {
-                    if (hash) {
-                        $scope.selectTab($location.hash());
-                    } else {
-                        $scope.selectTab($scope.tabsCollection[0].key);
-                    }
-                });
-            }
-        }
-    }]);
+  // From: http://jsfiddle.net/joshdmiller/NDFHg/
+  directives.directive('editInPlace', function () {
+    return {
+      restrict: 'E',
+      scope: {
+        value: '=',
+        ignoreBlanks: '=',
+        editable: '=',
+        done: '='
+      },
+      template: function (tElement, tAttrs) {
+        var elType = tAttrs.editor || 'input';
+        var placeholder = tAttrs.placeholder || 'Click to edit';
+        return '<span ng-click="editable && edit()" ng-bind="value" ng-class="{editable: editable}"></span>' +
+          '<span ng-click="editable && edit()" ng-show="editable && !value" ng-class="{editable: editable}">' + placeholder + '</span>' +
+          '<{elType} ng-model="value" class="rd-form-control"></{elType}>'.replace('{elType}', elType);
+      },
+      link: function ($scope, element, attrs) {
+        // Let's get a reference to the input element, as we'll want to reference it.
+        var inputElement = angular.element(element.children()[2]);
 
-    // From: http://jsfiddle.net/joshdmiller/NDFHg/
-    directives.directive('editInPlace', function () {
-        return {
-            restrict: 'E',
-            scope: {
-                value: '=',
-                ignoreBlanks: '=',
-                editable: '=',
-                done: '='
-            },
-            template: function(tElement, tAttrs) {
-                var elType = tAttrs.editor || 'input';
-                var placeholder = tAttrs.placeholder || 'Click to edit';
-                return '<span ng-click="editable && edit()" ng-bind="value" ng-class="{editable: editable}"></span>' +
-                       '<span ng-click="editable && edit()" ng-show="editable && !value" ng-class="{editable: editable}">' + placeholder + '</span>' +
-                       '<{elType} ng-model="value" class="rd-form-control"></{elType}>'.replace('{elType}', elType);
-            },
-            link: function ($scope, element, attrs) {
-                // Let's get a reference to the input element, as we'll want to reference it.
-                var inputElement = angular.element(element.children()[2]);
+        // This directive should have a set class so we can style it.
+        element.addClass('edit-in-place');
 
-                // This directive should have a set class so we can style it.
-                element.addClass('edit-in-place');
+        // Initially, we're not editing.
+        $scope.editing = false;
 
-                // Initially, we're not editing.
-                $scope.editing = false;
+        // ng-click handler to activate edit-in-place
+        $scope.edit = function () {
+          $scope.oldValue = $scope.value;
 
-                // ng-click handler to activate edit-in-place
-                $scope.edit = function () {
-                    $scope.oldValue = $scope.value;
+          $scope.editing = true;
 
-                    $scope.editing = true;
+          // We control display through a class on the directive itself. See the CSS.
+          element.addClass('active');
 
-                    // We control display through a class on the directive itself. See the CSS.
-                    element.addClass('active');
-
-                    // And we must focus the element.
-                    // `angular.element()` provides a chainable array, like jQuery so to access a native DOM function,
-                    // we have to reference the first element in the array.
-                    inputElement[0].focus();
-                };
-
-                function save() {
-                    if ($scope.editing) {
-                        if ($scope.ignoreBlanks && _.isEmpty($scope.value)) {
-                            $scope.value = $scope.oldValue;
-                        }
-                        $scope.editing = false;
-                        element.removeClass('active');
-
-                        if ($scope.value !== $scope.oldValue) {
-                            $scope.done && $scope.done();
-                        }
-                    }
-                }
-
-                $(inputElement).keydown(function(e) {
-                    // 'return' or 'enter' key pressed
-                    // allow 'shift' to break lines
-                    if (e.which === 13 && !e.shiftKey) {
-                        save();
-                    } else if (e.which === 27) {
-                        $scope.value = $scope.oldValue;
-                        $scope.$apply(function() {
-                            $(inputElement[0]).blur();
-                        });
-                    }
-                }).blur(function() {
-                    save();
-                });
-            }
+          // And we must focus the element.
+          // `angular.element()` provides a chainable array, like jQuery so to access a native DOM function,
+          // we have to reference the first element in the array.
+          inputElement[0].focus();
         };
-    });
 
-    // http://stackoverflow.com/a/17904092/1559840
-    directives.directive('jsonText', function() {
-        return {
-            restrict: 'A',
-            require: 'ngModel',
-            link: function(scope, element, attr, ngModel) {
-              function into(input) {
-                return JSON.parse(input);
-              }
-              function out(data) {
-                return JSON.stringify(data, undefined, 2);
-              }
-              ngModel.$parsers.push(into);
-              ngModel.$formatters.push(out);
-
-              scope.$watch(attr.ngModel, function(newValue) {
-                element[0].value = out(newValue);
-              }, true);
+        function save() {
+          if ($scope.editing) {
+            if ($scope.ignoreBlanks && _.isEmpty($scope.value)) {
+              $scope.value = $scope.oldValue;
             }
-        };
-    });
+            $scope.editing = false;
+            element.removeClass('active');
 
-    directives.directive('rdTimer', [function () {
-        return {
-            restrict: 'E',
-            scope: { timestamp: '=' },
-            template: '{{currentTime}}',
-            controller: ['$scope' ,function ($scope) {
-                $scope.currentTime = "00:00:00";
-
-                // We're using setInterval directly instead of $timeout, to avoid using $apply, to
-                // prevent the digest loop being run every second.
-                var currentTimer = setInterval(function() {
-                    $scope.currentTime = moment(moment() - moment($scope.timestamp)).utc().format("HH:mm:ss");
-                    $scope.$digest();
-                }, 1000);
-
-                $scope.$on('$destroy', function () {
-                    if (currentTimer) {
-                        clearInterval(currentTimer);
-                        currentTimer = null;
-                    }
-                });
-            }]
-        };
-    }]);
-
-    directives.directive('rdTimeAgo', function() {
-        return {
-            restrict: 'E',
-            scope: {
-                value: '='
-            },
-            template: '<span>' +
-                        '<span ng-show="value" am-time-ago="value"></span>' +
-                        '<span ng-hide="value">-</span>' +
-                      '</span>'
+            if ($scope.value !== $scope.oldValue) {
+              $scope.done && $scope.done();
+            }
+          }
         }
-    });
+
+        $(inputElement).keydown(function (e) {
+          // 'return' or 'enter' key pressed
+          // allow 'shift' to break lines
+          if (e.which === 13 && !e.shiftKey) {
+            save();
+          } else if (e.which === 27) {
+            $scope.value = $scope.oldValue;
+            $scope.$apply(function () {
+              $(inputElement[0]).blur();
+            });
+          }
+        }).blur(function () {
+          save();
+        });
+      }
+    };
+  });
+
+  // http://stackoverflow.com/a/17904092/1559840
+  directives.directive('jsonText', function () {
+    return {
+      restrict: 'A',
+      require: 'ngModel',
+      link: function (scope, element, attr, ngModel) {
+        function into(input) {
+          return JSON.parse(input);
+        }
+
+        function out(data) {
+          return JSON.stringify(data, undefined, 2);
+        }
+
+        ngModel.$parsers.push(into);
+        ngModel.$formatters.push(out);
+
+        scope.$watch(attr.ngModel, function (newValue) {
+          element[0].value = out(newValue);
+        }, true);
+      }
+    };
+  });
+
+  directives.directive('rdTimer', [function () {
+    return {
+      restrict: 'E',
+      scope: { timestamp: '=' },
+      template: '{{currentTime}}',
+      controller: ['$scope' , function ($scope) {
+        $scope.currentTime = "00:00:00";
+
+        // We're using setInterval directly instead of $timeout, to avoid using $apply, to
+        // prevent the digest loop being run every second.
+        var currentTimer = setInterval(function () {
+          $scope.currentTime = moment(moment() - moment($scope.timestamp)).utc().format("HH:mm:ss");
+          $scope.$digest();
+        }, 1000);
+
+        $scope.$on('$destroy', function () {
+          if (currentTimer) {
+            clearInterval(currentTimer);
+            currentTimer = null;
+          }
+        });
+      }]
+    };
+  }]);
+
+  directives.directive('rdTimeAgo', function () {
+    return {
+      restrict: 'E',
+      scope: {
+        value: '='
+      },
+      template: '<span>' +
+        '<span ng-show="value" am-time-ago="value"></span>' +
+        '<span ng-hide="value">-</span>' +
+        '</span>'
+    }
+  });
 })();

rd_ui/app/scripts/directives/query_directives.js

@@ -38,6 +38,26 @@
     }
   }
 
+  function queryResultCSVLink() {
+    return {
+      restrict: 'A',
+      link: function (scope, element) {
+        scope.$watch('queryResult && queryResult.getData()', function(data) {
+          if (!data) {
+            return;
+          }
+
+          if (scope.queryResult.getId() == null) {
+            element.attr('href', '');
+          } else {
+            element.attr('href', '/api/queries/' + scope.query.id + '/results/' + scope.queryResult.getId() + '.csv');
+            element.attr('download', scope.query.name.replace(" ", "_") + moment(scope.queryResult.getUpdatedAt()).format("_YYYY_MM_DD") + ".csv");
+          }
+        });
+      }
+    }
+  }
+
   function queryEditor() {
     return {
       restrict: 'E',
@@ -135,6 +155,7 @@
   angular.module('redash.directives')
   .directive('queryLink', queryLink)
   .directive('querySourceLink', querySourceLink)
+  .directive('queryResultLink', queryResultCSVLink)
   .directive('queryEditor', queryEditor)
   .directive('queryRefreshSelect', queryRefreshSelect)
   .directive('queryFormatter', ['$http', queryFormatter]);

rd_ui/app/scripts/filters.js

@@ -1,61 +1,75 @@
 var durationHumanize = function (duration) {
-    var humanized = "";
-    if (duration == undefined) {
-        humanized = "-";
-    } else if (duration < 60) {
-        humanized = Math.round(duration) + "s";
-    } else if (duration > 3600*24) {
-        var days = Math.round(parseFloat(duration) / 60.0 / 60.0 / 24.0);
-        humanized = days + "days";
-    } else if (duration >= 3600) {
-        var hours = Math.round(parseFloat(duration) / 60.0 / 60.0);
-        humanized = hours + "h";
-    } else {
-        var minutes = Math.round(parseFloat(duration) / 60.0);
-        humanized = minutes + "m";
-    }
-    return humanized;
-}
+  var humanized = "";
+  if (duration == undefined) {
+    humanized = "-";
+  } else if (duration < 60) {
+    humanized = Math.round(duration) + "s";
+  } else if (duration > 3600 * 24) {
+    var days = Math.round(parseFloat(duration) / 60.0 / 60.0 / 24.0);
+    humanized = days + "days";
+  } else if (duration >= 3600) {
+    var hours = Math.round(parseFloat(duration) / 60.0 / 60.0);
+    humanized = hours + "h";
+  } else {
+    var minutes = Math.round(parseFloat(duration) / 60.0);
+    humanized = minutes + "m";
+  }
+  return humanized;
+};
+
+var urlPattern = /(^|[\s\n]|<br\/?>)((?:https?|ftp):\/\/[\-A-Z0-9+\u0026\u2019@#\/%?=()~_|!:,.;]*[\-A-Z0-9+\u0026@#\/%=~()_|])/gi;
 
 angular.module('redash.filters', []).
-    filter('durationHumanize', function () {
-        return durationHumanize;
-    })
+  filter('durationHumanize', function () {
+    return durationHumanize;
+  })
 
-    .filter('refreshRateHumanize', function () {
-        return function (ttl) {
-            if (ttl==-1) {
-                return "Never";
-            } else {
-                return "Every " + durationHumanize(ttl);
-            }
-        }
-    })
+  .filter('refreshRateHumanize', function () {
+    return function (ttl) {
+      if (ttl == -1) {
+        return "Never";
+      } else {
+        return "Every " + durationHumanize(ttl);
+      }
+    }
+  })
 
-    .filter('toHuman', function() {
-        return function(text) {
-            return text.replace(/_/g, ' ').replace(/(?:^|\s)\S/g, function (a) {
-                return a.toUpperCase();
-            });
-        }
-    })
+  .filter('toHuman', function () {
+    return function (text) {
+      return text.replace(/_/g, ' ').replace(/(?:^|\s)\S/g, function (a) {
+        return a.toUpperCase();
+      });
+    }
+  })
 
-    .filter('colWidth', function () {
-        return function (widgetWidth) {
-            if (widgetWidth == 1) {
-                return 6;
-            }
-            return 12;
-        }
-    })
-    
-    .filter('capitalize', function () {
-        return function (text) {
-            if (text) {
-                return text[0].toUpperCase() + text.slice(1).toLowerCase();
-            } else {
-                return null;
-            }
-            
-        }
-    });
+  .filter('colWidth', function () {
+    return function (widgetWidth) {
+      if (widgetWidth == 1) {
+        return 6;
+      }
+      return 12;
+    }
+  })
+
+  .filter('capitalize', function () {
+    return function (text) {
+      if (text) {
+        return _.str.capitalize(text);
+      } else {
+        return null;
+      }
+
+    }
+  })
+
+  .filter('linkify', function () {
+    return function (text) {
+      return text.replace(urlPattern, "$1<a href='$2' target='_blank'>$2</a>");
+    };
+  })
+
+  .filter('markdown', ['$sce', function ($sce) {
+    return function (text) {
+      return $sce.trustAsHtml(marked(text));
+    }
+  }]);

rd_ui/app/scripts/ng_highchart.js

@@ -13,11 +13,23 @@
     xAxis: {
       type: 'datetime'
     },
-    yAxis: {
-      title: {
-        text: null
+    yAxis: [
+      {
+        title: {
+          text: null
+        },
+        // showEmpty: true // by default
+      },
+      {
+        title: {
+          text: null
+        },
+        opposite: true,
+        showEmpty: false
       }
-    },
+    ],
+
+
     tooltip: {
       valueDecimals: 2,
       formatter: function () {
@@ -81,6 +93,45 @@
                   series.update({stacking: newStacking}, true);
                 });
               }
+            },
+            {
+              text: 'Select All',
+              onclick: function () {
+                _.each(this.series, function (s) {
+                  s.setVisible(true, false);
+                });
+                this.redraw();
+              }
+            },
+            {
+              text: 'Unselect All',
+              onclick: function () {
+                _.each(this.series, function (s) {
+                  s.setVisible(false, false);
+                });
+                this.redraw();
+              }
+            },
+            {
+              text: 'Show Total',
+              onclick: function () {
+                var data = {};
+                _.each(this.series, function (s) {
+                  s.setVisible(false, false);
+                  _.each(s.data, function (p) {
+                    data[p.x] = data[p.x] || {'x': p.x, 'y': 0};
+                    data[p.x].y = data[p.x].y + p.y;
+                  });
+                });
+
+                this.addSeries({
+                  data: _.values(data),
+                  type: 'line',
+                  name: 'Total'
+                }, false)
+
+                this.redraw();
+              }
             }
           ]
         }
@@ -226,7 +277,7 @@
                 }
               }
 
-              if (chartOptions['xAxis']['type'] == 'category') {
+              if (chartOptions['xAxis']['type'] == 'category' || chartOptions['series']['type']=='pie') {
                 if (!angular.isDefined(scope.series[0].data[0].name)) {
                   // We need to make sure that for each category, each series has a value.
                   var categories = _.union.apply(this, _.map(scope.series, function (s) {

rd_ui/app/scripts/ng_smart_table.js

@@ -217,7 +217,7 @@
                             element.html('<div editable-cell="" row="dataRow" column="column" type="column.type"></div>');
                             compile(element.contents())(scope);
                         } else {
-                            element.text(scope.formatedValue);
+                            element.html(scope.formatedValue);
                         }
                     }
 

rd_ui/app/scripts/services/notifications.js

@@ -1,5 +1,5 @@
 (function () {
-    var notifications = function () {
+    var notifications = function (Events) {
         var notificationService = {};
         var lastNotification = null;
 
@@ -40,6 +40,7 @@
             notification.onclick = function () {
                 window.focus();
                 this.cancel();
+                Events.record(currentUser, 'click', 'notification');
             };
 
             notification.show()
@@ -49,5 +50,5 @@
     }
 
     angular.module('redash.services')
-        .factory('notifications', notifications);
+        .factory('notifications', ['Events', notifications]);
 })();

rd_ui/app/scripts/services/resources.js

@@ -1,357 +1,454 @@
 (function () {
-    var QueryResult = function($resource, $timeout) {
-        var QueryResultResource = $resource('/api/query_results/:id', {id: '@id'}, {'post': {'method': 'POST'}});
-        var Job = $resource('/api/jobs/:id', {id: '@id'});
+  var QueryResult = function ($resource, $timeout, $q) {
+    var QueryResultResource = $resource('/api/query_results/:id', {id: '@id'}, {'post': {'method': 'POST'}});
+    var Job = $resource('/api/jobs/:id', {id: '@id'});
 
-        var updateFunction = function (props) {
-            angular.extend(this, props);
-            if ('query_result' in props) {
-                this.status = "done";
-                this.filters = undefined;
-                this.filterFreeze = undefined;
+    var updateFunction = function (props) {
+      angular.extend(this, props);
+      if ('query_result' in props) {
+        this.status = "done";
+        this.filters = undefined;
+        this.filterFreeze = undefined;
 
-                _.each(this.query_result.data.rows, function (row) {
-                    _.each(row, function (v, k) {
-                        if (_.isString(v) && v.match(/^\d{4}-\d{2}-\d{2}/)) {
-                            row[k] = moment(v);
-                        }
-                    });
-                });
-            } else if (this.job.status == 3) {
-                this.status = "processing";
-            } else {
-                this.status = undefined;
+        var columnTypes = {};
+
+        _.each(this.query_result.data.rows, function (row) {
+          _.each(row, function (v, k) {
+            if (angular.isNumber(v)) {
+              columnTypes[k] = 'float';
+            } else if (_.isString(v) && v.match(/^\d{4}-\d{2}-\d{2}T/)) {
+              row[k] = moment(v);
+              columnTypes[k] = 'datetime';
+            } else if (_.isString(v) && v.match(/^\d{4}-\d{2}-\d{2}/)) {
+              row[k] = moment(v);
+              columnTypes[k] = 'date';
             }
-        }
+          }, this);
+        }, this);
 
-        function QueryResult(props) {
-            this.job = {};
-            this.query_result = {};
-            this.status = "waiting";
-            this.filters = undefined;
-            this.filterFreeze = undefined;
+        _.each(this.query_result.data.columns, function(column) {
+          if (columnTypes[column.name]) {
+            column.type = columnTypes[column.name];
+          }
+        });
 
-            this.updatedAt = moment();
-
-            if (props) {
-                updateFunction.apply(this, [props]);
-            }
-        }
-
-        var statuses = {
-            1: "waiting",
-            2: "processing",
-            3: "done",
-            4: "failed"
-        }
-
-        QueryResult.prototype.update = updateFunction;
-
-        QueryResult.prototype.getId = function() {
-            var id = null;
-            if ('query_result' in this) {
-                id = this.query_result.id;
-            }
-            return id;
-        }
-
-        QueryResult.prototype.cancelExecution = function() {
-            Job.delete({id: this.job.id});
-        }
-
-        QueryResult.prototype.getStatus = function() {
-            return this.status || statuses[this.job.status];
-        }
-
-        QueryResult.prototype.getError = function() {
-            // TODO: move this logic to the server...
-            if (this.job.error == "None") {
-                return undefined;
-            }
-
-            return this.job.error;
-        }
-
-        QueryResult.prototype.getUpdatedAt = function() {
-            return this.query_result.retrieved_at || this.job.updated_at*1000.0 || this.updatedAt;
-        }
-
-        QueryResult.prototype.getRuntime = function() {
-            return this.query_result.runtime;
-        }
-
-        QueryResult.prototype.getRawData = function() {
-            if (!this.query_result.data) {
-                return null;
-            }
-
-            var data = this.query_result.data.rows;
-
-            return data;
-        }
-
-        QueryResult.prototype.getData = function() {
-            if (!this.query_result.data) {
-                return null;
-            }
-
-            var filterValues = function(filters) {
-                if (!filters) {
-                    return null;
-                }
-
-                return _.reduce(filters, function(str, filter) {
-                    return str + filter.current;
-                }, "")
-            }
-
-            var filters = this.getFilters();
-            var filterFreeze = filterValues(filters);
-
-            if (this.filterFreeze != filterFreeze) {
-                this.filterFreeze = filterFreeze;
-
-                if (filters) {
-                    this.filteredData = _.filter(this.query_result.data.rows, function (row) {
-                        return _.reduce(filters, function (memo, filter) {
-                            return (memo && row[filter.name] == filter.current);
-                        }, true);
-                    });
-                } else {
-                    this.filteredData = this.query_result.data.rows;
-                }
-            }
-
-            return this.filteredData;
-        }
-
-        QueryResult.prototype.getChartData = function () {
-            var series = {};
-
-            _.each(this.getData(), function (row) {
-                var point = {};
-                var seriesName = undefined;
-                var xValue = 0;
-                var yValues = {};
-
-                _.each(row, function (value, definition) {
-                    var type = definition.split("::")[1];
-                    var name = definition.split("::")[0];
-
-                    if (type == 'x') {
-                        xValue = value;
-                        point[type] = value;
-                    }
-
-                    if (type == 'y') {
-                        yValues[name] = value;
-                        point[type] = value;
-                    }
-
-                    if (type == 'series') {
-                        seriesName = String(value);
-                    }
-                });
-
-                var addPointToSeries = function(seriesName, point) {
-                    if (series[seriesName] == undefined) {
-                        series[seriesName] = {
-                            name: seriesName,
-                            type: 'column',
-                            data: []
-                        }
-                    }
-
-                    series[seriesName]['data'].push(point);
-                }
-
-                if (seriesName === undefined) {
-                    _.each(yValues, function(yValue, seriesName) {
-                        addPointToSeries(seriesName, {'x': xValue, 'y': yValue});
-                    });
-                } else {
-                    addPointToSeries(seriesName, point);
-                }
-            });
-
-            _.each(series, function(series) {
-                series.data = _.sortBy(series.data, 'x');
-            });
-
-            return _.values(series);
-        };
-
-        QueryResult.prototype.getColumns = function () {
-            if (this.columns == undefined && this.query_result.data) {
-                this.columns = _.map(this.query_result.data.columns, function(v) {
-                    return v.name;
-                });
-            }
-
-            return this.columns;
-        }
-
-        QueryResult.prototype.getColumnCleanName = function (column) {
-            var parts = column.split('::');
-            var name = parts[1];
-            if (parts[0] != '') {
-                // TODO: it's probably time to generalize this.
-                // see also getColumnFriendlyName
-                name = parts[0].replace(/%/g, '__pct').replace(/ /g, '_').replace(/\?/g,'');
-            }
-            return name;
-        }
-
-        QueryResult.prototype.getColumnFriendlyName = function (column) {
-            return this.getColumnCleanName(column).replace('__pct', '%').replace(/_/g, ' ').replace(/(?:^|\s)\S/g, function (a) {
-                return a.toUpperCase();
-            });
-        }
-
-        QueryResult.prototype.getColumnCleanNames = function () {
-            return _.map(this.getColumns(), function (col) {
-                return this.getColumnCleanName(col);
-            }, this);
-        }
-
-        QueryResult.prototype.getColumnFriendlyNames = function () {
-            return _.map(this.getColumns(), function (col) {
-                return this.getColumnFriendlyName(col);
-            }, this);
-        }
-
-        QueryResult.prototype.getFilters = function () {
-            if (!this.filters) {
-                this.prepareFilters();
-            }
-
-            return this.filters;
-        };
-
-        QueryResult.prototype.prepareFilters = function() {
-            var filterNames = [];
-            _.each(this.getColumns(), function (col) {
-                if (col.split('::')[1] == 'filter') {
-                    filterNames.push(col);
-                }
-            });
-
-            var filterValues = [];
-            _.each(this.getRawData(), function (row) {
-                _.each(filterNames, function (filter, i) {
-                    if (filterValues[i] == undefined) {
-                        filterValues[i] = [];
-                    }
-                    filterValues[i].push(row[filter]);
-                })
-            });
-
-            this.filters = _.map(filterNames, function (filter, i) {
-                var f = {
-                    name: filter,
-                    friendlyName: this.getColumnFriendlyName(filter),
-                    values: _.uniq(filterValues[i])
-                };
-
-                f.current = f.values[0];
-                return f;
-            }, this);
-        }
-
-        var refreshStatus = function(queryResult, query, ttl) {
-            Job.get({'id': queryResult.job.id}, function(response) {
-                queryResult.update(response);
-
-                if (queryResult.getStatus() == "processing" && queryResult.job.query_result_id && queryResult.job.query_result_id != "None") {
-                    QueryResultResource.get({'id': queryResult.job.query_result_id}, function(response) {
-                        queryResult.update(response);
-                    });
-                } else if (queryResult.getStatus() != "failed") {
-                    $timeout(function () {
-                        refreshStatus(queryResult, query, ttl);
-                    }, 3000);
-                }
-            })
-        }
-
-        QueryResult.getById = function (id) {
-            var queryResult = new QueryResult();
-
-            QueryResultResource.get({'id': id}, function (response) {
-                queryResult.update(response);
-            });
-
-            return queryResult;
-        }
-
-        QueryResult.get = function (data_source_id, query, ttl) {
-            var queryResult = new QueryResult();
-
-            QueryResultResource.post({'data_source_id': data_source_id, 'query': query, 'ttl': ttl}, function (response) {
-                queryResult.update(response);
-
-                if ('job' in response) {
-                    refreshStatus(queryResult, query, ttl);
-                }
-            });
-
-            return queryResult;
-        }
-
-        return QueryResult;
-    };
-
-    var Query = function ($resource, QueryResult, DataSource) {
-        var Query = $resource('/api/queries/:id', {id: '@id'});
-
-        Query.newQuery = function() {
-            return new Query({
-                query: "",
-                name: "New Query",
-                ttl: -1,
-                user: currentUser
-            });
-        };
-
-        Query.prototype.getSourceLink = function() {
-            return '/queries/' + this.id + '/source';
-        };
-
-        Query.prototype.getQueryResult = function(ttl) {
-            if (ttl == undefined) {
-                ttl = this.ttl;
-            }
-
-            var queryResult = null;
-            if (this.latest_query_data && ttl != 0) {
-                queryResult = new QueryResult({'query_result': this.latest_query_data});
-            } else if (this.latest_query_data_id && ttl != 0) {
-                queryResult = QueryResult.getById(this.latest_query_data_id);
-            } else if (this.data_source_id) {
-                queryResult = QueryResult.get(this.data_source_id, this.query, ttl);
-            }
-
-            return queryResult;
-        };
-
-        return Query;
-    };
-
-    var DataSource = function($resource) {
-        var DataSourceResource = $resource('/api/data_sources/:id', {id: '@id'}, {'get': {'method': 'GET', 'cache': true, 'isArray': true}});
-
-        return DataSourceResource;
+        this.deferred.resolve(this);
+      } else if (this.job.status == 3) {
+        this.status = "processing";
+      } else {
+        this.status = undefined;
+      }
     }
 
-    var Widget = function($resource) {
-        var WidgetResource = $resource('/api/widgets/:id', {id: '@id'});
+    function QueryResult(props) {
+      this.deferred = $q.defer();
+      this.job = {};
+      this.query_result = {};
+      this.status = "waiting";
+      this.filters = undefined;
+      this.filterFreeze = undefined;
 
-        return WidgetResource;
+      this.updatedAt = moment();
+
+      if (props) {
+        updateFunction.apply(this, [props]);
+      }
     }
 
-    angular.module('redash.services')
-        .factory('QueryResult', ['$resource', '$timeout', QueryResult])
-        .factory('Query', ['$resource', 'QueryResult', 'DataSource', Query])
-        .factory('DataSource', ['$resource', DataSource])
-        .factory('Widget', ['$resource', Widget]);
+    var statuses = {
+      1: "waiting",
+      2: "processing",
+      3: "done",
+      4: "failed"
+    }
+
+    QueryResult.prototype.update = updateFunction;
+
+    QueryResult.prototype.getId = function () {
+      var id = null;
+      if ('query_result' in this) {
+        id = this.query_result.id;
+      }
+      return id;
+    }
+
+    QueryResult.prototype.cancelExecution = function () {
+      Job.delete({id: this.job.id});
+    }
+
+    QueryResult.prototype.getStatus = function () {
+      return this.status || statuses[this.job.status];
+    }
+
+    QueryResult.prototype.getError = function () {
+      // TODO: move this logic to the server...
+      if (this.job.error == "None") {
+        return undefined;
+      }
+
+      return this.job.error;
+    }
+
+    QueryResult.prototype.getUpdatedAt = function () {
+      return this.query_result.retrieved_at || this.job.updated_at * 1000.0 || this.updatedAt;
+    }
+
+    QueryResult.prototype.getRuntime = function () {
+      return this.query_result.runtime;
+    }
+
+    QueryResult.prototype.getRawData = function () {
+      if (!this.query_result.data) {
+        return null;
+      }
+
+      var data = this.query_result.data.rows;
+
+      return data;
+    }
+
+    QueryResult.prototype.getData = function () {
+      if (!this.query_result.data) {
+        return null;
+      }
+
+      var filterValues = function (filters) {
+        if (!filters) {
+          return null;
+        }
+
+        return _.reduce(filters, function (str, filter) {
+          return str + filter.current;
+        }, "")
+      }
+
+      var filters = this.getFilters();
+      var filterFreeze = filterValues(filters);
+
+      if (this.filterFreeze != filterFreeze) {
+        this.filterFreeze = filterFreeze;
+
+        if (filters) {
+          this.filteredData = _.filter(this.query_result.data.rows, function (row) {
+            return _.reduce(filters, function (memo, filter) {
+              if (!_.isArray(filter.current)) {
+                filter.current = [filter.current];
+              };
+
+              return (memo && _.some(filter.current, function(v) {
+                // We compare with either the value or the String representation of the value,
+                // because Select2 casts true/false to "true"/"false".
+                return v == row[filter.name] || String(row[filter.name]) == v
+              }));
+            }, true);
+          });
+        } else {
+          this.filteredData = this.query_result.data.rows;
+        }
+      }
+
+      return this.filteredData;
+    }
+
+    QueryResult.prototype.getChartData = function (mapping) {
+      var series = {};
+
+      _.each(this.getData(), function (row) {
+        var point = {};
+        var seriesName = undefined;
+        var xValue = 0;
+        var yValues = {};
+
+        _.each(row, function (value, definition) {
+          var name = definition.split("::")[0];
+          var type = definition.split("::")[1];
+          if (mapping) {
+            type = mapping[definition];
+          }
+
+          if (type == 'unused') {
+            return;
+          }
+
+          if (type == 'x') {
+            xValue = value;
+            point[type] = value;
+          }
+          if (type == 'y') {
+            if (value == null) {
+              value = 0;
+            }
+            yValues[name] = value;
+            point[type] = value;
+          }
+
+          if (type == 'series') {
+            seriesName = String(value);
+          }
+
+          if (type == 'multi-filter') {
+            seriesName = String(value);
+          }
+        });
+
+        var addPointToSeries = function (seriesName, point) {
+          if (series[seriesName] == undefined) {
+            series[seriesName] = {
+              name: seriesName,
+              type: 'column',
+              data: []
+            }
+          }
+
+          series[seriesName]['data'].push(point);
+        }
+
+        if (seriesName === undefined) {
+          _.each(yValues, function (yValue, seriesName) {
+            addPointToSeries(seriesName, {'x': xValue, 'y': yValue});
+          });
+        } else {
+          addPointToSeries(seriesName, point);
+        }
+      });
+
+      _.each(series, function (series) {
+        series.data = _.sortBy(series.data, 'x');
+      });
+
+      return _.values(series);
+    };
+
+    QueryResult.prototype.getColumns = function () {
+      if (this.columns == undefined && this.query_result.data) {
+        this.columns = this.query_result.data.columns;
+      }
+
+      return this.columns;
+    }
+
+    QueryResult.prototype.getColumnNames = function () {
+      if (this.columnNames == undefined && this.query_result.data) {
+        this.columnNames = _.map(this.query_result.data.columns, function (v) {
+          return v.name;
+        });
+      }
+
+      return this.columnNames;
+    }
+
+    QueryResult.prototype.getColumnNameWithoutType = function (column) {
+      var parts = column.split('::');
+      if (parts[0] == "" && parts.length == 2) {
+        return parts[1];
+      }
+      return parts[0];
+    };
+
+    var charConversionMap = {
+      '__pct': /%/g,
+      '_': / /g,
+      '__qm': /\?/g,
+      '__brkt': /[\(\)\[\]]/g,
+      '__dash': /-/g,
+      '__amp': /&/g,
+      '__sl': /\//g,
+      '__fsl': /\\/g,
+    };
+
+    QueryResult.prototype.getColumnCleanName = function (column) {
+      var name = this.getColumnNameWithoutType(column);
+
+      if (name != '') {
+        _.each(charConversionMap, function(regex, replacement) {
+          name = name.replace(regex, replacement);
+        });
+      }
+
+      return name;
+    }
+
+    QueryResult.prototype.getColumnFriendlyName = function (column) {
+      return this.getColumnNameWithoutType(column).replace(/(?:^|\s)\S/g, function (a) {
+        return a.toUpperCase();
+      });
+    }
+
+    QueryResult.prototype.getColumnCleanNames = function () {
+      return _.map(this.getColumnNames(), function (col) {
+        return this.getColumnCleanName(col);
+      }, this);
+    }
+
+    QueryResult.prototype.getColumnFriendlyNames = function () {
+      return _.map(this.getColumnNames(), function (col) {
+        return this.getColumnFriendlyName(col);
+      }, this);
+    }
+
+    QueryResult.prototype.getFilters = function () {
+      if (!this.filters) {
+        this.prepareFilters();
+      }
+
+      return this.filters;
+    };
+
+    QueryResult.prototype.prepareFilters = function () {
+      var filters = [];
+      var filterTypes = ['filter', 'multi-filter'];
+      _.each(this.getColumnNames(), function (col) {
+        var type = col.split('::')[1]
+        if (_.contains(filterTypes, type)) {
+          // filter found
+          var filter = {
+            name: col,
+            friendlyName: this.getColumnFriendlyName(col),
+            values: [],
+            multiple: (type=='multi-filter')
+          }
+          filters.push(filter);
+        }
+      }, this);
+
+      _.each(this.getRawData(), function (row) {
+        _.each(filters, function (filter) {
+          filter.values.push(row[filter.name]);
+          if (filter.values.length == 1) {
+            filter.current = row[filter.name];
+          }
+        })
+      });
+
+      _.each(filters, function(filter) {
+        filter.values = _.uniq(filter.values);
+      });
+
+      this.filters = filters;
+    }
+
+    var refreshStatus = function (queryResult, query, ttl) {
+      Job.get({'id': queryResult.job.id}, function (response) {
+        queryResult.update(response);
+
+        if (queryResult.getStatus() == "processing" && queryResult.job.query_result_id && queryResult.job.query_result_id != "None") {
+          QueryResultResource.get({'id': queryResult.job.query_result_id}, function (response) {
+            queryResult.update(response);
+          });
+        } else if (queryResult.getStatus() != "failed") {
+          $timeout(function () {
+            refreshStatus(queryResult, query, ttl);
+          }, 3000);
+        }
+      })
+    }
+
+    QueryResult.getById = function (id) {
+      var queryResult = new QueryResult();
+
+      QueryResultResource.get({'id': id}, function (response) {
+        queryResult.update(response);
+      });
+
+      return queryResult;
+    };
+
+    QueryResult.prototype.toPromise = function() {
+      return this.deferred.promise;
+    }
+
+    QueryResult.get = function (data_source_id, query, ttl) {
+      var queryResult = new QueryResult();
+
+      QueryResultResource.post({'data_source_id': data_source_id, 'query': query, 'ttl': ttl}, function (response) {
+        queryResult.update(response);
+
+        if ('job' in response) {
+          refreshStatus(queryResult, query, ttl);
+        }
+      });
+
+      return queryResult;
+    }
+
+    return QueryResult;
+  };
+
+  var Query = function ($resource, QueryResult, DataSource) {
+    var Query = $resource('/api/queries/:id', {id: '@id'});
+
+    Query.newQuery = function () {
+      return new Query({
+        query: "",
+        name: "New Query",
+        ttl: -1,
+        user: currentUser
+      });
+    };
+
+    Query.prototype.getSourceLink = function () {
+      return '/queries/' + this.id + '/source';
+    };
+
+    Query.prototype.getQueryResult = function (ttl) {
+      if (ttl == undefined) {
+        ttl = this.ttl;
+      }
+
+      if (this.latest_query_data && ttl != 0) {
+        if (!this.queryResult) {
+          this.queryResult = new QueryResult({'query_result': this.latest_query_data});
+        }
+      } else if (this.latest_query_data_id && ttl != 0) {
+        if (!this.queryResult) {
+          this.queryResult = QueryResult.getById(this.latest_query_data_id);
+        }
+      } else if (this.data_source_id) {
+        this.queryResult = QueryResult.get(this.data_source_id, this.query, ttl);
+      }
+
+      return this.queryResult;
+    };
+
+    Query.prototype.getQueryResultPromise = function() {
+      return this.getQueryResult().toPromise();
+    }
+
+    return Query;
+  };
+
+
+
+  var DataSource = function ($resource) {
+    var DataSourceResource = $resource('/api/data_sources/:id', {id: '@id'}, {'get': {'method': 'GET', 'cache': true, 'isArray': true}});
+
+    return DataSourceResource;
+  }
+
+  var Widget = function ($resource, Query) {
+    var WidgetResource = $resource('/api/widgets/:id', {id: '@id'});
+
+    WidgetResource.prototype.getQuery = function () {
+      if (!this.query && this.visualization) {
+         this.query = new Query(this.visualization.query);
+      }
+
+      return this.query;
+    };
+
+    WidgetResource.prototype.getName = function () {
+      if (this.visualization) {
+        return this.visualization.query.name + ' (' + this.visualization.name + ')';
+      }
+      return _.str.truncate(this.text, 20);
+    };
+
+    return WidgetResource;
+  }
+
+  angular.module('redash.services')
+      .factory('QueryResult', ['$resource', '$timeout', '$q', QueryResult])
+      .factory('Query', ['$resource', 'QueryResult', 'DataSource', Query])
+      .factory('DataSource', ['$resource', DataSource])
+      .factory('Widget', ['$resource', 'Query', Widget]);
 })();

rd_ui/app/scripts/services/services.js

@@ -1,24 +1,52 @@
-(function() {
+(function () {
   'use strict'
 
   function KeyboardShortcuts() {
     this.bind = function bind(keymap) {
-      _.forEach(keymap, function(fn, key) {
-        Mousetrap.bindGlobal(key, function(e) {
-            e.preventDefault();
-            fn();
+      _.forEach(keymap, function (fn, key) {
+        Mousetrap.bindGlobal(key, function (e) {
+          e.preventDefault();
+          fn();
         });
       });
 
     }
 
     this.unbind = function unbind(keymap) {
-      _.forEach(keymap, function(fn, key) {
+      _.forEach(keymap, function (fn, key) {
         Mousetrap.unbind(key);
       });
     }
   }
 
+  function Events($http) {
+    this.events = [];
+
+    this.post = _.debounce(function() {
+      var events = this.events;
+      this.events = [];
+
+      $http.post('/api/events', events);
+
+    }, 1000);
+
+    this.record = function (user, action, object_type, object_id, additional_properties) {
+
+      var event = {
+        "user_id": user.id,
+        "action": action,
+        "object_type": object_type,
+        "object_id": object_id,
+        "timestamp": Date.now()/1000.0
+      };
+      _.extend(event, additional_properties);
+      this.events.push(event);
+
+      this.post();
+    };
+  }
+
   angular.module('redash.services', [])
-    .service('KeyboardShortcuts', [KeyboardShortcuts])
+      .service('KeyboardShortcuts', [KeyboardShortcuts])
+      .service('Events', ['$http', Events])
 })();

rd_ui/app/scripts/visualizations/base.js

@@ -1,170 +1,214 @@
 (function () {
-    var VisualizationProvider = function() {
-        this.visualizations = {};
-        this.visualizationTypes = {};
-        var defaultConfig = {
-            defaultOptions: {},
-            skipTypes: false,
-            editorTemplate: null
-        }
-
-        this.registerVisualization = function(config) {
-            var visualization = _.extend({}, defaultConfig, config);
-
-            // TODO: this is prone to errors; better refactor.
-            if (_.isEmpty(this.visualizations)) {
-                this.defaultVisualization = visualization;
-            }
-
-            this.visualizations[config.type] = visualization;
-
-            if (!config.skipTypes) {
-                this.visualizationTypes[config.name] = config.type;
-            };
-        };
-
-        this.getSwitchTemplate = function(property) {
-            var pattern = /(<[a-zA-Z0-9-]*?)( |>)/
-
-            var mergedTemplates = _.reduce(this.visualizations, function(templates, visualization) {
-                if (visualization[property]) {
-                    var ngSwitch = '$1 ng-switch-when="' + visualization.type + '" $2';
-                    var template = visualization[property].replace(pattern, ngSwitch);
-
-                    return templates + "\n" + template;
-                }
-
-                return templates;
-            }, "");
-
-            mergedTemplates = '<div ng-switch on="visualization.type">'+ mergedTemplates + "</div>";
-
-            return mergedTemplates;
-        }
-
-        this.$get = ['$resource', function($resource) {
-            var Visualization = $resource('/api/visualizations/:id', {id: '@id'});
-            Visualization.visualizations = this.visualizations;
-            Visualization.visualizationTypes = this.visualizationTypes;
-            Visualization.renderVisualizationsTemplate = this.getSwitchTemplate('renderTemplate');
-            Visualization.editorTemplate = this.getSwitchTemplate('editorTemplate');
-            Visualization.defaultVisualization = this.defaultVisualization;
-
-            return Visualization;
-        }];
-    };
-
-    var VisualizationRenderer = function(Visualization) {
-        return {
-            restrict: 'E',
-            scope: {
-                visualization: '=',
-                queryResult: '='
-            },
-            // TODO: using switch here (and in the options editor) might introduce errors and bad
-            // performance wise. It's better to eventually show the correct template based on the
-            // visualization type and not make the browser render all of them.
-            template: '<filters></filters>\n' + Visualization.renderVisualizationsTemplate,
-            replace: false,
-            link: function(scope) {
-                scope.$watch('queryResult && queryResult.getFilters()', function(filters) {
-                    if (filters) {
-                        scope.filters = filters;
-                    }
-                });
-            }
-        }
-    };
-
-    var VisualizationOptionsEditor = function(Visualization) {
-        return {
-            restrict: 'E',
-            template: Visualization.editorTemplate,
-            replace: false
-        }
-    };
-
-    var Filters = function() {
-        return {
-            restrict: 'E',
-            templateUrl: '/views/visualizations/filters.html'
-        }
+  var VisualizationProvider = function () {
+    this.visualizations = {};
+    this.visualizationTypes = {};
+    var defaultConfig = {
+      defaultOptions: {},
+      skipTypes: false,
+      editorTemplate: null
     }
 
-    var EditVisualizationForm = function(Visualization, growl) {
-        return {
-            restrict: 'E',
-            templateUrl: '/views/visualizations/edit_visualization.html',
-            replace: true,
-            scope: {
-                query: '=',
-                queryResult: '=',
-                visualization: '=?',
-                onNewSuccess: '=?'
-            },
-            link: function (scope, element, attrs) {
-                scope.editRawOptions = currentUser.hasPermission('edit_raw_chart');
-                scope.visTypes = Visualization.visualizationTypes;
+    this.registerVisualization = function (config) {
+      var visualization = _.extend({}, defaultConfig, config);
 
-                scope.newVisualization = function(q) {
-                    return {
-                        'query_id': q.id,
-                        'type': Visualization.defaultVisualization.type,
-                        'name': Visualization.defaultVisualization.name,
-                        'description': q.description || '',
-                        'options': Visualization.defaultVisualization.defaultOptions
-                    };
-                }
+      // TODO: this is prone to errors; better refactor.
+      if (_.isEmpty(this.visualizations)) {
+        this.defaultVisualization = visualization;
+      }
 
-                if (!scope.visualization) {
-                    // create new visualization
-                    // wait for query to load to populate with defaults
-                    var unwatch = scope.$watch('query', function (q) {
-                        if (q && q.id) {
-                            unwatch();
+      this.visualizations[config.type] = visualization;
 
-                            scope.visualization = scope.newVisualization(q);
-                        }
-                    }, true);
-                }
-
-                scope.$watch('visualization.type', function (type, oldType) {
-                    // if not edited by user, set name to match type
-                    if (type && oldType != type && scope.visualization && !scope.visForm.name.$dirty) {
-                        // poor man's titlecase
-                        scope.visualization.name = scope.visualization.type[0] + scope.visualization.type.slice(1).toLowerCase();
-                    }
-                });
-
-                scope.submit = function () {
-                    Visualization.save(scope.visualization, function success(result) {
-                        growl.addSuccessMessage("Visualization saved");
-
-                        scope.visualization = scope.newVisualization(scope.query);
-
-                        var visIds = _.pluck(scope.query.visualizations, 'id');
-                        var index = visIds.indexOf(result.id);
-                        if (index > -1) {
-                            scope.query.visualizations[index] = result;
-                        } else {
-                            // new visualization
-                            scope.query.visualizations.push(result);
-                            scope.onNewSuccess && scope.onNewSuccess(result);
-                        }
-                    }, function error() {
-                        growl.addErrorMessage("Visualization could not be saved");
-                    });
-                };
-            }
-        }
+      if (!config.skipTypes) {
+        this.visualizationTypes[config.name] = config.type;
+      }
+      ;
     };
 
+    this.getSwitchTemplate = function (property) {
+      var pattern = /(<[a-zA-Z0-9-]*?)( |>)/
+
+      var mergedTemplates = _.reduce(this.visualizations, function (templates, visualization) {
+        if (visualization[property]) {
+          var ngSwitch = '$1 ng-switch-when="' + visualization.type + '" $2';
+          var template = visualization[property].replace(pattern, ngSwitch);
+
+          return templates + "\n" + template;
+        }
+
+        return templates;
+      }, "");
+
+      mergedTemplates = '<div ng-switch on="visualization.type">' + mergedTemplates + "</div>";
+
+      return mergedTemplates;
+    }
+
+    this.$get = ['$resource', function ($resource) {
+      var Visualization = $resource('/api/visualizations/:id', {id: '@id'});
+      Visualization.visualizations = this.visualizations;
+      Visualization.visualizationTypes = this.visualizationTypes;
+      Visualization.renderVisualizationsTemplate = this.getSwitchTemplate('renderTemplate');
+      Visualization.editorTemplate = this.getSwitchTemplate('editorTemplate');
+      Visualization.defaultVisualization = this.defaultVisualization;
+
+      return Visualization;
+    }];
+  };
+
+  var VisualizationRenderer = function ($location, Visualization) {
+    return {
+      restrict: 'E',
+      scope: {
+        visualization: '=',
+        queryResult: '='
+      },
+      // TODO: using switch here (and in the options editor) might introduce errors and bad
+      // performance wise. It's better to eventually show the correct template based on the
+      // visualization type and not make the browser render all of them.
+      template: '<filters></filters>\n' + Visualization.renderVisualizationsTemplate,
+      replace: false,
+      link: function (scope) {
+
+        scope.select2Options = {
+          width: '50%'
+        };
+
+        function readURL() {
+          var searchFilters = angular.fromJson($location.search().filters);
+          if (searchFilters) {
+            _.forEach(scope.filters, function(filter) {
+              var value = searchFilters[filter.friendlyName];
+              if (value) {
+                filter.current = value;
+              }
+            });
+          }
+        }
+
+        function updateURL(filters) {
+          var current = {};
+          _.each(filters, function(filter) {
+            if (filter.current) {
+              current[filter.friendlyName] = filter.current;
+            }
+          });
+
+          var newSearch = angular.extend($location.search(), {
+            filters: angular.toJson(current)
+          });
+          $location.search(newSearch);
+        }
+
+        scope.$watch('queryResult && queryResult.getFilters()', function (filters) {
+          if (filters) {
+            scope.filters = filters;
+
+            if (filters.length) {
+              readURL();
+
+              // start watching for changes and update URL
+              scope.$watch('filters', updateURL, true);
+            }
+          }
+        });
+      }
+    }
+  };
+
+  var VisualizationOptionsEditor = function (Visualization) {
+    return {
+      restrict: 'E',
+      template: Visualization.editorTemplate,
+      replace: false
+    }
+  };
+
+  var Filters = function () {
+    return {
+      restrict: 'E',
+      templateUrl: '/views/visualizations/filters.html'
+    }
+  }
+
+  var EditVisualizationForm = function (Events, Visualization, growl) {
+    return {
+      restrict: 'E',
+      templateUrl: '/views/visualizations/edit_visualization.html',
+      replace: true,
+      scope: {
+        query: '=',
+        queryResult: '=',
+        visualization: '=?',
+        openEditor: '=?',
+        onNewSuccess: '=?'
+      },
+      link: function (scope, element, attrs) {
+        scope.editRawOptions = currentUser.hasPermission('edit_raw_chart');
+        scope.visTypes = Visualization.visualizationTypes;
+
+        scope.newVisualization = function () {
+          return {
+            'type': Visualization.defaultVisualization.type,
+            'name': Visualization.defaultVisualization.name,
+            'description': '',
+            'options': Visualization.defaultVisualization.defaultOptions
+          };
+        }
+
+        if (!scope.visualization) {
+          var unwatch = scope.$watch('query.id', function (queryId) {
+            if (queryId) {
+              unwatch();
+
+              scope.visualization = scope.newVisualization();
+            }
+          });
+        }
+
+        scope.$watch('visualization.type', function (type, oldType) {
+          // if not edited by user, set name to match type
+          if (type && oldType != type && scope.visualization && !scope.visForm.name.$dirty) {
+            // poor man's titlecase
+            scope.visualization.name = scope.visualization.type[0] + scope.visualization.type.slice(1).toLowerCase();
+          }
+        });
+
+        scope.submit = function () {
+          if (scope.visualization.id) {
+            Events.record(currentUser, "update", "visualization", scope.visualization.id, {'type': scope.visualization.type});
+          } else {
+            Events.record(currentUser, "create", "visualization", null, {'type': scope.visualization.type});
+          }
+
+          scope.visualization.query_id = scope.query.id;
+
+          Visualization.save(scope.visualization, function success(result) {
+            growl.addSuccessMessage("Visualization saved");
+
+            scope.visualization = scope.newVisualization(scope.query);
+
+            var visIds = _.pluck(scope.query.visualizations, 'id');
+            var index = visIds.indexOf(result.id);
+            if (index > -1) {
+              scope.query.visualizations[index] = result;
+            } else {
+              // new visualization
+              scope.query.visualizations.push(result);
+              scope.onNewSuccess && scope.onNewSuccess(result);
+            }
+          }, function error() {
+            growl.addErrorMessage("Visualization could not be saved");
+          });
+        };
+      }
+    }
+  };
 
 
-    angular.module('redash.visualization', [])
-        .provider('Visualization', VisualizationProvider)
-        .directive('visualizationRenderer', ['Visualization', VisualizationRenderer])
-        .directive('visualizationOptionsEditor', ['Visualization', VisualizationOptionsEditor])
-        .directive('filters', Filters)
-        .directive('editVisulatizationForm', ['Visualization', 'growl', EditVisualizationForm])
+  angular.module('redash.visualization', [])
+      .provider('Visualization', VisualizationProvider)
+      .directive('visualizationRenderer', ['$location', 'Visualization', VisualizationRenderer])
+      .directive('visualizationOptionsEditor', ['Visualization', VisualizationOptionsEditor])
+      .directive('filters', Filters)
+      .directive('editVisulatizationForm', ['Events', 'Visualization', 'growl', EditVisualizationForm])
 })();

rd_ui/app/scripts/visualizations/chart.js

@@ -6,7 +6,7 @@
     var editTemplate = '<chart-editor></chart-editor>';
     var defaultOptions = {
       'series': {
-        'type': 'column',
+//        'type': 'column',
         'stacking': null
       }
     };
@@ -33,24 +33,45 @@
         $scope.chartSeries = [];
         $scope.chartOptions = {};
 
+        var reloadData = _.throttle(function(data) {
+          if (!data || ($scope.queryResult && $scope.queryResult.getData()) == null) {
+            $scope.chartSeries.splice(0, $scope.chartSeries.length);
+          } else {
+            $scope.chartSeries.splice(0, $scope.chartSeries.length);
+
+            _.each($scope.queryResult.getChartData($scope.options.columnMapping), function (s) {
+              var additional = {'stacking': 'normal'};
+              if ($scope.options.seriesOptions && $scope.options.seriesOptions[s.name]) {
+                additional = $scope.options.seriesOptions[s.name];
+                if (!additional.name || additional.name == "") {
+                  additional.name = s.name;
+                }
+              }
+              $scope.chartSeries.push(_.extend(s, additional));
+            });
+          }
+        }, 500);
+
         $scope.$watch('options', function (chartOptions) {
           if (chartOptions) {
             $scope.chartOptions = chartOptions;
           }
         });
-        $scope.$watch('queryResult && queryResult.getData()', function (data) {
-          if (!data || $scope.queryResult.getData() == null) {
-            $scope.chartSeries.splice(0, $scope.chartSeries.length);
-          } else {
-            $scope.chartSeries.splice(0, $scope.chartSeries.length);
 
-            _.each($scope.queryResult.getChartData(), function (s) {
-              $scope.chartSeries.push(_.extend(s, {'stacking': 'normal'}));
-            });
-          }
+        $scope.$watch('options.seriesOptions', function () {
+          reloadData(true);
+        }, true);
+
+
+        $scope.$watchCollection('options.columnMapping', function (chartOptions) {
+          reloadData(true);
+        });
+
+        $scope.$watch('queryResult && queryResult.getData()', function (data) {
+          reloadData(data);
         });
       }]
-    }
+    };
   });
 
   chartVisualization.directive('chartEditor', function () {
@@ -81,10 +102,26 @@
         scope.xAxisType = "datetime";
         scope.stacking = "none";
 
-        var chartOptionsUnwatch = null;
 
-        scope.$watch('visualization', function (visualization) {
-          if (visualization && visualization.type == 'CHART') {
+        scope.columnTypes = {
+          "X": "x",
+//          "X (Date time)": "x",
+//          "X (Linear)": "x-linear",
+//          "X (Category)": "x-category",
+          "Y": "y",
+          "Series": "series",
+          "Unused": "unused"
+        };
+
+        scope.series = [];
+
+        scope.columnTypeSelection = {};
+
+        var chartOptionsUnwatch = null,
+            columnsWatch = null;
+
+        scope.$watch('visualization.type', function (visualizationType) {
+          if (visualizationType == 'CHART') {
             if (scope.visualization.options.series.stacking === null) {
               scope.stacking = "none";
             } else if (scope.visualization.options.series.stacking === undefined) {
@@ -93,6 +130,72 @@
               scope.stacking = scope.visualization.options.series.stacking;
             }
 
+            var refreshSeries = function() {
+              scope.series = _.map(scope.queryResult.getChartData(scope.visualization.options.columnMapping), function (s) { return s.name; });
+
+              // TODO: remove uneeded ones?
+              if (scope.visualization.options.seriesOptions == undefined) {
+                scope.visualization.options.seriesOptions = {};
+              };
+
+              _.each(scope.series, function(s, i) {
+                if (scope.visualization.options.seriesOptions[s] == undefined) {
+                  scope.visualization.options.seriesOptions[s] = {'type': 'column', 'yAxis': 0};
+                }
+                scope.visualization.options.seriesOptions[s].zIndex = i;
+
+              });
+              scope.zIndexes = _.range(scope.series.length);
+              scope.yAxes = [[0, 'left'], [1, 'right']];
+            };
+
+            var initColumnMapping = function() {
+              scope.columns = scope.queryResult.getColumns();
+
+              if (scope.visualization.options.columnMapping == undefined) {
+                scope.visualization.options.columnMapping = {};
+              }
+
+              scope.columnTypeSelection = scope.visualization.options.columnMapping;
+
+              _.each(scope.columns, function(column) {
+                var definition = column.name.split("::"),
+                    definedColumns = _.keys(scope.visualization.options.columnMapping);
+
+                if (_.indexOf(definedColumns, column.name) != -1) {
+                  // Skip already defined columns.
+                  return;
+                };
+
+                if (definition.length == 1) {
+                  scope.columnTypeSelection[column.name] = scope.visualization.options.columnMapping[column.name] = 'unused';
+                } else if (definition == 'multi-filter') {
+                  scope.columnTypeSelection[column.name] = scope.visualization.options.columnMapping[column.name] = 'series';
+                } else if (_.indexOf(_.values(scope.columnTypes), definition[1]) != -1) {
+                  scope.columnTypeSelection[column.name] = scope.visualization.options.columnMapping[column.name] = definition[1];
+                } else {
+                  scope.columnTypeSelection[column.name] = scope.visualization.options.columnMapping[column.name] = 'unused';
+                }
+              });
+            };
+
+            columnsWatch = scope.$watch('queryResult.getId()', function(id) {
+              if (!id) {
+                return;
+              }
+
+              initColumnMapping();
+              refreshSeries();
+            });
+
+            scope.$watchCollection('columnTypeSelection', function(selections) {
+              _.each(scope.columnTypeSelection, function(type, name) {
+                scope.visualization.options.columnMapping[name] = type;
+              });
+
+              refreshSeries();
+            });
+
             chartOptionsUnwatch = scope.$watch("stacking", function (stacking) {
               if (stacking == "none") {
                 scope.visualization.options.series.stacking = null;
@@ -101,6 +204,8 @@
               }
             });
 
+            scope.xAxisType = (scope.visualization.options.xAxis && scope.visualization.options.xAxis.type) || scope.xAxisType;
+
             xAxisUnwatch = scope.$watch("xAxisType", function (xAxisType) {
               scope.visualization.options.xAxis = scope.visualization.options.xAxis || {};
               scope.visualization.options.xAxis.type = xAxisType;
@@ -111,6 +216,11 @@
               chartOptionsUnwatch = null;
             }
 
+            if (columnsWatch) {
+              columnWatch();
+              columnWatch = null;
+            }
+
             if (xAxisUnwatch) {
               xAxisUnwatch();
               xAxisUnwatch = null;

rd_ui/app/scripts/visualizations/cohort.js

@@ -28,9 +28,13 @@
                     } else {
                         var sortedData = _.sortBy($scope.queryResult.getData(), "date");
                         var grouped = _.groupBy(sortedData, "date");
+                        var maxColumns = _.reduce(grouped, function(memo, data){ 
+                            return (data.length > memo)? data.length : memo;
+                        }, 0);
                         var data = _.map(grouped, function(values, date) {
                            var row = [values[0].total];
                             _.each(values, function(value) { row.push(value.value); });
+                            _.each(_.range(values.length, maxColumns), function() { row.push(null); });
                             return row;
                         });
 

rd_ui/app/scripts/visualizations/table.js

@@ -1,90 +1,109 @@
 (function () {
-    var tableVisualization = angular.module('redash.visualization');
+  var tableVisualization = angular.module('redash.visualization');
 
-    tableVisualization.config(['VisualizationProvider', function(VisualizationProvider) {
-        VisualizationProvider.registerVisualization({
-            type: 'TABLE',
-            name: 'Table',
-            renderTemplate: '<grid-renderer options="visualization.options" query-result="queryResult"></grid-renderer>',
-            skipTypes: true
-        });
-    }]);
+  tableVisualization.config(['VisualizationProvider', function (VisualizationProvider) {
+    VisualizationProvider.registerVisualization({
+      type: 'TABLE',
+      name: 'Table',
+      renderTemplate: '<grid-renderer options="visualization.options" query-result="queryResult"></grid-renderer>',
+      skipTypes: true
+    });
+  }]);
 
-    tableVisualization.directive('gridRenderer', function () {
-        return {
-            restrict: 'E',
-            scope: {
-                queryResult: '=',
-                itemsPerPage: '='
-            },
-            templateUrl: "/views/grid_renderer.html",
-            replace: false,
-            controller: ['$scope', function ($scope) {
-                $scope.gridColumns = [];
-                $scope.gridData = [];
-                $scope.gridConfig = {
-                    isPaginationEnabled: true,
-                    itemsByPage: $scope.itemsPerPage || 15,
-                    maxSize: 8
+  tableVisualization.directive('gridRenderer', function () {
+    return {
+      restrict: 'E',
+      scope: {
+        queryResult: '=',
+        itemsPerPage: '='
+      },
+      templateUrl: "/views/grid_renderer.html",
+      replace: false,
+      controller: ['$scope', '$filter', function ($scope, $filter) {
+        $scope.gridColumns = [];
+        $scope.gridData = [];
+        $scope.gridConfig = {
+          isPaginationEnabled: true,
+          itemsByPage: $scope.itemsPerPage || 15,
+          maxSize: 8
+        };
+
+        $scope.$watch('queryResult && queryResult.getData()', function (data) {
+          if (!data) {
+            return;
+          }
+
+          if ($scope.queryResult.getData() == null) {
+            $scope.gridColumns = [];
+            $scope.gridData = [];
+            $scope.filters = [];
+          } else {
+            $scope.filters = $scope.queryResult.getFilters();
+
+            var prepareGridData = function (data) {
+              var gridData = _.map(data, function (row) {
+                var newRow = {};
+                _.each(row, function (val, key) {
+                  newRow[$scope.queryResult.getColumnCleanName(key)] = val;
+                })
+                return newRow;
+              });
+
+              return gridData;
+            };
+
+            $scope.gridData = prepareGridData($scope.queryResult.getData());
+
+            var columns = $scope.queryResult.getColumns();
+            $scope.gridColumns = _.map($scope.queryResult.getColumnCleanNames(), function (col, i) {
+              var columnDefinition = {
+                'label': $scope.queryResult.getColumnFriendlyNames()[i],
+                'map': col
+              };
+
+              var columnType = columns[i].type;
+
+              if (columnType === 'integer') {
+                columnDefinition.formatFunction = 'number';
+                columnDefinition.formatParameter = 0;
+              } else if (columnType === 'float') {
+                columnDefinition.formatFunction = 'number';
+                columnDefinition.formatParameter = 2;
+              } else if (columnType === 'boolean') {
+                columnDefinition.formatFunction = function (value) {
+                  if (value !== undefined) {
+                    return "" + value;
+                  }
+                  return value;
                 };
+              } else if (columnType === 'date') {
+                columnDefinition.formatFunction = function (value) {
+                  if (value) {
+                    return value.format("DD/MM/YY");
+                  }
+                  return value;
+                };
+              } else if (columnType === 'datetime') {
+                columnDefinition.formatFunction = function (value) {
+                  if (value) {
+                    return value.format("DD/MM/YY HH:mm");
+                  }
+                  return value;
+                };
+              } else {
+                columnDefinition.formatFunction = function (value) {
+                  if (angular.isString(value)) {
+                    value = $filter('linkify')(value);
+                  }
+                  return value;
+                }
+              }
 
-                $scope.$watch('queryResult && queryResult.getData()', function (data) {
-                    if (!data) {
-                        return;
-                    }
-
-                    if ($scope.queryResult.getData() == null) {
-                        $scope.gridColumns = [];
-                        $scope.gridData = [];
-                        $scope.filters = [];
-                    } else {
-                        $scope.filters = $scope.queryResult.getFilters();
-
-                        var prepareGridData = function(data) {
-                            var gridData = _.map(data, function (row) {
-                                var newRow = {};
-                                _.each(row, function (val, key) {
-                                    newRow[$scope.queryResult.getColumnCleanName(key)] = val;
-                                })
-                                return newRow;
-                            });
-
-                            return gridData;
-                        };
-
-                        $scope.gridData = prepareGridData($scope.queryResult.getData());
-
-                        $scope.gridColumns = _.map($scope.queryResult.getColumnCleanNames(), function (col, i) {
-                            var columnDefinition = {
-                                'label': $scope.queryResult.getColumnFriendlyNames()[i],
-                                'map': col
-                            };
-
-                            var rawData = $scope.queryResult.getRawData();
-
-                            if (rawData.length > 0) {
-                                var exampleData = rawData[0][col];
-                                if (angular.isNumber(exampleData)) {
-                                    columnDefinition['formatFunction'] = 'number';
-                                    columnDefinition['formatParameter'] = 2;
-                                } else if (moment.isMoment(exampleData)) {
-                                    columnDefinition['formatFunction'] = function(value) {
-                                        // TODO: this is very hackish way to determine if we need
-                                        // to show the value as a time or date only. Better solution
-                                        // is to complete #70 and use the information it returns.
-                                        if (value._i.match(/^\d{4}-\d{2}-\d{2}T/)) {
-                                            return value.format("DD/MM/YY HH:mm");
-                                        }
-                                        return value.format("DD/MM/YY");
-                                    }
-                                }
-                            }
-
-                            return columnDefinition;
-                        });
-                    }
-                });
-            }]
-        }
-    })
+              return columnDefinition;
+            });
+          }
+        });
+      }]
+    }
+  })
 }());

rd_ui/app/styles/redash.css

@@ -245,6 +245,9 @@ to add those CSS styles here. */
     background-color: #FF8080;
     border-radius: 50%;
 }
+.nav-tabs > li.rd-tab-btn {
+    float: right;
+}
 
 /* light version of bootstrap's form-control */
 .rd-form-control {
@@ -264,10 +267,18 @@ to add those CSS styles here. */
 .rd-form-control {
      width: 100%;
 }
-visualization-renderer > div {
+pivot-table-renderer > table, grid-renderer > div, visualization-renderer > div {
     overflow: auto;
 }
 
+.rd-widget-textbox p {
+    margin-bottom: 0;
+}
+
+.iframe-container {
+    height: 100%;
+}
+
 /*
 bootstrap's hidden-xs class adds display:block when not hidden
 use this class when you need to keep the original display value
@@ -276,4 +287,4 @@ use this class when you need to keep the original display value
     .rd-hidden-xs {
         display: none !important;
     }
-}
+}

rd_ui/app/views/admin_status.html

@@ -21,30 +21,20 @@
                     Started
                 </li>
                 <li class="list-group-item">
-                    <span class="badge">{{manager.queue_size}}</span>
-                    Queue Size
+                    <span class="badge">{{manager.outdated_queries_count}}</span>
+                    Outdated Queries Count
+                </li>
+
+                <li class="list-group-item" ng-if="flowerUrl">
+                    <a href="/admin/workers">Workers' Status</a>
                 </li>
             </ul>
             <ul class="list-group col-lg-4">
-                <div ng-repeat="worker in workers">
-                    <li class="list-group-item active">Worker {{$index+1}}</li>
-                    <li class="list-group-item">
-                    <span class="badge" am-time-ago="worker.updated_at*1000.0"></span>
-                    Updated
+                <li class="list-group-item active">Queues</li>
+                <li class="list-group-item" ng-repeat="(name, value) in manager.queues">
+                    <span class="badge">{{value.size}}</span>
+                    {{name}} ({{value.data_sources}})
                 </li>
-                <li class="list-group-item">
-                    <span class="badge" am-time-ago="worker.started_at*1000.0"></span>
-                    Started
-                </li>
-                <li class="list-group-item">
-                    <span class="badge">{{worker.jobs_count}}</span>
-                    Jobs Received
-                </li>
-                <li class="list-group-item">
-                    <span class="badge">{{worker.done_jobs_count}}</span>
-                    Jobs Done
-                </li>
-                </div>
             </ul>
         </div>
         <div class="panel-footer">Next refresh: <span am-time-ago="refresh_time"></span></div>

rd_ui/app/views/admin_workers.html

@@ -0,0 +1,3 @@
+<div class="container-fluid iframe-container">
+    <iframe src="{{flowerUrl}}" style="width:100%; height:100%; background-color:transparent;"></iframe>
+</div>

rd_ui/app/views/dashboard.html

@@ -14,6 +14,7 @@
                 </button>
             </span>
     </h2>
+    <filters></filters>
 </div>
 
 <div class="container" id="dashboard">
@@ -21,7 +22,7 @@
         <div ng-repeat="widget in row" class="col-lg-{{widget.width | colWidth}}"
              ng-controller='WidgetCtrl'>
 
-            <div class="panel panel-default">
+            <div class="panel panel-default" ng-if="type=='visualization'">
                 <div class="panel-heading">
                     <h3 class="panel-title">
                         <p>
@@ -44,8 +45,28 @@
                         <button type="button" class="btn btn-default btn-xs" ng-show="dashboard.canEdit()" ng-click="deleteWidget()" title="Remove Widget"><span class="glyphicon glyphicon-trash"></span></button>
                     </span>
 
+                    <span class="pull-right">
+                        <a class="btn btn-default btn-xs" ng-disabled="!queryResult.getData()" query-result-link target="_self">
+                            <span class="glyphicon glyphicon-cloud-download"></span>
+                        </a>
+                    </span>
+                </div>
+            </div>
+
+            <div class="panel panel-default rd-widget-textbox" ng-if="type=='textbox'" ng-mouseenter="showControls = true" ng-mouseleave="showControls = false">
+                <div class="panel-body">
+                    <div class="row">
+                        <div class="col-lg-11">
+                            <p ng-bind-html="widget.text | markdown"></p>
+                        </div>
+                        <div class="col-lg-1">
+                          <span class="pull-right" ng-show="showControls">
+                              <button type="button" class="btn btn-default btn-xs" ng-show="dashboard.canEdit()" ng-click="deleteWidget()" title="Remove Widget"><span class="glyphicon glyphicon-trash"></span></button>
+                          </span>
+                        </div>
+                    </div>
                 </div>
             </div>
         </div>
     </div>
-</div>
+</div>

rd_ui/app/views/new_widget_form.html

@@ -6,35 +6,54 @@
                 <h4 class="modal-title">Add Widget</h4>
             </div>
             <div class="modal-body">
-                <p>
-                    <form class="form-inline" role="form" ng-submit="loadVisualizations()">
-                      <div class="form-group">
-                        <input class="form-control" placeholder="Query Id" ng-model="queryId">
-                      </div>
-                      <button type="submit" class="btn btn-primary" ng-disabled="!queryId">
-                        Load visualizations
-                      </button>
-                    </form>
+                <p class="btn-group">
+                    <button type="button" class="btn btn-default" ng-class="{active: isVisualization()}" ng-click="setType('visualization')">Visualization</button>
+                    <button type="button" class="btn btn-default" ng-class="{active: isTextBox()}" ng-click="setType('textbox')">Text Box</button>
                 </p>
 
-                <div ng-show="query">
-                  <div class="form-group">
-                      <label for="">Choose Visualation</label>
-                      <select ng-model="selectedVis" ng-options="vis as vis.name group by vis.type for vis in query.visualizations" class="form-control"></select>
-                  </div>
+                <div ng-show="isTextBox()">
+                    <div class="form-group">
+                        <textarea class="form-control" ng-model="text" rows="3"></textarea>
+                    </div>
+                    <div ng-show="text">
+                        <strong>Preview:</strong>
+                        <p ng-bind-html="text | markdown"></p>
+                    </div>
+                </div>
 
-                  <div class="form-group">
-                      <label for="">Widget Size</label>
-                      <select class="form-control" ng-model="widgetSize" ng-options="c.value as c.name for c in widgetSizes"></select>
-                  </div>
+                <div ng-show="isVisualization()">
+                    <p>
+                        <form class="form-inline" role="form" ng-submit="loadVisualizations()">
+                            <div class="form-group">
+                                <input class="form-control" placeholder="Query Id" ng-model="queryId">
+                            </div>
+                            <button type="submit" class="btn btn-primary" ng-disabled="!queryId">
+                                Load visualizations
+                            </button>
+                        </form>
+                    </p>
+
+                    <div ng-show="query">
+                      <div class="form-group">
+                          <label for="">Choose Visualization</label>
+                          <select ng-model="selectedVis" ng-options="vis as vis.name group by vis.type for vis in query.visualizations" class="form-control"></select>
+                      </div>
+                    </div>
+                </div>
+
+                <div class="form-group">
+                    <label for="">Widget Size</label>
+                    <select class="form-control" ng-model="widgetSize"
+                            ng-options="c.value as c.name for c in widgetSizes"></select>
                 </div>
             </div>
-            <div class="modal-footer" ng-if="selectedVis">
+
+            <div class="modal-footer">
                 <button type="button" class="btn btn-default" ng-disabled="saveInProgress" data-dismiss="modal">Close</button>
-                <button type="button" class="btn btn-primary" ng-disabled="saveInProgress" ng-click="saveWidget()">Add to Dashboard</button>
+                <button type="button" class="btn btn-primary" ng-disabled="saveInProgress || !(selectedVis || isTextBox())" ng-click="saveWidget()">Add to Dashboard</button>
             </div>
         </div>
         <!-- /.modal-content -->
     </div>
     <!-- /.modal-dialog -->
-</div>
+</div>

rd_ui/app/views/query.html

@@ -122,7 +122,7 @@
             <hr>
 
             <p>
-                <a class="btn btn-primary btn-sm" ng-disabled="queryExecuting || !queryResult.getData()" ng-href="{{dataUri}}" download="{{dataFilename}}" target="_self">
+                <a class="btn btn-primary btn-sm" ng-disabled="queryExecuting || !queryResult.getData()" query-result-link target="_self">
                     <span class="glyphicon glyphicon-cloud-download"></span>
                     <span class="rd-hidden-xs">Download Dataset</span>
                 </a>
@@ -142,7 +142,7 @@
             <div class="alert alert-danger" ng-show="queryResult.getError()">Error running query: <strong>{{queryResult.getError()}}</strong></div>
 
             <!-- tabs and data -->
-            <div ng-show="queryResult.getStatus() == 'done'">
+            <div ng-show="showDataset">
                 <div class="row">
                     <div class="col-lg-12">
                         <ul class="nav nav-tabs">
@@ -152,6 +152,7 @@
                                 <span class="remove" ng-click="deleteVisualization($event, vis)" ng-show="canEdit"> &times;</span>
                             </rd-tab>
                             <rd-tab tab-id="add" name="&plus; New" removeable="true" ng-show="canEdit"></rd-tab>
+                            <li ng-if="!sourceMode" class="rd-tab-btn"><button class="btn btn-sm btn-default" ng-click="executeQuery()" ng-disabled="queryExecuting" title="Refresh Dataset"><span class="glyphicon glyphicon-refresh"></span></button></li>
                         </ul>
                     </div>
                 </div>
@@ -170,9 +171,9 @@
                             <edit-visulatization-form visualization="vis" query="query" query-result="queryResult" ng-show="canEdit"></edit-visulatization-form>
                         </div>
 
-                        <div ng-show="selectedTab == 'add'">
+                        <div ng-if="canEdit" ng-show="selectedTab == 'add'">
                             <visualization-renderer visualization="newVisualization" query-result="queryResult"></visualization-renderer>
-                            <edit-visulatization-form visualization="newVisualization" query="query" ng-show="canEdit" on-new-success="setVisualizationTab"></edit-visulatization-form>
+                            <edit-visulatization-form visualization="newVisualization" query="query" query-result="queryResult" ng-show="canEdit" open-editor="true" on-new-success="setVisualizationTab"></edit-visulatization-form>
                         </div>
                     </div>
                 </div>

rd_ui/app/views/visualizations/chart_editor.html

@@ -1,14 +1,91 @@
-<div>
-  <div class="form-group">
-      <label class="control-label">Chart Type</label>
-      <select required ng-model="visualization.options.series.type" ng-options="value as key for (key, value) in seriesTypes" class="form-control"></select>
-  </div>
+<div class="form-horizontal">
+    <div class="panel panel-default">
+        <div class="panel-body">
+            <div class="form-group">
+                <label class="control-label col-sm-2">Stacking</label>
 
-  <div class="form-group">
-      <label class="control-label">Stacking</label>
-      <select required ng-model="stacking" ng-options="value as key for (key, value) in stackingOptions" class="form-control"></select>
+                <div class="col-sm-10">
+                    <select required ng-model="stacking"
+                            ng-options="value as key for (key, value) in stackingOptions"
+                            class="form-control"></select>
+                </div>
+            </div>
+            <div class="form-group">
+                <label class="control-label col-sm-2">X Axis Type</label>
 
-      <label class="control-label">X Axis Type</label>
-      <select required ng-model="xAxisType" ng-options="value as key for (key, value) in xAxisOptions" class="form-control"></select>
-  </div>
+                <div class="col-sm-10">
+                    <select required ng-model="xAxisType" ng-options="value as key for (key, value) in xAxisOptions"
+                            class="form-control"></select>
+                </div>
+            </div>
+        </div>
+    </div>
+
+    <div class="row">
+        <div class="col-lg-6">
+            <div class="list-group">
+                <div class="list-group-item active">
+                    Columns Mapping
+                </div>
+                <div class="list-group-item">
+                    <div class="form-group" ng-repeat="column in columns">
+                        <label class="control-label col-sm-4">{{column.name}}</label>
+
+                        <div class="col-sm-8">
+                            <select ng-options="value as key for (key, value) in columnTypes" class="form-control"
+                                    ng-model="columnTypeSelection[column.name]"></select>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+
+        <div class="col-lg-6" ng-if="series.length > 0">
+            <div class="list-group" ng-repeat="seriesName in series">
+                <div class="list-group-item active">
+                    {{seriesName}}
+                </div>
+                <div class="list-group-item">
+                    <div class="form-group">
+                        <label class="control-label col-sm-3">Type</label>
+
+                        <div class="col-sm-9">
+                            <select required ng-model="visualization.options.seriesOptions[seriesName].type"
+                                    ng-options="value as key for (key, value) in seriesTypes"
+                                    class="form-control"></select>
+                        </div>
+                    </div>
+                    <div class="form-group">
+                        <label class="control-label col-sm-3">zIndex</label>
+
+                        <div class="col-sm-9">
+                            <select required ng-model="visualization.options.seriesOptions[seriesName].zIndex"
+                                    ng-options="o as o for o in zIndexes"
+                                    class="form-control"></select>
+                        </div>
+                    </div>
+                    <div class="form-group">
+                        <label class="control-label col-sm-3">y Axis</label>
+
+                        <div class="col-sm-9">
+                            <select required ng-model="visualization.options.seriesOptions[seriesName].yAxis"
+                                    ng-options="o[0] as o[1] for o in yAxes"
+                                    class="form-control"></select>
+                        </div>
+                    </div>
+                    <div class="form-group">
+                        <label class="control-label col-sm-3">Name</label>
+
+                        <div class="col-sm-9">
+                            <input name="seriesName" type="text" class="form-control"
+                                   ng-model="visualization.options.seriesOptions[seriesName].name"
+                                   placeholder="{{seriesName}}">
+                        </div>
+                    </div>
+                </div>
+            </div>
+
+
+        </div>
+    </div>
 </div>

rd_ui/app/views/visualizations/edit_visualization.html

@@ -1,7 +1,7 @@
 <div>
-  <span ng-click="visEdit=!visEdit" class="details-toggle" ng-class="{open: visEdit}">Edit</span>
+  <span ng-click="openEditor=!openEditor" class="details-toggle" ng-class="{open: openEditor}">Edit</span>
 
-  <form ng-if="visEdit" role="form" name="visForm" ng-submit="submit()">
+  <form ng-if="openEditor" role="form" name="visForm" ng-submit="submit()">
     <div class="form-group">
       <label class="control-label">Name</label>
       <input name="name" type="text" class="form-control" ng-model="visualization.name" placeholder="{{visualization.type | capitalize}}">

rd_ui/app/views/visualizations/filters.html

@@ -1,12 +1,8 @@
 <div class="well well-sm" ng-show="filters">
-    <div class="btn-group" ng-repeat="filter in filters">
-        <button type="button" class="btn btn-default dropdown-toggle" data-toggle="dropdown">
-            {{filter.friendlyName}}: {{filter.current}}<span class="caret"></span>
-        </button>
-        <ul class="dropdown-menu" role="menu">
-            <li ng-repeat="value in filter.values">
-                <a href="" ng-click="filter.current = value">{{value}}</a>
-            </li>
-        </ul>
+    <div ng-repeat="filter in filters">
+        {{filter.friendlyName}}:
+        <select ui-select2='select2Options' ng-model="filter.current" ng-multiple="{{filter.multiple}}">
+            <option ng-repeat="value in filter.values" value="{{value}}">{{value}}</option>
+        </select>
     </div>
 </div>

rd_ui/bower.json

@@ -11,15 +11,21 @@
     "moment": "2.1.0",
     "angular-ui-bootstrap": "0.5.0",
     "angular-ui-codemirror": "0.0.5",
-    "highcharts": "3.0.1",
+    "highcharts": "3.0.10",
     "underscore": "1.5.1",
     "angular-resource": "1.2.15",
     "angular-growl": "0.3.1",
     "angular-route": "1.2.7",
-    "pivottable": "https://github.com/arikfr/pivottable.git",
+    "pivottable": "~1.1.1",
     "cornelius": "https://github.com/restorando/cornelius.git",
     "gridster": "0.2.0",
-    "mousetrap": "~1.4.6"
+    "mousetrap": "~1.4.6",
+    "angular-ui-select2": "~0.0.5",
+    "jquery-ui": "~1.10.4",
+    "underscore.string": "~2.3.3",
+    "marked": "~0.3.2",
+    "bucky": "~0.2.6",
+    "pace": "~0.5.1"
   },
   "devDependencies": {
     "angular-mocks": "~1.0.7",

rd_ui/karma-e2e.conf.js

@@ -1,58 +0,0 @@
-// Karma E2E configuration
-
-// base path, that will be used to resolve files and exclude
-basePath = '';
-
-// list of files / patterns to load in the browser
-files = [
-  ANGULAR_SCENARIO,
-  ANGULAR_SCENARIO_ADAPTER,
-  'test/e2e/**/*.js'
-];
-
-// list of files to exclude
-exclude = [];
-
-// test results reporter to use
-// possible values: dots || progress || growl
-reporters = ['progress'];
-
-// web server port
-port = 8080;
-
-// cli runner port
-runnerPort = 9100;
-
-// enable / disable colors in the output (reporters and logs)
-colors = true;
-
-// level of logging
-// possible values: LOG_DISABLE || LOG_ERROR || LOG_WARN || LOG_INFO || LOG_DEBUG
-logLevel = LOG_INFO;
-
-// enable / disable watching file and executing tests whenever any file changes
-autoWatch = false;
-
-// Start these browsers, currently available:
-// - Chrome
-// - ChromeCanary
-// - Firefox
-// - Opera
-// - Safari (only Mac)
-// - PhantomJS
-// - IE (only Windows)
-browsers = ['Chrome'];
-
-// If browser does not capture in given timeout [ms], kill it
-captureTimeout = 5000;
-
-// Continuous Integration mode
-// if true, it capture browsers, run tests and exit
-singleRun = false;
-
-// Uncomment the following lines if you are using grunt's server to run the tests
-// proxies = {
-//   '/': 'http://localhost:9000/'
-// };
-// URL root prevent conflicts with the site root
-// urlRoot = '_karma_';

rd_ui/karma.conf.js

@@ -1,56 +0,0 @@
-// Karma configuration
-
-// base path, that will be used to resolve files and exclude
-basePath = '';
-
-// list of files / patterns to load in the browser
-files = [
-  JASMINE,
-  JASMINE_ADAPTER,
-  'app/bower_components/angular/angular.js',
-  'app/bower_components/angular-mocks/angular-mocks.js',
-  'app/scripts/*.js',
-  'app/scripts/**/*.js',
-  'test/mock/**/*.js',
-  'test/spec/**/*.js'
-];
-
-// list of files to exclude
-exclude = [];
-
-// test results reporter to use
-// possible values: dots || progress || growl
-reporters = ['progress'];
-
-// web server port
-port = 8080;
-
-// cli runner port
-runnerPort = 9100;
-
-// enable / disable colors in the output (reporters and logs)
-colors = true;
-
-// level of logging
-// possible values: LOG_DISABLE || LOG_ERROR || LOG_WARN || LOG_INFO || LOG_DEBUG
-logLevel = LOG_INFO;
-
-// enable / disable watching file and executing tests whenever any file changes
-autoWatch = false;
-
-// Start these browsers, currently available:
-// - Chrome
-// - ChromeCanary
-// - Firefox
-// - Opera
-// - Safari (only Mac)
-// - PhantomJS
-// - IE (only Windows)
-browsers = ['Chrome'];
-
-// If browser does not capture in given timeout [ms], kill it
-captureTimeout = 5000;
-
-// Continuous Integration mode
-// if true, it capture browsers, run tests and exit
-singleRun = false;

rd_ui/package.json

@@ -1,38 +1,39 @@
 {
-  "name": "rd-ui",
-  "version": "0.1.0",
-  "dependencies": {
-  },
+  "name": "rdui",
+  "version": "0.0.0",
+  "dependencies": {},
   "devDependencies": {
-    "grunt": "git+https://github.com/gruntjs/grunt.git#08a3af5",
-    "grunt-contrib-copy": "~0.4.1",
-    "grunt-contrib-concat": "~0.3.0",
-    "grunt-contrib-coffee": "~0.7.0",
-    "grunt-contrib-uglify": "~0.2.0",
-    "grunt-contrib-compass": "~0.5.0",
-    "grunt-contrib-jshint": "~0.6.0",
-    "grunt-contrib-cssmin": "~0.6.0",
-    "grunt-contrib-connect": "~0.3.0",
-    "grunt-contrib-clean": "~0.5.0",
-    "grunt-contrib-htmlmin": "~0.1.3",
-    "grunt-contrib-imagemin": "~0.2.0",
-    "grunt-contrib-watch": "~0.5.2",
-    "grunt-autoprefixer": "~0.2.0",
-    "grunt-usemin": "~0.1.11",
-    "grunt-svgmin": "~0.2.0",
-    "grunt-rev": "~0.1.0",
-    "grunt-open": "~0.2.0",
-    "grunt-concurrent": "~0.3.0",
-    "load-grunt-tasks": "~0.1.0",
-    "connect-livereload": "~0.2.0",
-    "grunt-google-cdn": "~0.2.0",
-    "grunt-ngmin": "~0.0.2",
-    "time-grunt": "~0.1.0",
-    "bower": "~1.2.7",
-    "grunt-cli": "~0.1.9"
+    "grunt": "^0.4.1",
+    "grunt-autoprefixer": "^0.7.3",
+    "grunt-concurrent": "^0.5.0",
+    "grunt-contrib-clean": "^0.5.0",
+    "grunt-contrib-concat": "^0.4.0",
+    "grunt-contrib-connect": "^0.7.1",
+    "grunt-contrib-copy": "^0.5.0",
+    "grunt-contrib-cssmin": "^0.9.0",
+    "grunt-contrib-htmlmin": "^0.3.0",
+    "grunt-contrib-imagemin": "^0.7.0",
+    "grunt-contrib-jshint": "^0.10.0",
+    "grunt-contrib-uglify": "^0.4.0",
+    "grunt-contrib-watch": "^0.6.1",
+    "grunt-filerev": "^0.2.1",
+    "grunt-google-cdn": "^0.4.0",
+    "grunt-newer": "^0.7.0",
+    "grunt-ngmin": "^0.0.3",
+    "grunt-svgmin": "^0.4.0",
+    "grunt-usemin": "^2.1.1",
+    "grunt-wiredep": "^1.7.0",
+    "jshint-stylish": "^0.2.0",
+    "load-grunt-tasks": "^0.4.0",
+    "time-grunt": "^0.3.1",
+    "karma-jasmine": "~0.1.5",
+    "grunt-karma": "~0.8.3",
+    "karma-phantomjs-launcher": "~0.1.4",
+    "karma": "~0.12.19",
+    "karma-ng-html2js-preprocessor": "~0.1.0"
   },
   "engines": {
-    "node": ">=0.8.0"
+    "node": ">=0.10.0"
   },
   "scripts": {
     "test": "grunt test"

rd_ui/test/.jshintrc

@@ -29,6 +29,7 @@
     "expect": false,
     "inject": false,
     "it": false,
+    "jasmine": false,
     "spyOn": false
   }
 }

rd_ui/test/karma.conf.js

@@ -0,0 +1,131 @@
+// Karma configuration
+// http://karma-runner.github.io/0.12/config/configuration-file.html
+// Generated on 2014-07-30 using
+// generator-karma 0.8.3
+
+module.exports = function(config) {
+  'use strict';
+
+  config.set({
+    // enable / disable watching file and executing tests whenever any file changes
+    autoWatch: true,
+
+    // base path, that will be used to resolve files and exclude
+    basePath: '../',
+
+    // testing framework to use (jasmine/mocha/qunit/...)
+    frameworks: ['jasmine'],
+
+    // list of files / patterns to load in the browser
+    files: [
+      'app/bower_components/jquery/jquery.js',
+      'app/bower_components/jquery-ui/ui/jquery-ui.js',
+
+      'app/bower_components/angular/angular.js',
+      'app/bower_components/angular-route/angular-route.js',
+      'app/bower_components/angular-mocks/angular-mocks.js',
+
+      'app/bower_components/bootstrap/js/collapse.js',
+      'app/bower_components/bootstrap/js/modal.js',
+      'app/bower_components/angular-resource/angular-resource.js',
+      'app/bower_components/underscore/underscore.js',
+      'app/bower_components/moment/moment.js',
+      'app/bower_components/angular-moment/angular-moment.js',
+      'app/bower_components/codemirror/lib/codemirror.js',
+      'app/bower_components/codemirror/addon/edit/matchbrackets.js',
+      'app/bower_components/codemirror/addon/edit/closebrackets.js',
+      'app/bower_components/codemirror/mode/sql/sql.js',
+      'app/bower_components/codemirror/mode/javascript/javascript.js',
+      'app/bower_components/angular-ui-codemirror/ui-codemirror.js',
+      'app/bower_components/highcharts/highcharts.js',
+      'app/bower_components/highcharts/modules/exporting.js',
+      'app/bower_components/gridster/dist/jquery.gridster.js',
+      'app/bower_components/angular-growl/build/angular-growl.js',
+      'app/bower_components/pivottable/dist/pivot.js',
+      'app/bower_components/cornelius/src/cornelius.js',
+      'app/bower_components/mousetrap/mousetrap.js',
+      'app/bower_components/mousetrap/plugins/global-bind/mousetrap-global-bind.js',
+      'app/bower_components/select2/select2.js',
+      'app/bower_components/angular-ui-select2/src/select2.js',
+      'app/bower_components/underscore.string/lib/underscore.string.js',
+      'app/bower_components/marked/lib/marked.js',
+      'app/scripts/ng_highchart.js',
+      'app/scripts/ng_smart_table.js',
+      'app/scripts/ui-bootstrap-tpls-0.5.0.min.js',
+      'app/bower_components/bucky/bucky.js',
+      'app/bower_components/pace/pace.js',
+
+      'app/scripts/app.js',
+      'app/scripts/services/services.js',
+      'app/scripts/services/resources.js',
+      'app/scripts/services/notifications.js',
+      'app/scripts/services/dashboards.js',
+      'app/scripts/controllers/controllers.js',
+      'app/scripts/controllers/dashboard.js',
+      'app/scripts/controllers/admin_controllers.js',
+      'app/scripts/controllers/query_view.js',
+      'app/scripts/controllers/query_source.js',
+      'app/scripts/visualizations/base.js',
+      'app/scripts/visualizations/chart.js',
+      'app/scripts/visualizations/cohort.js',
+      'app/scripts/visualizations/table.js',
+      'app/scripts/visualizations/pivot.js',
+      'app/scripts/directives/directives.js',
+      'app/scripts/directives/query_directives.js',
+      'app/scripts/directives/dashboard_directives.js',
+      'app/scripts/filters.js',
+
+      'app/views/**/*.html',
+
+      'test/mocks/*.js',
+      'test/unit/*.js'
+    ],
+
+    // generate js files from html templates
+    preprocessors: {
+      'app/views/**/*.html': 'ng-html2js'
+    },
+
+    // list of files / patterns to exclude
+    exclude: [],
+
+    // web server port
+    port: 8080,
+
+    // Start these browsers, currently available:
+    // - Chrome
+    // - ChromeCanary
+    // - Firefox
+    // - Opera
+    // - Safari (only Mac)
+    // - PhantomJS
+    // - IE (only Windows)
+    browsers: [
+      'PhantomJS'
+    ],
+
+    // Which plugins to enable
+    plugins: [
+      'karma-phantomjs-launcher',
+      'karma-jasmine',
+      'karma-ng-html2js-preprocessor'
+    ],
+
+    // Continuous Integration mode
+    // if true, it capture browsers, run tests and exit
+    singleRun: false,
+
+    colors: true,
+
+    // level of logging
+    // possible values: LOG_DISABLE || LOG_ERROR || LOG_WARN || LOG_INFO || LOG_DEBUG
+    logLevel: config.LOG_INFO,
+
+    // Uncomment the following lines if you are using grunt's server to run the tests
+    // proxies: {
+    //   '/': 'http://localhost:9000/'
+    // },
+    // URL root prevent conflicts with the site root
+    // urlRoot: '_karma_'
+  });
+};

rd_ui/test/mocks/redash_mocks.js

@@ -0,0 +1,108 @@
+featureFlags = [];
+currentUser = {
+  id: 1,
+  name: 'John Mock',
+  email: 'john@example.com',
+  groups: ['default'],
+  permissions: [],
+  canEdit: function(object) {
+    var user_id = object.user_id || (object.user && object.user.id);
+    return user_id && (user_id == currentUser.id);
+  },
+  hasPermission: function(permission) {
+    return this.permissions.indexOf(permission) != -1;
+  }
+};
+
+
+angular.module('redashMocks', [])
+  .value('MockData', {
+    query: {
+      "ttl": -1,
+      "query": "select name from users;",
+      "id": 1803,
+      "description": "",
+      "name": "my test query",
+      "created_at": "2014-01-07T16:11:31.859528+02:00",
+      "query_hash": "c89c235bc73e462e9702debc56adc309",
+
+      "user": {
+        "email": "amirn@everything.me",
+        "id": 48,
+        "name": "Amir Nissim"
+      },
+
+      "visualizations": [{
+        "description": "",
+        "options": {},
+        "type": "TABLE",
+        "id": 636,
+        "name": "Table"
+      }],
+
+      "api_key": "123456789",
+
+      "data_source_id": 1,
+
+      "latest_query_data_id": 106632,
+
+      "latest_query_data": {
+        "retrieved_at": "2014-07-29T10:49:10.951364+03:00",
+        "query_hash": "c89c235bc73e462e9702debc56adc309",
+        "query": "select name from users;",
+        "runtime": 0.0139260292053223,
+        "data": {
+          "rows": [{
+            "name": "Amir Nissim"
+          }, {
+            "name": "Arik Fraimovich"
+          }],
+          "columns": [{
+            "friendly_name": "name",
+            "type": null,
+            "name": "name"
+          }, {
+            "friendly_name": "mail::filter",
+            "type": null,
+            "name": "mail::filter"
+          }]
+        },
+        "id": 106632,
+        "data_source_id": 1
+      }
+
+    },
+
+    queryResult: {
+      "job": {},
+      "query_result": {
+        "retrieved_at": "2014-08-04T13:33:45.563486+03:00",
+        "query_hash": "9951c38c9cf00e6ee8aecce026b51c19",
+        "query": "select name as \"name::filter\" from users",
+        "runtime": 0.00896096229553223,
+        "data": {
+          "rows": [],
+          "columns": [{
+            "friendly_name": "name::filter",
+            "type": null,
+            "name": "name::filter"
+          }]
+        },
+        "id": 106673,
+        "data_source_id": 1
+      },
+      "status": "done",
+      "filters": [],
+      "filterFreeze": "test@example.com",
+      "updatedAt": "2014-08-05T13:13:40.833Z",
+      "columnNames": ["name::filter"],
+      "filteredData": [{
+        "name::filter": "test@example.com"
+      }],
+      "columns": [{
+        "friendly_name": "name::filter",
+        "type": null,
+        "name": "name::filter"
+      }]
+    }
+  });

rd_ui/test/unit/example_test.js

@@ -0,0 +1,5 @@
+describe('example test', function() {
+  it('should expect the obvious', function() {
+    expect(0).toBe(0);
+  });
+});

rd_ui/test/unit/test_query_view.js

@@ -0,0 +1,34 @@
+'use strict';
+
+describe('QueryViewCtrl', function() {
+  var scope;
+  var MockData;
+
+  beforeEach(module('redash', 'redashMocks'));
+
+  beforeEach(inject(function($injector, $controller, $rootScope, Query, _MockData_) {
+    MockData = _MockData_;
+    scope = $rootScope.$new();
+
+    var route = {
+      current: {
+        locals: {
+          query: new Query(MockData.query)
+        }
+      }
+    };
+
+    $controller('QueryViewCtrl', {$scope: scope, $route: route});
+  }));
+
+  it('should have a query', function() {
+    expect(scope.query).toBeDefined();
+  });
+
+  it('should update the executing state', function() {
+    expect(scope.queryExecuting).toBe(false);
+    scope.executeQuery();
+    expect(scope.queryExecuting).toBe(true);
+  });
+
+});

rd_ui/test/unit/test_visualization_renderer.js

@@ -0,0 +1,89 @@
+'use strict';
+
+describe('VisualizationRenderer', function() {
+  var element;
+  var scope;
+
+  var filters = [{
+    "name": "name::filter",
+    "friendlyName": "Name",
+    "values": ["test@example.com", "amirn@example.com"],
+    "multiple": false
+  }];
+
+  beforeEach(module('redash', 'redashMocks'));
+
+  // loading templates
+  beforeEach(module('app/views/grid_renderer.html',
+    'app/views/visualizations/filters.html'));
+
+  // serving templates
+  beforeEach(inject(function($httpBackend, $templateCache) {
+    $httpBackend.whenGET('/views/grid_renderer.html')
+      .respond($templateCache.get('app/views/grid_renderer.html'));
+
+    $httpBackend.whenGET('/views/visualizations/filters.html')
+      .respond($templateCache.get('app/views/visualizations/filters.html'));
+  }));
+
+  // directive setup
+  beforeEach(inject(function($rootScope, $compile, MockData, QueryResult) {
+    var qr = new QueryResult(MockData.queryResult)
+    qr.filters = filters;
+
+    $rootScope.queryResult = qr;
+
+    element = angular.element(
+      '<visualization-renderer query-result="queryResult">' +
+      '</visualization-renderer>');
+  }));
+
+
+  describe('scope', function() {
+    beforeEach(inject(function($rootScope, $compile) {
+      $compile(element)($rootScope);
+
+      // we will test the isolated scope of the directive
+      scope = element.isolateScope();
+      scope.$digest();
+    }));
+
+    it('should have filters', function() {
+      expect(scope.filters).toBeDefined();
+    });
+  });
+
+
+  describe('URL binding', function() {
+
+    beforeEach(inject(function($rootScope, $compile, $location) {
+      spyOn($location, 'search').andCallThrough();
+
+      // set initial search
+      var initialSearch = {};
+      initialSearch[filters[0].friendlyName] = filters[0].values[0];
+      $location.search('filters', initialSearch);
+
+      $compile(element)($rootScope);
+
+      // we will test the isolated scope of the directive
+      scope = element.isolateScope();
+      scope.$digest();
+    }));
+
+    it('should update scope from URL',
+      inject(function($location) {
+        expect($location.search).toHaveBeenCalled();
+        expect(scope.filters[0].current).toEqual(filters[0].values[0]);
+      }));
+
+    it('should update URL from scope',
+      inject(function($location) {
+        scope.filters[0].current = 'newValue';
+        scope.$digest();
+
+        var searchFilters = angular.fromJson($location.search().filters);
+        expect(searchFilters[filters[0].friendlyName]).toEqual('newValue');
+    }));
+  });
+});

redash/__init__.py

@@ -1,51 +1,35 @@
-import json
-import urlparse
 import logging
-from flask import Flask, make_response
-from flask.ext.restful import Api
-from flask_peewee.db import Database
-
+import urlparse
 import redis
 from statsd import StatsClient
-from redash import settings, utils
 
-__version__ = '0.3.5'
+from redash import settings, events
 
-logging.getLogger().addHandler(logging.StreamHandler())
-logging.getLogger().setLevel(settings.LOG_LEVEL)
-
-app = Flask(__name__,
-            template_folder=settings.STATIC_ASSETS_PATH,
-            static_folder=settings.STATIC_ASSETS_PATH,
-            static_path='/static')
-
-api = Api(app)
-
-# configure our database
-settings.DATABASE_CONFIG.update({'threadlocals': True})
-app.config['DATABASE'] = settings.DATABASE_CONFIG
-db = Database(app)
-
-from redash.authentication import setup_authentication
-auth = setup_authentication(app)
-
-@api.representation('application/json')
-def json_representation(data, code, headers=None):
-    resp = make_response(json.dumps(data, cls=utils.JSONEncoder), code)
-    resp.headers.extend(headers or {})
-    return resp
+__version__ = '0.4.0'
 
 
-redis_url = urlparse.urlparse(settings.REDIS_URL)
-if redis_url.path:
-    redis_db = redis_url.path[1]
-else:
-    redis_db = 0
+def setup_logging():
+    handler = logging.StreamHandler()
+    formatter = logging.Formatter('[%(asctime)s][PID:%(process)d][%(levelname)s][%(name)s] %(message)s')
+    handler.setFormatter(formatter)
+    logging.getLogger().addHandler(handler)
+    logging.getLogger().setLevel(settings.LOG_LEVEL)
 
-redis_connection = redis.StrictRedis(host=redis_url.hostname, port=redis_url.port, db=redis_db, password=redis_url.password)
-statsd_client = StatsClient(host=settings.STATSD_HOST, port=settings.STATSD_PORT, prefix=settings.STATSD_PREFIX)
+    events.setup_logging(settings.EVENTS_LOG_PATH, settings.EVENTS_CONSOLE_OUTPUT)
 
-from redash import data
-data_manager = data.Manager(redis_connection, statsd_client)
 
-from redash import controllers
+def create_redis_connection():
+    redis_url = urlparse.urlparse(settings.REDIS_URL)
+    if redis_url.path:
+        redis_db = redis_url.path[1]
+    else:
+        redis_db = 0
+
+    r = redis.StrictRedis(host=redis_url.hostname, port=redis_url.port, db=redis_db, password=redis_url.password)
+
+    return r
+
+
+setup_logging()
+redis_connection = create_redis_connection()
+statsd_client = StatsClient(host=settings.STATSD_HOST, port=settings.STATSD_PORT, prefix=settings.STATSD_PREFIX)

redash/authentication.py

@@ -5,14 +5,12 @@ import time
 import logging
 
 from flask import request, make_response, redirect, url_for
-from flask.ext.googleauth import GoogleAuth, login
 from flask.ext.login import LoginManager, login_user, current_user
+from flask.ext.googleauth import GoogleAuth, login
 from werkzeug.contrib.fixers import ProxyFix
 
-from models import AnonymousUser
 from redash import models, settings
 
-
 login_manager = LoginManager()
 logger = logging.getLogger('authentication')
 
@@ -78,8 +76,7 @@ def create_and_login_user(app, user):
             user_object.save()
     except models.User.DoesNotExist:
         logger.debug("Creating user object (%r)", user.name)
-        user_object = models.User.create(name=user.name, email=user.email,
-                                  is_admin=(user.email in settings.ADMINS))
+        user_object = models.User.create(name=user.name, email=user.email, groups = models.User.DEFAULT_GROUPS)
 
     login_user(user_object, remember=True)
 
@@ -100,7 +97,7 @@ def setup_authentication(app):
             openid_auth._OPENID_ENDPOINT = "https://www.google.com/a/%s/o8/ud?be=o8" % settings.GOOGLE_APPS_DOMAIN
 
     login_manager.init_app(app)
-    login_manager.anonymous_user = AnonymousUser
+    login_manager.anonymous_user = models.AnonymousUser
     app.wsgi_app = ProxyFix(app.wsgi_app)
     app.secret_key = settings.COOKIE_SECRET
 

redash/cache.py

@@ -0,0 +1,8 @@
+from flask import make_response
+from functools import update_wrapper
+
+ONE_YEAR = 60 * 60 * 24 * 365.25
+
+headers = {
+  'Cache-Control': 'max-age=%d' % ONE_YEAR
+}

redash/controllers.py

@@ -17,13 +17,16 @@ from flask.ext.restful import Resource, abort
 from flask_login import current_user, login_user, logout_user
 
 import sqlparse
+import events
 from permissions import require_permission
-from redash import settings, utils
-from redash import data
 
-from redash import app, auth, api, redis_connection, data_manager
-from redash import models
+from redash import redis_connection, statsd_client, models, settings, utils, __version__
+from redash.wsgi import app, auth, api
 
+import logging
+from tasks import QueryTask
+
+from cache import headers as cache_headers
 
 @app.route('/ping', methods=['GET'])
 def ping():
@@ -43,14 +46,20 @@ def index(**kwargs):
 
     user = {
         'gravatar_url': gravatar_url,
-        'is_admin': current_user.is_admin,
         'id': current_user.id,
         'name': current_user.name,
         'email': current_user.email,
+        'groups': current_user.groups,
         'permissions': current_user.permissions
     }
 
+    features = {
+        'clientSideMetrics': settings.CLIENT_SIDE_METRICS,
+        'flowerUrl': settings.CELERY_FLOWER_URL
+    }
+
     return render_template("index.html", user=json.dumps(user), name=settings.NAME,
+                           features=json.dumps(features),
                            analytics=settings.ANALYTICS)
 
 
@@ -92,18 +101,31 @@ def status_api():
     status = {}
     info = redis_connection.info()
     status['redis_used_memory'] = info['used_memory_human']
-
+    status['version'] = __version__
     status['queries_count'] = models.Query.select().count()
     status['query_results_count'] = models.QueryResult.select().count()
     status['dashboards_count'] = models.Dashboard.select().count()
     status['widgets_count'] = models.Widget.select().count()
 
-    status['workers'] = [redis_connection.hgetall(w)
-                         for w in redis_connection.smembers('workers')]
+    status['workers'] = []
 
-    manager_status = redis_connection.hgetall('manager:status')
+    manager_status = redis_connection.hgetall('redash:status')
     status['manager'] = manager_status
-    status['manager']['queue_size'] = redis_connection.zcard('jobs')
+    status['manager']['queue_size'] = redis_connection.llen('queries') + redis_connection.llen('scheduled_queries')
+    status['manager']['outdated_queries_count'] = models.Query.outdated_queries().count()
+
+    queues = {}
+    for ds in models.DataSource.select():
+        for queue in (ds.queue_name, ds.scheduled_queue_name):
+            queues.setdefault(queue, set())
+            queues[queue].add(ds.name)
+
+    status['manager']['queues'] = {}
+    for queue, sources in queues.iteritems():
+        status['manager']['queues'][queue] = {
+            'data_sources': ', '.join(sources),
+            'size': redis_connection.llen(queue)
+        }
 
     return jsonify(status)
 
@@ -128,10 +150,36 @@ class BaseResource(Resource):
     def current_user(self):
         return current_user._get_current_object()
 
+    def dispatch_request(self, *args, **kwargs):
+        with statsd_client.timer('requests.{}.{}'.format(request.endpoint, request.method.lower())):
+            response = super(BaseResource, self).dispatch_request(*args, **kwargs)
+        return response
+
+
+class EventAPI(BaseResource):
+    def post(self):
+        events_list = request.get_json(force=True)
+        for event in events_list:
+            events.record_event(event)
+
+
+api.add_resource(EventAPI, '/api/events', endpoint='events')
+
+
+class MetricsAPI(BaseResource):
+    def post(self):
+        for stat_line in request.data.split():
+            stat, value = stat_line.split(':')
+            statsd_client._send_stat('client.{}'.format(stat), value, 1)
+
+        return "OK."
+
+api.add_resource(MetricsAPI, '/api/metrics/v1/send', endpoint='metrics')
+
 
 class DataSourceListAPI(BaseResource):
     def get(self):
-        data_sources = [ds.to_dict() for ds in models.DataSource.select()]
+        data_sources = [ds.to_dict() for ds in models.DataSource.all()]
         return data_sources
 
 api.add_resource(DataSourceListAPI, '/api/data_sources', endpoint='data_sources')
@@ -247,11 +295,11 @@ class QueryListAPI(BaseResource):
 
         query.create_default_visualizations()
 
-        return query.to_dict(with_result=False)
+        return query.to_dict()
 
     @require_permission('view_query')
     def get(self):
-        return [q.to_dict(with_result=False, with_stats=True) for q in models.Query.all_queries()]
+        return [q.to_dict(with_stats=True) for q in models.Query.all_queries()]
 
 
 class QueryAPI(BaseResource):
@@ -271,7 +319,7 @@ class QueryAPI(BaseResource):
 
         query = models.Query.get_by_id(query_id)
 
-        return query.to_dict(with_result=False, with_visualizations=True)
+        return query.to_dict(with_visualizations=True)
 
     @require_permission('view_query')
     def get(self, query_id):
@@ -305,6 +353,7 @@ class VisualizationAPI(BaseResource):
         if 'options' in kwargs:
             kwargs['options'] = json.dumps(kwargs['options'])
         kwargs.pop('id', None)
+        kwargs.pop('query_id', None)
 
         update = models.Visualization.update(**kwargs).where(models.Visualization.id == visualization_id)
         update.execute()
@@ -327,6 +376,24 @@ class QueryResultListAPI(BaseResource):
     def post(self):
         params = request.json
 
+        if settings.FEATURE_TABLES_PERMISSIONS:
+            metadata = utils.SQLMetaData(params['query'])
+
+            if metadata.has_non_select_dml_statements or metadata.has_ddl_statements:
+                return {
+                    'job': {
+                        'error': 'Only SELECT statements are allowed'
+                    }
+                }
+
+            if len(metadata.used_tables - current_user.allowed_tables) > 0 and '*' not in current_user.allowed_tables:
+                logging.warning('Permission denied for user %s to table %s', self.current_user.name, metadata.used_tables)
+                return {
+                    'job': {
+                        'error': 'Access denied for table(s): %s' % (metadata.used_tables)
+                    }
+                }
+
         models.ActivityLog(
             user=self.current_user,
             type=models.ActivityLog.QUERY_EXECUTION,
@@ -342,7 +409,7 @@ class QueryResultListAPI(BaseResource):
             return {'query_result': query_result.to_dict()}
         else:
             data_source = models.DataSource.get_by_id(params['data_source_id'])
-            job = data_manager.add_job(params['query'], data.Job.HIGH_PRIORITY, data_source)
+            job = QueryTask.add_task(params['query'], data_source)
             return {'job': job.to_dict()}
 
 
@@ -351,7 +418,8 @@ class QueryResultAPI(BaseResource):
     def get(self, query_result_id):
         query_result = models.QueryResult.get_by_id(query_result_id)
         if query_result:
-            return {'query_result': query_result.to_dict()}
+            data = json.dumps({'query_result': query_result.to_dict()}, cls=utils.JSONEncoder)
+            return make_response(data, 200, cache_headers)
         else:
             abort(404)
 
@@ -393,11 +461,11 @@ api.add_resource(QueryResultAPI, '/api/query_results/<query_result_id>', endpoin
 class JobAPI(BaseResource):
     def get(self, job_id):
         # TODO: if finished, include the query result
-        job = data.Job.load(data_manager.redis_connection, job_id)
+        job = QueryTask(job_id=job_id)
         return {'job': job.to_dict()}
 
     def delete(self, job_id):
-        job = data.Job.load(data_manager.redis_connection, job_id)
+        job = QueryTask(job_id=job_id)
         job.cancel()
 
 api.add_resource(JobAPI, '/api/jobs/<job_id>', endpoint='job')

redash/data/__init__.py

@@ -1,2 +0,0 @@
-from manager import Manager
-from worker import Job

redash/data/manager.py

@@ -1,156 +0,0 @@
-"""
-Data manager. Used to manage and coordinate execution of queries.
-"""
-import time
-import logging
-import peewee
-import qr
-import redis
-import json
-from redash import models
-from redash.data import worker
-from redash.utils import gen_query_hash
-
-
-class JSONPriorityQueue(qr.PriorityQueue):
-    """ Use a JSON serializer to help with cross language support """
-    def __init__(self, key, **kwargs):
-        super(qr.PriorityQueue, self).__init__(key, **kwargs)
-        self.serializer = json
-
-
-class Manager(object):
-    def __init__(self, redis_connection, statsd_client):
-        self.statsd_client = statsd_client
-        self.redis_connection = redis_connection
-        self.workers = []
-        self.queue = JSONPriorityQueue("jobs", **self.redis_connection.connection_pool.connection_kwargs)
-        self.max_retries = 5
-        self.status = {
-            'last_refresh_at': 0,
-            'started_at': time.time()
-        }
-
-        self._save_status()
-
-    def add_job(self, query, priority, data_source):
-        query_hash = gen_query_hash(query)
-        logging.info("[Manager][%s] Inserting job with priority=%s", query_hash, priority)
-        try_count = 0
-        job = None
-
-        while try_count < self.max_retries:
-            try_count += 1
-
-            pipe = self.redis_connection.pipeline()
-            try:
-                pipe.watch('query_hash_job:%s' % query_hash)
-                job_id = pipe.get('query_hash_job:%s' % query_hash)
-                if job_id:
-                    logging.info("[Manager][%s] Found existing job: %s", query_hash, job_id)
-                    job = worker.Job.load(self.redis_connection, job_id)
-                else:
-                    job = worker.Job(self.redis_connection, query=query, priority=priority,
-                                     data_source_id=data_source.id,
-                                     data_source_name=data_source.name,
-                                     data_source_type=data_source.type,
-                                     data_source_options=data_source.options)
-                    pipe.multi()
-                    job.save(pipe)
-                    logging.info("[Manager][%s] Created new job: %s", query_hash, job.id)
-                    self.queue.push(job.id, job.priority)
-                break
-
-            except redis.WatchError:
-                continue
-
-        if not job:
-            logging.error("[Manager][%s] Failed adding job for query.", query_hash)
-
-        return job
-
-    def report_status(self):
-        workers = [self.redis_connection.hgetall(w)
-                   for w in self.redis_connection.smembers('workers')]
-
-        for w in workers:
-            self.statsd_client.gauge('worker_{}.seconds_since_update'.format(w['id']),
-                                     time.time() - float(w['updated_at']))
-            self.statsd_client.gauge('worker_{}.jobs_received'.format(w['id']), int(w['jobs_count']))
-            self.statsd_client.gauge('worker_{}.jobs_done'.format(w['id']), int(w['done_jobs_count']))
-
-        manager_status = self.redis_connection.hgetall('manager:status')
-        self.statsd_client.gauge('manager.seconds_since_refresh',
-                                 time.time() - float(manager_status['last_refresh_at']))
-
-    def refresh_queries(self):
-        # TODO: this will only execute scheduled queries that were executed before. I think this is
-        # a reasonable assumption, but worth revisiting.
-
-        # TODO: move this logic to the model.
-        outdated_queries = models.Query.select(peewee.Func('first_value', models.Query.id)\
-            .over(partition_by=[models.Query.query_hash, models.Query.data_source]))\
-            .join(models.QueryResult)\
-            .where(models.Query.ttl > 0,
-                   (models.QueryResult.retrieved_at +
-                    (models.Query.ttl * peewee.SQL("interval '1 second'"))) <
-                   peewee.SQL("(now() at time zone 'utc')"))
-
-        queries = models.Query.select(models.Query, models.DataSource).join(models.DataSource)\
-            .where(models.Query.id << outdated_queries)
-
-        self.status['last_refresh_at'] = time.time()
-        self._save_status()
-
-        logging.info("Refreshing queries...")
-
-        outdated_queries_count = 0
-        for query in queries:
-            self.add_job(query.query, worker.Job.LOW_PRIORITY, query.data_source)
-            outdated_queries_count += 1
-
-        self.statsd_client.gauge('manager.outdated_queries', outdated_queries_count)
-        self.statsd_client.gauge('manager.queue_size', self.redis_connection.zcard('jobs'))
-
-        logging.info("Done refreshing queries... %d" % outdated_queries_count)
-
-    def store_query_result(self, data_source_id, query, data, run_time, retrieved_at):
-        query_hash = gen_query_hash(query)
-
-        query_result = models.QueryResult.create(query_hash=query_hash,
-                                                 query=query,
-                                                 runtime=run_time,
-                                                 data_source=data_source_id,
-                                                 retrieved_at=retrieved_at,
-                                                 data=data)
-
-        logging.info("[Manager][%s] Inserted query data; id=%s", query_hash, query_result.id)
-
-        # TODO: move this logic to the model?
-        updated_count = models.Query.update(latest_query_data=query_result).\
-            where(models.Query.query_hash==query_hash, models.Query.data_source==data_source_id).\
-            execute()
-
-        logging.info("[Manager][%s] Updated %s queries.", query_hash, updated_count)
-
-        return query_result.id
-
-    def start_workers(self, workers_count):
-        if self.workers:
-            return self.workers
-
-        redis_connection_params = self.redis_connection.connection_pool.connection_kwargs
-        self.workers = [worker.Worker(worker_id, self, redis_connection_params)
-                        for worker_id in xrange(workers_count)]
-        for w in self.workers:
-            w.start()
-
-        return self.workers
-
-    def stop_workers(self):
-        for w in self.workers:
-            w.continue_working = False
-            w.join()
-
-    def _save_status(self):
-        self.redis_connection.hmset('manager:status', self.status)

redash/data/query_runner_bigquery.py

@@ -1,7 +1,9 @@
+import datetime
 import httplib2
 import json
 import logging
 import sys
+import time
 
 try:
     import apiclient.errors
@@ -14,6 +16,39 @@ except ImportError:
 
 from redash.utils import JSONEncoder
 
+types_map = {
+    'INTEGER': 'integer',
+    'FLOAT': 'float',
+    'BOOLEAN': 'boolean',
+    'STRING': 'string',
+    'TIMESTAMP': 'datetime',
+}
+
+def transform_row(row, fields):
+    column_index = 0
+    row_data = {}
+
+    for cell in row["f"]:
+        field = fields[column_index]
+        cell_value = cell['v']
+
+        if cell_value is None:
+            pass
+        # Otherwise just cast the value
+        elif field['type'] == 'INTEGER':
+            cell_value = int(cell_value)
+        elif field['type'] == 'FLOAT':
+            cell_value = float(cell_value)
+        elif field['type'] == 'BOOLEAN':
+            cell_value = cell_value.lower() == "true"
+        elif field['type'] == 'TIMESTAMP':
+            cell_value = datetime.datetime.fromtimestamp(float(cell_value))
+
+        row_data[field["name"]] = cell_value
+        column_index += 1
+
+    return row_data
+
 def bigquery(connection_string):
     def load_key(filename):
         f = file(filename, "rb")
@@ -27,12 +62,22 @@ def bigquery(connection_string):
             "https://www.googleapis.com/auth/bigquery",
         ]
 
-        credentials = SignedJwtAssertionCredentials(connection_string["serviceAccount"], load_key(connection_string["privateKey"]), scope=scope)
+        credentials = SignedJwtAssertionCredentials(connection_string["serviceAccount"],
+                                                    load_key(connection_string["privateKey"]), scope=scope)
         http = httplib2.Http()
         http = credentials.authorize(http)
 
         return build("bigquery", "v2", http=http)
 
+    def get_query_results(jobs, project_id, job_id, start_index):
+        query_reply = jobs.getQueryResults(projectId=project_id, jobId=job_id, startIndex=start_index).execute()
+        logging.debug('query_reply %s', query_reply)
+        if not query_reply['jobComplete']:
+            time.sleep(10)
+            return get_query_results(jobs, project_id, job_id, start_index)
+
+        return query_reply
+
     def query_runner(query):
         bigquery_service = get_bigquery_service()
 
@@ -52,44 +97,39 @@ def bigquery(connection_string):
         try:
             insert_response = jobs.insert(projectId=project_id, body=job_data).execute()
             current_row = 0
-            query_reply = jobs.getQueryResults(projectId=project_id, jobId=insert_response['jobReference']['jobId'], startIndex=current_row).execute()
+            query_reply = get_query_results(jobs, project_id=project_id,
+                                            job_id=insert_response['jobReference']['jobId'], start_index=current_row)
+
+            logging.debug("bigquery replied: %s", query_reply)
 
             rows = []
-            field_names = []
-            for f in query_reply["schema"]["fields"]:
-                field_names.append(f["name"])
 
-            while(("rows" in query_reply) and current_row < query_reply['totalRows']):
+            while ("rows" in query_reply) and current_row < query_reply['totalRows']:
                 for row in query_reply["rows"]:
-                    row_data = {}
-                    column_index = 0
-                    for cell in row["f"]:
-                        row_data[field_names[column_index]] = cell["v"]
-                        column_index += 1
-
-                    rows.append(row_data)
+                    rows.append(transform_row(row, query_reply["schema"]["fields"]))
 
                 current_row += len(query_reply['rows'])
-                query_reply = jobs.getQueryResults(projectId=project_id, jobId=query_reply['jobReference']['jobId'], startIndex=current_row).execute()
+                query_reply = jobs.getQueryResults(projectId=project_id, jobId=query_reply['jobReference']['jobId'],
+                                                   startIndex=current_row).execute()
 
-            columns = [{'name': name,
-                        'friendly_name': name,
-                        'type': None} for name in field_names]
+            columns = [{'name': f["name"],
+                        'friendly_name': f["name"],
+                        'type': types_map.get(f['type'], "string")} for f in query_reply["schema"]["fields"]]
 
             data = {
-                "columns" : columns,
-                "rows" : rows
+                "columns": columns,
+                "rows": rows
             }
             error = None
 
             json_data = json.dumps(data, cls=JSONEncoder)
         except apiclient.errors.HttpError, e:
             json_data = None
-            error = e.args[1]
+            error = e.content
         except KeyboardInterrupt:
             error = "Query cancelled by user."
             json_data = None
-        except Exception as e:
+        except Exception:
             raise sys.exc_info()[1], None, sys.exc_info()[2]
 
         return json_data, error

redash/data/query_runner_pg.py

@@ -8,11 +8,29 @@ query language (for example: HiveQL).
 import json
 import sys
 import select
-
+import logging
 import psycopg2
 
 from redash.utils import JSONEncoder
 
+types_map = {
+    20: 'integer',
+    21: 'integer',
+    23: 'integer',
+    700: 'float',
+    1700: 'float',
+    701: 'float',
+    16: 'boolean',
+    1082: 'date',
+    1114: 'datetime',
+    1184: 'datetime',
+    1014: 'string',
+    1015: 'string',
+    1008: 'string',
+    1009: 'string',
+    2951: 'string'
+}
+
 
 def pg(connection_string):
     def column_friendly_name(column_name):
@@ -20,15 +38,18 @@ def pg(connection_string):
 
     def wait(conn):
         while 1:
-            state = conn.poll()
-            if state == psycopg2.extensions.POLL_OK:
-                break
-            elif state == psycopg2.extensions.POLL_WRITE:
-                select.select([], [conn.fileno()], [])
-            elif state == psycopg2.extensions.POLL_READ:
-                select.select([conn.fileno()], [], [])
-            else:
-                raise psycopg2.OperationalError("poll() returned %s" % state)
+            try:
+                state = conn.poll()
+                if state == psycopg2.extensions.POLL_OK:
+                    break
+                elif state == psycopg2.extensions.POLL_WRITE:
+                    select.select([], [conn.fileno()], [])
+                elif state == psycopg2.extensions.POLL_READ:
+                    select.select([conn.fileno()], [], [])
+                else:
+                    raise psycopg2.OperationalError("poll() returned %s" % state)
+            except select.error:
+                raise psycopg2.OperationalError("select.error received")
 
     def query_runner(query):
         connection = psycopg2.connect(connection_string, async=True)
@@ -40,17 +61,37 @@ def pg(connection_string):
             cursor.execute(query)
             wait(connection)
 
-            column_names = [col.name for col in cursor.description]
+            # While set would be more efficient here, it sorts the data which is not what we want, but due to the small
+            # size of the data we can assume it's ok.
+            column_names = []
+            columns = []
+            duplicates_counter = 1
+
+            for column in cursor.description:
+                # TODO: this deduplication needs to be generalized and reused in all query runners.
+                column_name = column.name
+                if column_name in column_names:
+                    column_name = column_name + str(duplicates_counter)
+                    duplicates_counter += 1
+
+                column_names.append(column_name)
+
+                columns.append({
+                    'name': column_name,
+                    'friendly_name': column_friendly_name(column_name),
+                    'type': types_map.get(column.type_code, None)
+                })
 
             rows = [dict(zip(column_names, row)) for row in cursor]
-            columns = [{'name': col.name,
-                        'friendly_name': column_friendly_name(col.name),
-                        'type': None} for col in cursor.description]
 
             data = {'columns': columns, 'rows': rows}
             json_data = json.dumps(data, cls=JSONEncoder)
             error = None
             cursor.close()
+        except (select.error, OSError, psycopg2.OperationalError) as e:
+            logging.exception(e)
+            error = "Query interrupted. Please retry."
+            json_data = None
         except psycopg2.DatabaseError as e:
             json_data = None
             error = e.message

redash/data/worker.py

@@ -1,330 +0,0 @@
-"""
-Worker implementation to execute incoming queries.
-"""
-import json
-import logging
-import os
-import threading
-import uuid
-import datetime
-import time
-import signal
-import setproctitle
-import redis
-from statsd import StatsClient
-from redash.utils import gen_query_hash
-from redash.data.query_runner import get_query_runner
-from redash import settings
-
-
-class RedisObject(object):
-    # The following should be overriden in the inheriting class:
-    fields = {}
-    conversions = {}
-    id_field = ''
-    name = ''
-
-    def __init__(self, redis_connection, **kwargs):
-        self.redis_connection = redis_connection
-        self.values = {}
-
-        if not self.fields:
-            raise ValueError("You must set the fields dictionary, before using RedisObject.")
-
-        if not self.name:
-            raise ValueError("You must set the name, before using RedisObject")
-
-        self.update(**kwargs)
-
-    def __getattr__(self, name):
-        if name in self.values:
-            return self.values[name]
-        else:
-            raise AttributeError
-
-    def update(self, **kwargs):
-        for field, default_value in self.fields.iteritems():
-            value = kwargs.get(field, self.values.get(field, default_value))
-            if callable(value):
-                value = value()
-
-            if value == 'None':
-                value = None
-
-            if field in self.conversions and value:
-                value = self.conversions[field](value)
-
-            self.values[field] = value
-
-    @classmethod
-    def _redis_key(cls, object_id):
-        return '{}:{}'.format(cls.name, object_id)
-
-    def save(self, pipe):
-        if not pipe:
-            pipe = self.redis_connection.pipeline()
-
-        pipe.sadd('{}_set'.format(self.name), self.id)
-        pipe.hmset(self._redis_key(self.id), self.values)
-        pipe.publish(self._redis_key(self.id), json.dumps(self.to_dict()))
-
-        pipe.execute()
-
-    @classmethod
-    def load(cls, redis_connection, object_id):
-        object_dict = redis_connection.hgetall(cls._redis_key(object_id))
-        obj = None
-        if object_dict:
-            obj = cls(redis_connection, **object_dict)
-
-        return obj
-
-
-def fix_unicode(string):
-    if isinstance(string, unicode):
-        return string
-
-    return string.decode('utf-8')
-
-
-class Job(RedisObject):
-    HIGH_PRIORITY = 1
-    LOW_PRIORITY = 2
-
-    WAITING = 1
-    PROCESSING = 2
-    DONE = 3
-    FAILED = 4
-
-    fields = {
-        'id': lambda: str(uuid.uuid1()),
-        'query': None,
-        'priority': None,
-        'query_hash': None,
-        'wait_time': 0,
-        'query_time': 0,
-        'error': None,
-        'updated_at': time.time,
-        'status': WAITING,
-        'process_id': None,
-        'query_result_id': None,
-        'data_source_id': None,
-        'data_source_name': None,
-        'data_source_type': None,
-        'data_source_options': None
-    }
-
-    conversions = {
-        'query': fix_unicode,
-        'priority': int,
-        'updated_at': float,
-        'status': int,
-        'wait_time': float,
-        'query_time': float,
-        'process_id': int,
-        'query_result_id': int
-    }
-
-    name = 'job'
-
-    def __init__(self, redis_connection, query, priority, **kwargs):
-        kwargs['query'] = fix_unicode(query)
-        kwargs['priority'] = priority
-        kwargs['query_hash'] = gen_query_hash(kwargs['query'])
-        self.new_job = 'id' not in kwargs
-        super(Job, self).__init__(redis_connection, **kwargs)
-
-    def to_dict(self):
-        return {
-            'query': self.query,
-            'priority': self.priority,
-            'id': self.id,
-            'wait_time': self.wait_time,
-            'query_time': self.query_time,
-            'updated_at': self.updated_at,
-            'status': self.status,
-            'error': self.error,
-            'query_result_id': self.query_result_id,
-            'process_id': self.process_id,
-            'data_source_name': self.data_source_name,
-            'data_source_type': self.data_source_type
-        }
-
-    def cancel(self):
-        # TODO: Race condition:
-        # it's possible that it will be picked up by worker while processing the cancel order
-        if self.is_finished():
-            return
-
-        if self.status == self.PROCESSING:
-            os.kill(self.process_id, signal.SIGINT)
-        else:
-            self.done(None, "Interrupted/Cancelled while running.")
-
-    def save(self, pipe=None):
-        if not pipe:
-            pipe = self.redis_connection.pipeline()
-
-        if self.new_job:
-            pipe.set('query_hash_job:%s' % self.query_hash, self.id)
-
-        if self.is_finished():
-            pipe.delete('query_hash_job:%s' % self.query_hash)
-
-        super(Job, self).save(pipe)
-
-    def processing(self, process_id):
-        self.update(status=self.PROCESSING,
-                    process_id=process_id,
-                    wait_time=time.time() - self.updated_at,
-                    updated_at=time.time())
-
-        self.save()
-
-    def is_finished(self):
-        return self.status in (self.FAILED, self.DONE)
-
-    def done(self, query_result_id, error):
-        if error:
-            new_status = self.FAILED
-        else:
-            new_status = self.DONE
-
-        self.update(status=new_status,
-                    query_result_id=query_result_id,
-                    error=error,
-                    query_time=time.time() - self.updated_at,
-                    updated_at=time.time())
-
-        self.save()
-
-    def __str__(self):
-        return "<Job:%s,priority:%d,status:%d>" % (self.id, self.priority, self.status)
-
-
-class Worker(threading.Thread):
-    def __init__(self, worker_id, manager, redis_connection_params, sleep_time=0.1):
-        self.manager = manager
-
-        self.statsd_client = StatsClient(host=settings.STATSD_HOST, port=settings.STATSD_PORT,
-                                         prefix=settings.STATSD_PREFIX)
-        self.redis_connection_params = {k: v for k, v in redis_connection_params.iteritems()
-                                        if k in ('host', 'db', 'password', 'port')}
-
-        self.continue_working = True
-        self.sleep_time = sleep_time
-        self.child_pid = None
-        self.worker_id = worker_id
-        self.status = {
-            'id': self.worker_id,
-            'jobs_count': 0,
-            'cancelled_jobs_count': 0,
-            'done_jobs_count': 0,
-            'updated_at': time.time(),
-            'started_at': time.time()
-        }
-        self._save_status()
-        self.manager.redis_connection.sadd('workers', self._key)
-
-        super(Worker, self).__init__(name="Worker-%s" % self.worker_id)
-
-    def set_title(self, title=None):
-        base_title = "redash worker:%s" % self.worker_id
-        if title:
-            full_title = "%s - %s" % (base_title, title)
-        else:
-            full_title = base_title
-
-        setproctitle.setproctitle(full_title)
-
-    def run(self):
-        logging.info("[%s] started.", self.name)
-        while self.continue_working:
-            job_id = self.manager.queue.pop()
-            if job_id:
-                self._update_status('jobs_count')
-                logging.info("[%s] Processing %s", self.name, job_id)
-                self._fork_and_process(job_id)
-                if self.child_pid == 0:
-                    return
-            else:
-                time.sleep(self.sleep_time)
-
-    def _update_status(self, counter):
-        self.status['updated_at'] = time.time()
-        self.status[counter] += 1
-        self._save_status()
-
-    @property
-    def _key(self):
-        return 'worker:%s' % self.worker_id
-
-    def _save_status(self):
-        self.manager.redis_connection.hmset(self._key, self.status)
-
-    def _fork_and_process(self, job_id):
-        self.child_pid = os.fork()
-        if self.child_pid == 0:
-            self.set_title("processing %s" % job_id)
-            self._process(job_id)
-        else:
-            logging.info("[%s] Waiting for pid: %d", self.name, self.child_pid)
-            _, status = os.waitpid(self.child_pid, 0)
-            self._update_status('done_jobs_count')
-            if status > 0:
-                job = Job.load(self.manager.redis_connection, job_id)
-                if not job.is_finished():
-                    self._update_status('cancelled_jobs_count')
-                    logging.info("[%s] process interrupted and job %s hasn't finished; registering interruption in job",
-                                 self.name, job_id)
-                    job.done(None, "Interrupted/Cancelled while running.")
-
-            logging.info("[%s] Finished Processing %s (pid: %d status: %d)",
-                         self.name, job_id, self.child_pid, status)
-
-    def _process(self, job_id):
-        redis_connection = redis.StrictRedis(**self.redis_connection_params)
-        job = Job.load(redis_connection, job_id)
-        if job.is_finished():
-            logging.warning("[%s][%s] tried to process finished job.", self.name, job)
-            return
-
-        pid = os.getpid()
-        job.processing(pid)
-
-        logging.info("[%s][%s] running query...", self.name, job.id)
-        start_time = time.time()
-        self.set_title("running query %s" % job_id)
-
-        logging.info("[%s][%s] Loading query runner (%s, %s)...", self.name, job.id,
-                     job.data_source_name, job.data_source_type)
-
-        query_runner = get_query_runner(job.data_source_type, job.data_source_options)
-
-        if getattr(query_runner, 'annotate_query', True):
-            annotated_query = "/* Pid: %s, Job Id: %s, Query hash: %s, Priority: %s */ %s" % \
-                              (pid, job.id, job.query_hash, job.priority, job.query)
-        else:
-            annotated_query = job.query
-
-        # TODO: here's the part that needs to be forked, not all of the worker process...
-        with self.statsd_client.timer('worker_{}.query_runner.{}.{}.run_time'.format(self.worker_id,
-                                                                                     job.data_source_type,
-                                                                                     job.data_source_name)):
-            data, error = query_runner(annotated_query)
-
-        run_time = time.time() - start_time
-        logging.info("[%s][%s] query finished... data length=%s, error=%s",
-                     self.name, job.id, data and len(data), error)
-
-        # TODO: it is possible that storing the data will fail, and we will need to retry
-        # while we already marked the job as done
-        query_result_id = None
-        if not error:
-            self.set_title("storing results %s" % job_id)
-            query_result_id = self.manager.store_query_result(job.data_source_id,
-                                                              job.query, data, run_time,
-                                                              datetime.datetime.utcnow())
-
-        self.set_title("marking job as done %s" % job_id)
-        job.done(query_result_id, error)

redash/events.py

@@ -0,0 +1,23 @@
+import logging
+import json
+
+logger = logging.getLogger("redash.events")
+logger.propagate = False
+
+
+def setup_logging(log_path, console_output=False):
+    if log_path:
+        fh = logging.FileHandler(log_path)
+        formatter = logging.Formatter('%(message)s')
+        fh.setFormatter(formatter)
+        logger.addHandler(fh)
+
+    if console_output:
+        handler = logging.StreamHandler()
+        formatter = logging.Formatter('[%(name)s] %(message)s')
+        handler.setFormatter(formatter)
+        logger.addHandler(handler)
+
+
+def record_event(event):
+    logger.info(json.dumps(event))

redash/import_export.py

@@ -128,7 +128,7 @@ def importer_with_mapping_file(mapping_filename):
 def get_data_source():
     try:
         data_source = models.DataSource.get(models.DataSource.name=="Import")
-    except models.DataSource.DoestNotExist:
+    except models.DataSource.DoesNotExist:
         data_source = models.DataSource.create(name="Import", type="import", options='{}')
 
     return data_source

redash/models.py

@@ -1,16 +1,60 @@
 import json
 import hashlib
+import logging
+import os
+import threading
 import time
 import datetime
-from flask.ext.peewee.utils import slugify
-from flask.ext.login import UserMixin, AnonymousUserMixin
-from passlib.apps import custom_app_context as pwd_context
+import itertools
+
 import peewee
+from passlib.apps import custom_app_context as pwd_context
 from playhouse.postgres_ext import ArrayField
-from redash import db, utils
+from flask.ext.login import UserMixin, AnonymousUserMixin
+
+from redash import utils, settings
 
 
-class BaseModel(db.Model):
+class Database(object):
+    def __init__(self):
+        self.database_config = dict(settings.DATABASE_CONFIG)
+        self.database_name = self.database_config.pop('name')
+        self.database = peewee.PostgresqlDatabase(self.database_name, **self.database_config)
+        self.app = None
+        self.pid = os.getpid()
+
+    def init_app(self, app):
+        self.app = app
+        self.register_handlers()
+
+    def connect_db(self):
+        self._check_pid()
+        self.database.connect()
+
+    def close_db(self, exc):
+        self._check_pid()
+        if not self.database.is_closed():
+            self.database.close()
+
+    def _check_pid(self):
+        current_pid = os.getpid()
+        if self.pid != current_pid:
+            logging.info("New pid detected (%d!=%d); resetting database lock.", self.pid, current_pid)
+            self.pid = os.getpid()
+            self.database._conn_lock = threading.Lock()
+
+    def register_handlers(self):
+        self.app.before_request(self.connect_db)
+        self.app.teardown_request(self.close_db)
+
+
+db = Database()
+
+
+class BaseModel(peewee.Model):
+    class Meta:
+        database = db.database
+
     @classmethod
     def get_by_id(cls, model_id):
         return cls.get(cls.id == model_id)
@@ -31,16 +75,40 @@ class ApiUser(UserMixin):
         return ['view_query']
 
 
-class User(BaseModel, UserMixin):
+class Group(BaseModel):
     DEFAULT_PERMISSIONS = ['create_dashboard', 'create_query', 'edit_dashboard', 'edit_query',
                            'view_query', 'view_source', 'execute_query']
 
+    id = peewee.PrimaryKeyField()
+    name = peewee.CharField(max_length=100)
+    permissions = ArrayField(peewee.CharField, default=DEFAULT_PERMISSIONS)
+    tables = ArrayField(peewee.CharField)
+    created_at = peewee.DateTimeField(default=datetime.datetime.now)
+
+    class Meta:
+        db_table = 'groups'
+
+    def to_dict(self):
+        return {
+            'id': self.id,
+            'name': self.name,
+            'permissions': self.permissions,
+            'tables': self.tables,
+            'created_at': self.created_at
+        }
+
+    def __unicode__(self):
+        return unicode(self.id)
+
+
+class User(BaseModel, UserMixin):
+    DEFAULT_GROUPS = ['default']
+
     id = peewee.PrimaryKeyField()
     name = peewee.CharField(max_length=320)
     email = peewee.CharField(max_length=320, index=True, unique=True)
     password_hash = peewee.CharField(max_length=128, null=True)
-    is_admin = peewee.BooleanField(default=False)
-    permissions = ArrayField(peewee.CharField, default=DEFAULT_PERMISSIONS)
+    groups = ArrayField(peewee.CharField, default=DEFAULT_GROUPS)
 
     class Meta:
         db_table = 'users'
@@ -49,10 +117,32 @@ class User(BaseModel, UserMixin):
         return {
             'id': self.id,
             'name': self.name,
-            'email': self.email,
-            'is_admin': self.is_admin
+            'email': self.email
         }
 
+    def __init__(self, *args, **kwargs):
+        super(User, self).__init__(*args, **kwargs)
+        self._allowed_tables = None
+
+    @property
+    def permissions(self):
+        # TODO: this should be cached.
+        return list(itertools.chain(*[g.permissions for g in
+                                      Group.select().where(Group.name << self.groups)]))
+
+    @property
+    def allowed_tables(self):
+        # TODO: cache this as weel
+        if self._allowed_tables is None:
+            self._allowed_tables = set([t.lower() for t in itertools.chain(*[g.tables for g in
+                                        Group.select().where(Group.name << self.groups)])])
+
+        return self._allowed_tables
+
+    @classmethod
+    def get_by_email(cls, email):
+        return cls.get(cls.email == email)
+
     def __unicode__(self):
         return '%r, %r' % (self.name, self.email)
 
@@ -65,7 +155,7 @@ class User(BaseModel, UserMixin):
 
 class ActivityLog(BaseModel):
     QUERY_EXECUTION = 1
-    
+
     id = peewee.PrimaryKeyField()
     user = peewee.ForeignKeyField(User)
     type = peewee.IntegerField()
@@ -93,6 +183,8 @@ class DataSource(BaseModel):
     name = peewee.CharField()
     type = peewee.CharField()
     options = peewee.TextField()
+    queue_name = peewee.CharField(default="queries")
+    scheduled_queue_name = peewee.CharField(default="queries")
     created_at = peewee.DateTimeField(default=datetime.datetime.now)
 
     class Meta:
@@ -105,6 +197,10 @@ class DataSource(BaseModel):
             'type': self.type
         }
 
+    @classmethod
+    def all(cls):
+        return cls.select().order_by(cls.id.asc())
+
 
 class QueryResult(BaseModel):
     id = peewee.PrimaryKeyField()
@@ -133,11 +229,35 @@ class QueryResult(BaseModel):
     def get_latest(cls, data_source, query, ttl=0):
         query_hash = utils.gen_query_hash(query)
 
-        query = cls.select().where(cls.query_hash == query_hash, cls.data_source == data_source,
-                                   peewee.SQL("retrieved_at + interval '%s second' >= now() at time zone 'utc'", ttl)).order_by(cls.retrieved_at.desc())
+        if ttl == -1:
+            query = cls.select().where(cls.query_hash == query_hash,
+                                       cls.data_source == data_source).order_by(cls.retrieved_at.desc())
+        else:
+            query = cls.select().where(cls.query_hash == query_hash, cls.data_source == data_source,
+                                       peewee.SQL("retrieved_at + interval '%s second' >= now() at time zone 'utc'",
+                                                  ttl)).order_by(cls.retrieved_at.desc())
 
         return query.first()
 
+    @classmethod
+    def store_result(cls, data_source_id, query_hash, query, data, run_time, retrieved_at):
+        query_result = cls.create(query_hash=query_hash,
+                                  query=query,
+                                  runtime=run_time,
+                                  data_source=data_source_id,
+                                  retrieved_at=retrieved_at,
+                                  data=data)
+
+        logging.info("Inserted query (%s) data; id=%s", query_hash, query_result.id)
+
+        updated_count = Query.update(latest_query_data=query_result).\
+            where(Query.query_hash==query_hash, Query.data_source==data_source_id).\
+            execute()
+
+        logging.info("Updated %s queries with result (%s).", updated_count, query_hash)
+
+        return query_result
+
     def __unicode__(self):
         return u"%d | %s | %s" % (self.id, self.query_hash, self.retrieved_at)
 
@@ -165,7 +285,7 @@ class Query(BaseModel):
                                             type="TABLE", options="{}")
         table_visualization.save()
 
-    def to_dict(self, with_result=True, with_stats=False, with_visualizations=False, with_user=True):
+    def to_dict(self, with_stats=False, with_visualizations=False, with_user=True):
         d = {
             'id': self.id,
             'latest_query_data_id': self._data.get('latest_query_data', None),
@@ -195,9 +315,6 @@ class Query(BaseModel):
             d['visualizations'] = [vis.to_dict(with_query=False)
                                    for vis in self.visualizations]
 
-        if with_result and self.latest_query_data:
-            d['latest_query_data'] = self.latest_query_data.to_dict()
-
         return d
 
     @classmethod
@@ -214,6 +331,23 @@ class Query(BaseModel):
 
         return q
 
+    @classmethod
+    def outdated_queries(cls):
+        # TODO: this will only find scheduled queries that were executed before. I think this is
+        # a reasonable assumption, but worth revisiting.
+        outdated_queries_ids = cls.select(
+            peewee.Func('first_value', cls.id).over(partition_by=[cls.query_hash, cls.data_source])) \
+            .join(QueryResult) \
+            .where(cls.ttl > 0,
+                   (QueryResult.retrieved_at +
+                    (cls.ttl * peewee.SQL("interval '1 second'"))) <
+                   peewee.SQL("(now() at time zone 'utc')"))
+
+        queries = cls.select(cls, DataSource).join(DataSource) \
+            .where(cls.id << outdated_queries_ids)
+
+        return queries
+
     @classmethod
     def update_instance(cls, query_id, **kwargs):
         if 'query' in kwargs:
@@ -243,6 +377,7 @@ class Dashboard(BaseModel):
     user_email = peewee.CharField(max_length=360, null=True)
     user = peewee.ForeignKeyField(User)
     layout = peewee.TextField()
+    dashboard_filters_enabled = peewee.BooleanField(default=False)
     is_archived = peewee.BooleanField(default=False, index=True)
     created_at = peewee.DateTimeField(default=datetime.datetime.now)
 
@@ -253,15 +388,29 @@ class Dashboard(BaseModel):
         layout = json.loads(self.layout)
 
         if with_widgets:
-            widgets = Widget.select(Widget, Visualization, Query, QueryResult, User)\
+            widgets = Widget.select(Widget, Visualization, Query, User)\
                 .where(Widget.dashboard == self.id)\
-                .join(Visualization)\
-                .join(Query)\
-                .join(User)\
-                .switch(Query)\
-                .join(QueryResult)
+                .join(Visualization, join_type=peewee.JOIN_LEFT_OUTER)\
+                .join(Query, join_type=peewee.JOIN_LEFT_OUTER)\
+                .join(User, join_type=peewee.JOIN_LEFT_OUTER)
             widgets = {w.id: w.to_dict() for w in widgets}
-            widgets_layout = map(lambda row: map(lambda widget_id: widgets.get(widget_id, None), row), layout)
+
+            # The following is a workaround for cases when the widget object gets deleted without the dashboard layout
+            # updated. This happens for users with old databases that didn't have a foreign key relationship between
+            # visualizations and widgets.
+            # It's temporary until better solution is implemented (we probably should move the position information
+            # to the widget).
+            widgets_layout = []
+            for row in layout:
+                new_row = []
+                for widget_id in row:
+                    widget = widgets.get(widget_id, None)
+                    if widget:
+                        new_row.append(widget)
+
+                widgets_layout.append(new_row)
+
+            # widgets_layout = map(lambda row: map(lambda widget_id: widgets.get(widget_id, None), row), layout)
         else:
             widgets_layout = None
 
@@ -271,6 +420,7 @@ class Dashboard(BaseModel):
             'name': self.name,
             'user_id': self._data['user'],
             'layout': layout,
+            'dashboard_filters_enabled': self.dashboard_filters_enabled,
             'widgets': widgets_layout
         }
 
@@ -280,11 +430,11 @@ class Dashboard(BaseModel):
 
     def save(self, *args, **kwargs):
         if not self.slug:
-            self.slug = slugify(self.name)
+            self.slug = utils.slugify(self.name)
 
             tries = 1
             while self.select().where(Dashboard.slug == self.slug).first() is not None:
-                self.slug = slugify(self.name) + "_{0}".format(tries)
+                self.slug = utils.slugify(self.name) + "_{0}".format(tries)
                 tries += 1
 
         super(Dashboard, self).save(*args, **kwargs)
@@ -324,8 +474,8 @@ class Visualization(BaseModel):
 
 class Widget(BaseModel):
     id = peewee.PrimaryKeyField()
-    visualization = peewee.ForeignKeyField(Visualization, related_name='widgets')
-
+    visualization = peewee.ForeignKeyField(Visualization, related_name='widgets', null=True)
+    text = peewee.TextField(null=True)
     width = peewee.IntegerField()
     options = peewee.TextField()
     dashboard = peewee.ForeignKeyField(Dashboard, related_name='widgets', index=True)
@@ -339,18 +489,44 @@ class Widget(BaseModel):
         db_table = 'widgets'
 
     def to_dict(self):
-        return {
+        d = {
             'id': self.id,
             'width': self.width,
             'options': json.loads(self.options),
-            'visualization': self.visualization.to_dict(),
-            'dashboard_id': self._data['dashboard']
+            'dashboard_id': self._data['dashboard'],
+            'text': self.text
         }
 
+        if self.visualization and self.visualization.id:
+            d['visualization'] = self.visualization.to_dict()
+
+        return d
+
     def __unicode__(self):
         return u"%s" % self.id
 
-all_models = (DataSource, User, QueryResult, Query, Dashboard, Visualization, Widget, ActivityLog)
+
+class Event(BaseModel):
+    user = peewee.ForeignKeyField(User, related_name="events")
+    action = peewee.CharField()
+    object_type = peewee.CharField()
+    object_id = peewee.CharField(null=True)
+    additional_properties = peewee.TextField(null=True)
+    created_at = peewee.DateTimeField(default=datetime.datetime.now)
+
+    class Meta:
+        db_table = 'events'
+
+    def __unicode__(self):
+        return u"%s,%s,%s,%s" % (self._data['user'], self.action, self.object_type, self.object_id)
+
+
+all_models = (DataSource, User, QueryResult, Query, Dashboard, Visualization, Widget, ActivityLog, Group, Event)
+
+
+def init_db():
+    Group.insert(name='admin', permissions=['admin'], tables=['*']).execute()
+    Group.insert(name='default', permissions=Group.DEFAULT_PERMISSIONS, tables=['*']).execute()
 
 
 def create_db(create_tables, drop_tables):
@@ -365,4 +541,4 @@ def create_db(create_tables, drop_tables):
         if create_tables and not model.table_exists():
             model.create_table()
 
-    db.close_db(None)
+    db.close_db(None)

redash/settings.py

@@ -5,9 +5,7 @@ import urlparse
 
 def parse_db_url(url):
     url_parts = urlparse.urlparse(url)
-    connection = {
-        'engine': 'peewee.PostgresqlDatabase',
-    }
+    connection = {'threadlocals': True}
 
     if url_parts.hostname and not url_parts.path:
         connection['name'] = url_parts.hostname
@@ -38,14 +36,14 @@ def parse_boolean(str):
     return json.loads(str.lower())
 
 
-REDIS_URL = os.environ.get('REDASH_REDIS_URL', "redis://localhost:6379")
+NAME = os.environ.get('REDASH_NAME', 're:dash')
+
+REDIS_URL = os.environ.get('REDASH_REDIS_URL', "redis://localhost:6379/0")
 
 STATSD_HOST = os.environ.get('REDASH_STATSD_HOST', "127.0.0.1")
 STATSD_PORT = int(os.environ.get('REDASH_STATSD_PORT', "8125"))
 STATSD_PREFIX = os.environ.get('REDASH_STATSD_PREFIX', "redash")
 
-NAME = os.environ.get('REDASH_NAME', 're:dash')
-
 # The following is kept for backward compatability, and shouldn't be used any more.
 CONNECTION_ADAPTER = os.environ.get("REDASH_CONNECTION_ADAPTER", "pg")
 CONNECTION_STRING = os.environ.get("REDASH_CONNECTION_STRING", "user= password= host= port=5439 dbname=")
@@ -53,16 +51,26 @@ CONNECTION_STRING = os.environ.get("REDASH_CONNECTION_STRING", "user= password=
 # Connection settings for re:dash's own database (where we store the queries, results, etc)
 DATABASE_CONFIG = parse_db_url(os.environ.get("REDASH_DATABASE_URL", "postgresql://postgres"))
 
+# Celery related settings
+CELERY_BROKER = os.environ.get("REDASH_CELERY_BROKER", REDIS_URL)
+CELERY_BACKEND = os.environ.get("REDASH_CELERY_BACKEND", REDIS_URL)
+CELERY_FLOWER_URL = os.environ.get("REDASH_CELERY_FLOWER_URL", "/flower")
+
 # Google Apps domain to allow access from; any user with email in this Google Apps will be allowed
 # access
 GOOGLE_APPS_DOMAIN = os.environ.get("REDASH_GOOGLE_APPS_DOMAIN", "")
 GOOGLE_OPENID_ENABLED = parse_boolean(os.environ.get("REDASH_GOOGLE_OPENID_ENABLED", "true"))
 PASSWORD_LOGIN_ENABLED = parse_boolean(os.environ.get("REDASH_PASSWORD_LOGIN_ENABLED", "false"))
-# Email addresses of admin users (comma separated)
-ADMINS = array_from_string(os.environ.get("REDASH_ADMINS", ''))
 ALLOWED_EXTERNAL_USERS = array_from_string(os.environ.get("REDASH_ALLOWED_EXTERNAL_USERS", ''))
 STATIC_ASSETS_PATH = fix_assets_path(os.environ.get("REDASH_STATIC_ASSETS_PATH", "../rd_ui/app/"))
 WORKERS_COUNT = int(os.environ.get("REDASH_WORKERS_COUNT", "2"))
+JOB_EXPIRY_TIME = int(os.environ.get("REDASH_JOB_EXPIRY_TIME", 3600*24))
 COOKIE_SECRET = os.environ.get("REDASH_COOKIE_SECRET", "c292a0a3aa32397cdb050e233733900f")
 LOG_LEVEL = os.environ.get("REDASH_LOG_LEVEL", "INFO")
-ANALYTICS = os.environ.get("REDASH_ANALYTICS", "")
+EVENTS_LOG_PATH = os.environ.get("REDASH_EVENTS_LOG_PATH", "")
+EVENTS_CONSOLE_OUTPUT = parse_boolean(os.environ.get("REDASH_EVENTS_CONSOLE_OUTPUT", "false"))
+CLIENT_SIDE_METRICS = parse_boolean(os.environ.get("REDASH_CLIENT_SIDE_METRICS", "false"))
+ANALYTICS = os.environ.get("REDASH_ANALYTICS", "")
+
+# Features:
+FEATURE_TABLES_PERMISSIONS = parse_boolean(os.environ.get("REDASH_FEATURE_TABLES_PERMISSIONS", "false"))

redash/tasks.py

@@ -0,0 +1,197 @@
+import time
+import datetime
+import logging
+import redis
+from celery import Task
+from celery.result import AsyncResult
+from celery.utils.log import get_task_logger
+from redash import redis_connection, models, statsd_client
+from redash.utils import gen_query_hash
+from redash.worker import celery
+from redash.data.query_runner import get_query_runner
+
+logger = get_task_logger(__name__)
+
+
+class BaseTask(Task):
+    abstract = True
+
+    def after_return(self, *args, **kwargs):
+        models.db.close_db(None)
+
+    def __call__(self, *args, **kwargs):
+        models.db.connect_db()
+        return super(BaseTask, self).__call__(*args, **kwargs)
+
+
+class QueryTask(object):
+    MAX_RETRIES = 5
+
+    # TODO: this is mapping to the old Job class statuses. Need to update the client side and remove this
+    STATUSES = {
+        'PENDING': 1,
+        'STARTED': 2,
+        'SUCCESS': 3,
+        'FAILURE': 4,
+        'REVOKED': 4
+    }
+
+    def __init__(self, job_id=None, async_result=None):
+        if async_result:
+            self._async_result = async_result
+        else:
+            self._async_result = AsyncResult(job_id, app=celery)
+
+    @property
+    def id(self):
+        return self._async_result.id
+
+    @classmethod
+    def add_task(cls, query, data_source, scheduled=False):
+        query_hash = gen_query_hash(query)
+        logging.info("[Manager][%s] Inserting job", query_hash)
+        try_count = 0
+        job = None
+
+        while try_count < cls.MAX_RETRIES:
+            try_count += 1
+
+            pipe = redis_connection.pipeline()
+            try:
+                pipe.watch(cls._job_lock_id(query_hash, data_source.id))
+                job_id = pipe.get(cls._job_lock_id(query_hash, data_source.id))
+                if job_id:
+                    logging.info("[Manager][%s] Found existing job: %s", query_hash, job_id)
+
+                    job = cls(job_id=job_id)
+                else:
+                    pipe.multi()
+
+                    if scheduled:
+                        queue_name = data_source.scheduled_queue_name
+                    else:
+                        queue_name = data_source.queue_name
+
+                    result = execute_query.apply_async(args=(query, data_source.id), queue=queue_name)
+                    job = cls(async_result=result)
+                    logging.info("[Manager][%s] Created new job: %s", query_hash, job.id)
+                    pipe.set(cls._job_lock_id(query_hash, data_source.id), job.id)
+                    pipe.execute()
+                break
+
+            except redis.WatchError:
+                continue
+
+        if not job:
+            logging.error("[Manager][%s] Failed adding job for query.", query_hash)
+
+        return job
+
+    def to_dict(self):
+        if self._async_result.status == 'STARTED':
+            updated_at = self._async_result.result.get('start_time', 0)
+        else:
+            updated_at = 0
+
+        if self._async_result.failed() and isinstance(self._async_result.result, Exception):
+            error = self._async_result.result.message
+        elif self._async_result.status == 'REVOKED':
+            error = 'Query execution cancelled.'
+        else:
+            error = ''
+
+        if self._async_result.successful():
+            query_result_id = self._async_result.result
+        else:
+            query_result_id = None
+
+        return {
+            'id': self._async_result.id,
+            'updated_at': updated_at,
+            'status': self.STATUSES[self._async_result.status],
+            'error': error,
+            'query_result_id': query_result_id,
+        }
+
+    def cancel(self):
+        return self._async_result.revoke(terminate=True)
+
+    @staticmethod
+    def _job_lock_id(query_hash, data_source_id):
+        return "query_hash_job:%s:%s" % (data_source_id, query_hash)
+
+
+@celery.task(base=BaseTask)
+def refresh_queries():
+    # self.status['last_refresh_at'] = time.time()
+    # self._save_status()
+
+    logger.info("Refreshing queries...")
+
+    outdated_queries_count = 0
+    for query in models.Query.outdated_queries():
+        # TODO: this should go into lower priority
+        QueryTask.add_task(query.query, query.data_source, scheduled=True)
+        outdated_queries_count += 1
+
+    statsd_client.gauge('manager.outdated_queries', outdated_queries_count)
+    # TODO: decide if we still need this
+    # statsd_client.gauge('manager.queue_size', self.redis_connection.zcard('jobs'))
+
+    logger.info("Done refreshing queries. Found %d outdated queries." % outdated_queries_count)
+
+    status = redis_connection.hgetall('redash:status')
+    now = time.time()
+
+    redis_connection.hmset('redash:status', {
+        'outdated_queries_count': outdated_queries_count,
+        'last_refresh_at': now
+    })
+
+    statsd_client.gauge('manager.seconds_since_refresh', now - float(status.get('last_refresh_at', now)))
+
+
+@celery.task(bind=True, base=BaseTask, track_started=True)
+def execute_query(self, query, data_source_id):
+    # TODO: maybe this should be a class?
+    start_time = time.time()
+
+    logger.info("Loading data source (%d)...", data_source_id)
+
+    # TODO: we should probably cache data sources in Redis
+    data_source = models.DataSource.get_by_id(data_source_id)
+
+    self.update_state(state='STARTED', meta={'start_time': start_time, 'custom_message': ''})
+
+    logger.info("Executing query:\n%s", query)
+
+    query_hash = gen_query_hash(query)
+    query_runner = get_query_runner(data_source.type, data_source.options)
+
+    if getattr(query_runner, 'annotate_query', True):
+        # TODO: anotate with queu ename
+        annotated_query = "/* Task Id: %s, Query hash: %s */ %s" % \
+                          (self.request.id, query_hash, query)
+    else:
+        annotated_query = query
+
+    with statsd_client.timer('query_runner.{}.{}.run_time'.format(data_source.type, data_source.name)):
+        data, error = query_runner(annotated_query)
+
+    run_time = time.time() - start_time
+    logger.info("Query finished... data length=%s, error=%s", data and len(data), error)
+
+    self.update_state(state='STARTED', meta={'start_time': start_time, 'error': error, 'custom_message': ''})
+
+    # Delete query_hash
+    redis_connection.delete(QueryTask._job_lock_id(query_hash, data_source.id))
+
+    # TODO: it is possible that storing the data will fail, and we will need to retry
+    # while we already marked the job as done
+    if not error:
+        query_result = models.QueryResult.store_result(data_source.id, query_hash, query, data, run_time, datetime.datetime.utcnow())
+    else:
+        raise Exception(error)
+
+    return query_result.id
+

redash/utils.py

@@ -6,10 +6,66 @@ import datetime
 import json
 import re
 import hashlib
+import sqlparse
 
 COMMENTS_REGEX = re.compile("/\*.*?\*/")
 
 
+class SQLMetaData(object):
+    TABLE_SELECTION_KEYWORDS = ('FROM', 'JOIN', 'LEFT JOIN', 'FULL JOIN', 'RIGHT JOIN', 'CROSS JOIN', 'INNER JOIN',
+                                'OUTER JOIN', 'LEFT OUTER JOIN', 'RIGHT OUTER JOIN', 'FULL OUTER JOIN')
+
+    def __init__(self, sql):
+        self.sql = sql
+        self.parsed_sql = sqlparse.parse(self.sql)
+
+        self.has_ddl_statements = self._find_ddl_statements()
+        self.has_non_select_dml_statements = self._find_dml_statements()
+        self.used_tables = self._find_tables()
+
+    def _find_ddl_statements(self):
+        for statement in self.parsed_sql:
+            if len([x for x in statement.flatten() if x.ttype == sqlparse.tokens.DDL]):
+                return True
+
+        return False
+
+    def _find_tables(self):
+        tables = set()
+        for statement in self.parsed_sql:
+            tables.update(self.extract_table_names(statement.tokens))
+
+        return tables
+
+    def extract_table_names(self, tokens):
+        tables = set()
+        tokens = [t for t in tokens if t.ttype not in (sqlparse.tokens.Whitespace, sqlparse.tokens.Newline)]
+
+        for i in range(len(tokens)):
+            if tokens[i].is_group():
+                tables.update(self.extract_table_names(tokens[i].tokens))
+            else:
+                if tokens[i].ttype == sqlparse.tokens.Keyword and tokens[i].normalized in self.TABLE_SELECTION_KEYWORDS:
+                    if isinstance(tokens[i + 1], sqlparse.sql.Identifier):
+                        tables.add(tokens[i + 1].value)
+
+                    if isinstance(tokens[i + 1], sqlparse.sql.IdentifierList):
+                        tables.update(set([t.value for t in tokens[i+1].get_identifiers()]))
+        return tables
+
+    def _find_dml_statements(self):
+        for statement in self.parsed_sql:
+            for token in statement.flatten():
+                if token.ttype == sqlparse.tokens.DML and token.normalized != 'SELECT':
+                    return True
+
+        return False
+
+
+def slugify(s):
+    return re.sub('[^a-z0-9_\-]+', '-', s.lower())
+
+
 def gen_query_hash(sql):
     """Returns hash of the given query after stripping all comments, line breaks and multiple
     spaces, and lower casing all text.

redash/worker.py

@@ -0,0 +1,21 @@
+from celery import Celery
+from datetime import timedelta
+from redash import settings
+
+
+celery = Celery('redash',
+                broker=settings.CELERY_BROKER,
+                include='redash.tasks')
+
+celery.conf.update(CELERY_RESULT_BACKEND=settings.CELERY_BACKEND,
+                   CELERYBEAT_SCHEDULE={
+                       'refresh_queries': {
+                           'task': 'redash.tasks.refresh_queries',
+                           'schedule': timedelta(seconds=30)
+                       },
+                   },
+                   CELERY_TIMEZONE='UTC')
+
+
+if __name__ == '__main__':
+    celery.start()

redash/wsgi.py

@@ -0,0 +1,32 @@
+import json
+from flask import Flask, make_response
+from flask.ext.restful import Api
+
+from redash import settings, utils
+from redash.models import db
+
+__version__ = '0.4.0'
+
+app = Flask(__name__,
+            template_folder=settings.STATIC_ASSETS_PATH,
+            static_folder=settings.STATIC_ASSETS_PATH,
+            static_path='/static')
+
+
+api = Api(app)
+
+# configure our database
+settings.DATABASE_CONFIG.update({'threadlocals': True})
+app.config['DATABASE'] = settings.DATABASE_CONFIG
+db.init_app(app)
+
+from redash.authentication import setup_authentication
+auth = setup_authentication(app)
+
+@api.representation('application/json')
+def json_representation(data, code, headers=None):
+    resp = make_response(json.dumps(data, cls=utils.JSONEncoder), code)
+    resp.headers.extend(headers or {})
+    return resp
+
+from redash import controllers

requirements.txt

@@ -5,25 +5,21 @@ Flask-Login==0.2.9
 passlib==1.6.2
 Jinja2==2.7.2
 MarkupSafe==0.18
-WTForms==1.0.5
 Werkzeug==0.9.4
 aniso8601==0.82
-atfork==0.1.2
 blinker==1.3
-flask-peewee==0.6.5
 itsdangerous==0.23
 peewee==2.2.2
 psycopg2==2.5.1
 python-dateutil==2.1
 pytz==2013.9
-qr==0.6.0
 redis==2.7.5
 requests==2.2.0
-setproctitle==1.1.8
 six==1.5.2
 sqlparse==0.1.8
 wsgiref==0.1.2
-wtf-peewee==0.2.2
 Flask-Script==0.6.6
 honcho==0.5.0
 statsd==2.1.2
+gunicorn==18.0
+celery==3.1.11

tests/__init__.py

@@ -1,27 +1,21 @@
 import logging
 from unittest import TestCase
-from redash import settings, db, app
-import redash.models
-
-# TODO: this isn't pretty...
+from redash import settings
 settings.DATABASE_CONFIG = {
     'name': 'circle_test',
-    'engine': 'peewee.PostgresqlDatabase',
     'threadlocals': True
 }
-app.config['DATABASE'] = settings.DATABASE_CONFIG
-db.load_database()
+
+from redash import models
 
 logging.getLogger('peewee').setLevel(logging.INFO)
 
-for model in redash.models.all_models:
-    model._meta.database = db.database
-
 
 class BaseTestCase(TestCase):
     def setUp(self):
-        redash.models.create_db(True, True)
+        models.create_db(True, True)
+        models.init_db()
 
     def tearDown(self):
-        db.close_db(None)
-        redash.models.create_db(False, True)
+        models.db.close_db(None)
+        models.create_db(False, True)

tests/factories.py

@@ -41,7 +41,7 @@ class Sequence(object):
 
 user_factory = ModelFactory(redash.models.User,
                             name='John Doe', email=Sequence('test{}@example.com'),
-                            is_admin=False)
+                            groups=['default'])
 
 
 data_source_factory = ModelFactory(redash.models.DataSource,

tests/test_authentication.py

@@ -36,7 +36,7 @@ class TestCreateAndLoginUser(BaseTestCase):
     def test_creates_vaild_new_user(self):
         openid_user = ObjectDict({'email': 'test@example.com', 'name': 'Test User'})
 
-        with patch.multiple(settings, GOOGLE_APPS_DOMAIN='example.com', ADMINS=['admin@example.com']), \
+        with patch.multiple(settings, GOOGLE_APPS_DOMAIN='example.com'), \
              patch('redash.authentication.login_user') as login_user_mock:
 
             create_and_login_user(None, openid_user)
@@ -44,20 +44,6 @@ class TestCreateAndLoginUser(BaseTestCase):
             self.assertTrue(login_user_mock.called)
             user = models.User.get(models.User.email == openid_user.email)
 
-            self.assertFalse(user.is_admin)
-
-    def test_creates_vaild_new_user_and_sets_is_admin(self):
-        openid_user = ObjectDict({'email': 'admin@example.com', 'name': 'Test User'})
-
-        with patch.multiple(settings, GOOGLE_APPS_DOMAIN='example.com', ADMINS=['admin@example.com']), \
-             patch('redash.authentication.login_user') as login_user_mock:
-
-            create_and_login_user(None, openid_user)
-
-            self.assertTrue(login_user_mock.called)
-            user = models.User.get(models.User.email == openid_user.email)
-            self.assertTrue(user.is_admin)
-
     def test_ignores_invliad_user(self):
         user = ObjectDict({'email': 'test@whatever.com'})
 

tests/test_controllers.py

@@ -8,7 +8,8 @@ from mock import patch
 from tests import BaseTestCase
 from tests.factories import dashboard_factory, widget_factory, visualization_factory, query_factory, \
     query_result_factory, user_factory, data_source_factory
-from redash import app, models, settings
+from redash import models, settings
+from redash.wsgi import app
 from redash.utils import json_dumps
 from redash.authentication import sign
 
@@ -77,7 +78,7 @@ class IndexTest(BaseTestCase, AuthenticationTestMixin):
 
 class StatusTest(BaseTestCase):
     def test_returns_data_for_admin(self):
-        admin = user_factory.create(permissions=['admin'])
+        admin = user_factory.create(groups=['admin', 'default'])
         with app.test_client() as c, authenticated_user(c, user=admin):
             rv = c.get('/status.json')
             self.assertEqual(rv.status_code, 200)
@@ -182,6 +183,23 @@ class WidgetAPITest(BaseTestCase):
                            [rv4.json['widget']['id']]])
         self.assertEquals(rv4.json['new_row'], True)
 
+    def test_create_text_widget(self):
+        dashboard = dashboard_factory.create()
+
+        data = {
+            'visualization_id': None,
+            'text': 'Sample text.',
+            'dashboard_id': dashboard.id,
+            'options': {},
+            'width': 2
+        }
+
+        with app.test_client() as c, authenticated_user(c):
+            rv = json_request(c.post, '/api/widgets', data=data)
+
+        self.assertEquals(rv.status_code, 200)
+        self.assertEquals(rv.json['widget']['text'], 'Sample text.')
+
     def test_delete_widget(self):
         widget = widget_factory.create()
 
@@ -304,6 +322,7 @@ class JobAPITest(BaseTestCase, AuthenticationTestMixin):
 class CsvQueryResultAPITest(BaseTestCase, AuthenticationTestMixin):
     def setUp(self):
         super(CsvQueryResultAPITest, self).setUp()
+
         self.paths = []
         self.query_result = query_result_factory.create()
         self.query = query_factory.create()
@@ -375,6 +394,7 @@ class TestLogin(BaseTestCase):
             self.assertFalse(login_user_mock.called)
 
     def test_submit_correct_user_and_password(self):
+
         user = user_factory.create()
         user.hash_password('password')
         user.save()

tests/test_job.py

@@ -1,100 +0,0 @@
-# coding=utf-8
-
-import time
-from unittest import TestCase
-from mock import patch
-from redash.data.worker import Job
-from redash import redis_connection
-from redash.utils import gen_query_hash
-
-
-class TestJob(TestCase):
-    def setUp(self):
-        self.priority = 1
-        self.query = "SELECT 1"
-        self.query_hash = gen_query_hash(self.query)
-
-    def test_job_creation(self):
-        now = time.time()
-        with patch('time.time', return_value=now):
-            job = Job(redis_connection, query=self.query, priority=self.priority)
-        self.assertIsNotNone(job.id)
-        self.assertTrue(job.new_job)
-        self.assertEquals(0, job.wait_time)
-        self.assertEquals(0, job.query_time)
-        self.assertEquals(None, job.process_id)
-        self.assertEquals(Job.WAITING, job.status)
-        self.assertEquals(self.priority, job.priority)
-        self.assertEquals(self.query, job.query)
-        self.assertEquals(self.query_hash, job.query_hash)
-        self.assertIsNone(job.error)
-        self.assertIsNone(job.query_result_id)
-
-    def test_job_loading(self):
-        job = Job(redis_connection, query=self.query, priority=self.priority)
-        job.save()
-
-        loaded_job = Job.load(redis_connection, job.id)
-
-        self.assertFalse(loaded_job.new_job)
-
-        self.assertEquals(loaded_job.id, job.id)
-        self.assertEquals(loaded_job.wait_time, job.wait_time)
-        self.assertEquals(loaded_job.query_time, job.query_time)
-        self.assertEquals(loaded_job.process_id, job.process_id)
-        self.assertEquals(loaded_job.status, job.status)
-        self.assertEquals(loaded_job.priority, job.priority)
-        self.assertEquals(loaded_job.query_hash, job.query_hash)
-        self.assertEquals(loaded_job.query, job.query)
-        self.assertEquals(loaded_job.error, job.error)
-        self.assertEquals(loaded_job.query_result_id, job.query_result_id)
-
-    def test_update(self):
-        job = Job(redis_connection, query=self.query, priority=self.priority)
-
-        job.update(process_id=1)
-        self.assertEquals(1, job.process_id)
-        self.assertEquals(self.query, job.query)
-        self.assertEquals(self.priority, job.priority)
-
-    def test_processing(self):
-        job = Job(redis_connection, query=self.query, priority=self.priority)
-
-        updated_at = job.updated_at
-        now = time.time()+10
-        with patch('time.time', return_value=now):
-            job.processing(10)
-
-        job = Job.load(redis_connection, job.id)
-
-        self.assertEquals(10, job.process_id)
-        self.assertEquals(Job.PROCESSING, job.status)
-        self.assertEquals(now, job.updated_at)
-        self.assertEquals(now - updated_at, job.wait_time)
-
-    def test_done(self):
-        job = Job(redis_connection, query=self.query, priority=self.priority)
-
-        updated_at = job.updated_at
-        now = time.time()+10
-        with patch('time.time', return_value=now):
-            job.done(1, None)
-
-        job = Job.load(redis_connection, job.id)
-
-        self.assertEquals(Job.DONE, job.status)
-        self.assertEquals(1, job.query_result_id)
-        self.assertEquals(now, job.updated_at)
-        self.assertEquals(now - updated_at, job.query_time)
-        self.assertIsNone(job.error)
-
-    def test_unicode_serialization(self):
-        unicode_query = u"יוניקוד"
-        job = Job(redis_connection, query=unicode_query, priority=self.priority)
-
-        self.assertEquals(job.query, unicode_query)
-
-        job.save()
-        loaded_job = Job.load(redis_connection, job.id)
-        self.assertEquals(loaded_job.query, unicode_query)
-

tests/test_manager.py

@@ -1,149 +0,0 @@
-import datetime
-from mock import patch, call
-from tests import BaseTestCase
-from redash.data import worker
-from redash import data_manager, models
-from tests.factories import query_factory, query_result_factory, data_source_factory
-from redash.utils import gen_query_hash
-
-
-class TestManagerRefresh(BaseTestCase):
-    def test_enqueues_outdated_queries(self):
-        query = query_factory.create(ttl=60)
-        retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10)
-        query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query,
-                                                   query_hash=query.query_hash)
-        query.latest_query_data = query_result
-        query.save()
-
-        with patch('redash.data.Manager.add_job') as add_job_mock:
-            data_manager.refresh_queries()
-            add_job_mock.assert_called_with(query.query, worker.Job.LOW_PRIORITY, query.data_source)
-
-    def test_skips_fresh_queries(self):
-        query = query_factory.create(ttl=1200)
-        retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10)
-        query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query,
-                                                   query_hash=query.query_hash)
-
-        with patch('redash.data.Manager.add_job') as add_job_mock:
-            data_manager.refresh_queries()
-            self.assertFalse(add_job_mock.called)
-
-    def test_skips_queries_with_no_ttl(self):
-        query = query_factory.create(ttl=-1)
-        retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10)
-        query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query,
-                                                   query_hash=query.query_hash)
-
-        with patch('redash.data.Manager.add_job') as add_job_mock:
-            data_manager.refresh_queries()
-            self.assertFalse(add_job_mock.called)
-
-    def test_enqueues_query_only_once(self):
-        query = query_factory.create(ttl=60)
-        query2 = query_factory.create(ttl=60, query=query.query, query_hash=query.query_hash,
-                                      data_source=query.data_source)
-        retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10)
-        query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query,
-                                                   query_hash=query.query_hash)
-        query.latest_query_data = query_result
-        query2.latest_query_data = query_result
-        query.save()
-        query2.save()
-
-        with patch('redash.data.Manager.add_job') as add_job_mock:
-            data_manager.refresh_queries()
-            add_job_mock.assert_called_once_with(query.query, worker.Job.LOW_PRIORITY, query.data_source)
-
-    def test_enqueues_query_with_correct_data_source(self):
-        query = query_factory.create(ttl=60)
-        query2 = query_factory.create(ttl=60, query=query.query, query_hash=query.query_hash)
-        retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10)
-        query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query,
-                                                   query_hash=query.query_hash)
-        query.latest_query_data = query_result
-        query2.latest_query_data = query_result
-        query.save()
-        query2.save()
-
-        with patch('redash.data.Manager.add_job') as add_job_mock:
-            data_manager.refresh_queries()
-            add_job_mock.assert_has_calls([call(query2.query, worker.Job.LOW_PRIORITY, query2.data_source), call(query.query, worker.Job.LOW_PRIORITY, query.data_source)], any_order=True)
-            self.assertEquals(2, add_job_mock.call_count)
-
-    def test_enqueues_only_for_relevant_data_source(self):
-        query = query_factory.create(ttl=60)
-        query2 = query_factory.create(ttl=3600, query=query.query, query_hash=query.query_hash)
-        retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10)
-        query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query,
-                                                   query_hash=query.query_hash)
-        query.latest_query_data = query_result
-        query2.latest_query_data = query_result
-        query.save()
-        query2.save()
-
-        with patch('redash.data.Manager.add_job') as add_job_mock:
-            data_manager.refresh_queries()
-            add_job_mock.assert_called_once_with(query.query, worker.Job.LOW_PRIORITY, query.data_source)
-
-
-class TestManagerStoreResults(BaseTestCase):
-    def setUp(self):
-        super(TestManagerStoreResults, self).setUp()
-        self.data_source = data_source_factory.create()
-        self.query = "SELECT 1"
-        self.query_hash = gen_query_hash(self.query)
-        self.runtime = 123
-        self.utcnow = datetime.datetime.utcnow()
-        self.data = "data"
-
-    def test_stores_the_result(self):
-        query_result_id = data_manager.store_query_result(self.data_source.id, self.query,
-                                                          self.data, self.runtime, self.utcnow)
-
-        query_result = models.QueryResult.get_by_id(query_result_id)
-
-        self.assertEqual(query_result.data, self.data)
-        self.assertEqual(query_result.runtime, self.runtime)
-        self.assertEqual(query_result.retrieved_at, self.utcnow)
-        self.assertEqual(query_result.query, self.query)
-        self.assertEqual(query_result.query_hash, self.query_hash)
-        self.assertEqual(query_result.data_source, self.data_source)
-
-    def test_updates_existing_queries(self):
-        query1 = query_factory.create(query=self.query, data_source=self.data_source)
-        query2 = query_factory.create(query=self.query, data_source=self.data_source)
-        query3 = query_factory.create(query=self.query, data_source=self.data_source)
-
-        query_result_id = data_manager.store_query_result(self.data_source.id, self.query,
-                                                          self.data, self.runtime, self.utcnow)
-
-        self.assertEqual(models.Query.get_by_id(query1.id)._data['latest_query_data'], query_result_id)
-        self.assertEqual(models.Query.get_by_id(query2.id)._data['latest_query_data'], query_result_id)
-        self.assertEqual(models.Query.get_by_id(query3.id)._data['latest_query_data'], query_result_id)
-
-    def test_doesnt_update_queries_with_different_hash(self):
-        query1 = query_factory.create(query=self.query, data_source=self.data_source)
-        query2 = query_factory.create(query=self.query, data_source=self.data_source)
-        query3 = query_factory.create(query=self.query + "123", data_source=self.data_source)
-
-        query_result_id = data_manager.store_query_result(self.data_source.id, self.query,
-                                                          self.data, self.runtime, self.utcnow)
-
-        self.assertEqual(models.Query.get_by_id(query1.id)._data['latest_query_data'], query_result_id)
-        self.assertEqual(models.Query.get_by_id(query2.id)._data['latest_query_data'], query_result_id)
-        self.assertNotEqual(models.Query.get_by_id(query3.id)._data['latest_query_data'], query_result_id)
-
-
-    def test_doesnt_update_queries_with_different_data_source(self):
-        query1 = query_factory.create(query=self.query, data_source=self.data_source)
-        query2 = query_factory.create(query=self.query, data_source=self.data_source)
-        query3 = query_factory.create(query=self.query, data_source=data_source_factory.create())
-
-        query_result_id = data_manager.store_query_result(self.data_source.id, self.query,
-                                                          self.data, self.runtime, self.utcnow)
-
-        self.assertEqual(models.Query.get_by_id(query1.id)._data['latest_query_data'], query_result_id)
-        self.assertEqual(models.Query.get_by_id(query2.id)._data['latest_query_data'], query_result_id)
-        self.assertNotEqual(models.Query.get_by_id(query3.id)._data['latest_query_data'], query_result_id)

tests/test_models.py

@@ -2,6 +2,7 @@ import datetime
 from tests import BaseTestCase
 from redash import models
 from factories import dashboard_factory, query_factory, data_source_factory, query_result_factory
+from redash.utils import gen_query_hash
 
 
 class DashboardTest(BaseTestCase):
@@ -80,4 +81,71 @@ class QueryResultTest(BaseTestCase):
 
         found_query_result = models.QueryResult.get_latest(qr.data_source, qr.query, 60)
 
-        self.assertEqual(found_query_result.id, qr.id)
+        self.assertEqual(found_query_result.id, qr.id)
+
+    def test_get_latest_returns_the_last_cached_result_for_negative_ttl(self):
+        yesterday = datetime.datetime.now() + datetime.timedelta(days=-100)
+        very_old = query_result_factory.create(retrieved_at=yesterday)
+
+        yesterday = datetime.datetime.now() + datetime.timedelta(days=-1)
+        qr = query_result_factory.create(retrieved_at=yesterday)
+        found_query_result = models.QueryResult.get_latest(qr.data_source, qr.query, -1)
+
+        self.assertEqual(found_query_result.id, qr.id)
+
+class TestQueryResultStoreResult(BaseTestCase):
+    def setUp(self):
+        super(TestQueryResultStoreResult, self).setUp()
+        self.data_source = data_source_factory.create()
+        self.query = "SELECT 1"
+        self.query_hash = gen_query_hash(self.query)
+        self.runtime = 123
+        self.utcnow = datetime.datetime.utcnow()
+        self.data = "data"
+
+    def test_stores_the_result(self):
+        query_result = models.QueryResult.store_result(self.data_source.id, self.query_hash, self.query,
+                                                          self.data, self.runtime, self.utcnow)
+
+        self.assertEqual(query_result.data, self.data)
+        self.assertEqual(query_result.runtime, self.runtime)
+        self.assertEqual(query_result.retrieved_at, self.utcnow)
+        self.assertEqual(query_result.query, self.query)
+        self.assertEqual(query_result.query_hash, self.query_hash)
+        self.assertEqual(query_result.data_source, self.data_source)
+
+    def test_updates_existing_queries(self):
+        query1 = query_factory.create(query=self.query, data_source=self.data_source)
+        query2 = query_factory.create(query=self.query, data_source=self.data_source)
+        query3 = query_factory.create(query=self.query, data_source=self.data_source)
+
+        query_result = models.QueryResult.store_result(self.data_source.id, self.query_hash, self.query, self.data,
+                                                       self.runtime, self.utcnow)
+
+        self.assertEqual(models.Query.get_by_id(query1.id)._data['latest_query_data'], query_result.id)
+        self.assertEqual(models.Query.get_by_id(query2.id)._data['latest_query_data'], query_result.id)
+        self.assertEqual(models.Query.get_by_id(query3.id)._data['latest_query_data'], query_result.id)
+
+    def test_doesnt_update_queries_with_different_hash(self):
+        query1 = query_factory.create(query=self.query, data_source=self.data_source)
+        query2 = query_factory.create(query=self.query, data_source=self.data_source)
+        query3 = query_factory.create(query=self.query + "123", data_source=self.data_source)
+
+        query_result = models.QueryResult.store_result(self.data_source.id, self.query_hash, self.query, self.data,
+                                                       self.runtime, self.utcnow)
+
+        self.assertEqual(models.Query.get_by_id(query1.id)._data['latest_query_data'], query_result.id)
+        self.assertEqual(models.Query.get_by_id(query2.id)._data['latest_query_data'], query_result.id)
+        self.assertNotEqual(models.Query.get_by_id(query3.id)._data['latest_query_data'], query_result.id)
+
+    def test_doesnt_update_queries_with_different_data_source(self):
+        query1 = query_factory.create(query=self.query, data_source=self.data_source)
+        query2 = query_factory.create(query=self.query, data_source=self.data_source)
+        query3 = query_factory.create(query=self.query, data_source=data_source_factory.create())
+
+        query_result = models.QueryResult.store_result(self.data_source.id, self.query_hash, self.query, self.data,
+                                                       self.runtime, self.utcnow)
+
+        self.assertEqual(models.Query.get_by_id(query1.id)._data['latest_query_data'], query_result.id)
+        self.assertEqual(models.Query.get_by_id(query2.id)._data['latest_query_data'], query_result.id)
+        self.assertNotEqual(models.Query.get_by_id(query3.id)._data['latest_query_data'], query_result.id)

tests/test_refresh_queries.py

@@ -0,0 +1,89 @@
+import datetime
+from mock import patch, call
+from tests import BaseTestCase
+from tests.factories import query_factory, query_result_factory
+from redash.tasks import refresh_queries
+
+
+# TODO: this test should be split into two:
+# 1. tests for Query.outdated_queries method
+# 2. test for the refresh_query task
+class TestRefreshQueries(BaseTestCase):
+    def test_enqueues_outdated_queries(self):
+        query = query_factory.create(ttl=60)
+        retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10)
+        query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query,
+                                                   query_hash=query.query_hash)
+        query.latest_query_data = query_result
+        query.save()
+
+        with patch('redash.tasks.QueryTask.add_task') as add_job_mock:
+            refresh_queries()
+            add_job_mock.assert_called_with(query.query, query.data_source, scheduled=True)
+
+    def test_skips_fresh_queries(self):
+        query = query_factory.create(ttl=1200)
+        retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10)
+        query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query,
+                                                   query_hash=query.query_hash)
+
+        with patch('redash.tasks.QueryTask.add_task') as add_job_mock:
+            refresh_queries()
+            self.assertFalse(add_job_mock.called)
+
+    def test_skips_queries_with_no_ttl(self):
+        query = query_factory.create(ttl=-1)
+        retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10)
+        query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query,
+                                                   query_hash=query.query_hash)
+
+        with patch('redash.tasks.QueryTask.add_task') as add_job_mock:
+            refresh_queries()
+            self.assertFalse(add_job_mock.called)
+
+    def test_enqueues_query_only_once(self):
+        query = query_factory.create(ttl=60)
+        query2 = query_factory.create(ttl=60, query=query.query, query_hash=query.query_hash,
+                                      data_source=query.data_source)
+        retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10)
+        query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query,
+                                                   query_hash=query.query_hash)
+        query.latest_query_data = query_result
+        query2.latest_query_data = query_result
+        query.save()
+        query2.save()
+
+        with patch('redash.tasks.QueryTask.add_task') as add_job_mock:
+            refresh_queries()
+            add_job_mock.assert_called_once_with(query.query, query.data_source, scheduled=True)
+
+    def test_enqueues_query_with_correct_data_source(self):
+        query = query_factory.create(ttl=60)
+        query2 = query_factory.create(ttl=60, query=query.query, query_hash=query.query_hash)
+        retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10)
+        query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query,
+                                                   query_hash=query.query_hash)
+        query.latest_query_data = query_result
+        query2.latest_query_data = query_result
+        query.save()
+        query2.save()
+
+        with patch('redash.tasks.QueryTask.add_task') as add_job_mock:
+            refresh_queries()
+            add_job_mock.assert_has_calls([call(query2.query, query2.data_source, scheduled=True), call(query.query, query.data_source, scheduled=True)], any_order=True)
+            self.assertEquals(2, add_job_mock.call_count)
+
+    def test_enqueues_only_for_relevant_data_source(self):
+        query = query_factory.create(ttl=60)
+        query2 = query_factory.create(ttl=3600, query=query.query, query_hash=query.query_hash)
+        retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10)
+        query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query,
+                                                   query_hash=query.query_hash)
+        query.latest_query_data = query_result
+        query2.latest_query_data = query_result
+        query.save()
+        query2.save()
+
+        with patch('redash.tasks.QueryTask.add_task') as add_job_mock:
+            refresh_queries()
+            add_job_mock.assert_called_once_with(query.query, query.data_source, scheduled=True)

tests/test_sql_meta_data.py

@@ -0,0 +1,28 @@
+from redash.utils import SQLMetaData
+from unittest import TestCase
+
+
+class TestSQLMetaData(TestCase):
+    def test_simple_select(self):
+        metadata = SQLMetaData("SELECT t FROM test")
+        self.assertEquals(metadata.used_tables, set(("test",)))
+        self.assertFalse(metadata.has_ddl_statements)
+        self.assertFalse(metadata.has_non_select_dml_statements)
+
+    def test_multiple_select(self):
+        metadata = SQLMetaData("SELECT t FROM test, test2 WHERE t > 1; SELECT a, b, c FROM testing as tbl")
+        self.assertEquals(metadata.used_tables, set(("test", "test2", "testing")))
+        self.assertFalse(metadata.has_ddl_statements)
+        self.assertFalse(metadata.has_non_select_dml_statements)
+
+    def test_detects_ddl(self):
+        metadata = SQLMetaData("SELECT t FROM test; DROP TABLE test")
+        self.assertEquals(metadata.used_tables, set(("test",)))
+        self.assertTrue(metadata.has_ddl_statements)
+        self.assertFalse(metadata.has_non_select_dml_statements)
+
+    def test_detects_dml(self):
+        metadata = SQLMetaData("SELECT t FROM test; DELETE * FROM test")
+        self.assertEquals(metadata.used_tables, set(("test",)))
+        self.assertFalse(metadata.has_ddl_statements)
+        self.assertTrue(metadata.has_non_select_dml_statements)