Comment out filters url sync tests.

Disable filters url syncing

.gitignore

@@ -4,9 +4,17 @@
 .coverage
 rd_ui/dist
 .DS_Store
+celerybeat-schedule*
+.#*
+\#*#
+*~
 
 # Vagrant related
 .vagrant
 Berksfile.lock
 redash/dump.rdb
 .env
+.ruby-version
+venv
+
+dump.rdb

Makefile

@@ -1,19 +1,22 @@
 NAME=redash
 VERSION=`python ./manage.py version`
 FULL_VERSION=$(VERSION)+b$(CIRCLE_BUILD_NUM)
-FILENAME=$(CIRCLE_ARTIFACTS)/$(NAME).$(FULL_VERSION).tar.gz
+# VERSION gets evaluated every time it's referenced, therefore we need to use VERSION here instead of FULL_VERSION.
+FILENAME=$(CIRCLE_ARTIFACTS)/$(NAME).$(VERSION).tar.gz
 
 deps:
 	cd rd_ui && npm install
-	cd rd_ui && npm install grunt-cli bower
+	cd rd_ui && npm install -g bower grunt-cli
 	cd rd_ui && bower install
 	cd rd_ui && grunt build
 
 pack:
+	sed -ri "s/^__version__ = '([0-9.]*)'/__version__ = '$(FULL_VERSION)'/" redash/__init__.py
 	tar -zcv -f $(FILENAME) --exclude=".git*" --exclude="*.pyc" --exclude="*.pyo" --exclude="venv" --exclude="rd_ui/node_modules" --exclude="rd_ui/dist/bower_components" --exclude="rd_ui/app" *
 
 upload:
-	python bin/upload_version.py $(FULL_VERSION) $(FILENAME)
+	python bin/upload_version.py $(VERSION) $(FILENAME)
 
 test:
 	nosetests --with-coverage --cover-package=redash tests/*.py
+	cd rd_ui && grunt test

Procfile.dev

@@ -1,2 +1,2 @@
 web: ./manage.py runserver -p $PORT
-worker: ./manage.py runworkers
+worker: ./bin/run celery worker --app=redash.worker --beat -Qqueries,celery,scheduled_queries

Procfile.heroku

@@ -1,2 +1,2 @@
 web: ./manage.py runserver -p $PORT --host 0.0.0.0 -d -r
-worker: ./manage.py runworkers
+worker: ./bin/run celery worker --app=redash.worker --beat -Qqueries,celery,scheduled_queries

bin/latest_release.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+import sys
+import requests
+
+if __name__ == '__main__':
+    response = requests.get('https://api.github.com/repos/EverythingMe/redash/releases')
+
+    if response.status_code != 200:
+        exit("Failed getting releases (status code: %s)." % response.status_code)
+
+    sorted_releases = sorted(response.json(), key=lambda release: release['id'], reverse=True)
+
+    latest_release = sorted_releases[0]
+    asset_url = latest_release['assets'][0]['url']
+    filename = latest_release['assets'][0]['name']
+
+    wget_command = 'wget --header="Accept: application/octet-stream" %s -O %s' % (asset_url, filename)
+
+    if '--url-only' in sys.argv:
+        print asset_url
+    elif '--wget' in sys.argv:
+        print wget_command
+    else:
+        print "Latest release: %s" % latest_release['tag_name']
+        print latest_release['body']
+
+        print "\nTarball URL: %s" % asset_url
+        print 'wget: %s' % (wget_command)
+
+

bin/run

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+# Ideally I would use stdin with source, but in older bash versions this
+# wasn't supported properly.
+TEMP_ENV_FILE=`mktemp /tmp/redash_env.XXXXXX`
+sed 's/^REDASH/export REDASH/' .env > $TEMP_ENV_FILE
+source $TEMP_ENV_FILE
+rm $TEMP_ENV_FILE
+
+exec "$@"

bin/test_multithreading.py

@@ -0,0 +1,63 @@
+"""
+Script to test concurrency (multithreading/multiprocess) issues with the workers. Use with caution.
+"""
+import json
+import atfork
+atfork.monkeypatch_os_fork_functions()
+import atfork.stdlib_fixer
+atfork.stdlib_fixer.fix_logging_module()
+
+import time
+from redash.data import worker
+from redash import models, data_manager, redis_connection
+
+if __name__ == '__main__':
+    models.create_db(True, False)
+
+    print "Creating data source..."
+    data_source = models.DataSource.create(name="Concurrency", type="pg", options="dbname=postgres")
+
+    print "Clear jobs/hashes:"
+    redis_connection.delete("jobs")
+    query_hashes = redis_connection.keys("query_hash_*")
+    if query_hashes:
+        redis_connection.delete(*query_hashes)
+
+    starting_query_results_count = models.QueryResult.select().count()
+    jobs_count = 5000
+    workers_count = 10
+
+    print "Creating jobs..."
+    for i in xrange(jobs_count):
+        query = "SELECT {}".format(i)
+        print "Inserting: {}".format(query)
+        data_manager.add_job(query=query, priority=worker.Job.LOW_PRIORITY,
+                             data_source=data_source)
+
+    print "Starting workers..."
+    workers = data_manager.start_workers(workers_count)
+
+    print "Waiting for jobs to be done..."
+    keep_waiting = True
+    while keep_waiting:
+        results_count = models.QueryResult.select().count() - starting_query_results_count
+        print "QueryResults: {}".format(results_count)
+        time.sleep(5)
+        if results_count == jobs_count:
+            print "Yay done..."
+            keep_waiting = False
+
+    data_manager.stop_workers()
+
+    qr_count = 0
+    for qr in models.QueryResult.select():
+        number = int(qr.query.split()[1])
+        data_number = json.loads(qr.data)['rows'][0].values()[0]
+
+        if number != data_number:
+            print "Oops? {} != {} ({})".format(number, data_number, qr.id)
+        qr_count += 1
+
+    print "Verified {} query results.".format(qr_count)
+
+    print "Done."

bin/upload_version.py

@@ -3,30 +3,44 @@ import os
 import sys
 import json
 import requests
+import subprocess
+
+
+def capture_output(command):
+    proc = subprocess.Popen(command, stdout=subprocess.PIPE)
+    return proc.stdout.read()
+
 
 if __name__ == '__main__':
-  version = sys.argv[1]
-  filepath = sys.argv[2]
-  filename = filepath.split('/')[-1]
-  github_token = os.environ['GITHUB_TOKEN']
-  auth = (github_token, 'x-oauth-basic')
-  commit_sha = os.environ['CIRCLE_SHA1']
+    version = sys.argv[1]
+    filepath = sys.argv[2]
+    filename = filepath.split('/')[-1]
+    github_token = os.environ['GITHUB_TOKEN']
+    auth = (github_token, 'x-oauth-basic')
+    commit_sha = os.environ['CIRCLE_SHA1']
 
-  params = json.dumps({
-    'tag_name': 'v{0}'.format(version),
-    'name': 're:dash v{0}'.format(version),
-    'target_commitish': commit_sha,
-    'prerelease': True
-  })
+    commit_body = capture_output(["git", "log", "--format=%b", "-n", "1", commit_sha])
+    file_md5_checksum = capture_output(["md5sum", filepath]).split()[0]
+    file_sha256_checksum = capture_output(["sha256sum", filepath]).split()[0]
+    version_body = "%s\n\nMD5: %s\nSHA256: %s" % (commit_body, file_md5_checksum, file_sha256_checksum)
 
-  response = requests.post('https://api.github.com/repos/everythingme/redash/releases',
-                            data=params,
-                            auth=auth)
+    params = json.dumps({
+        'tag_name': 'v{0}'.format(version),
+        'name': 're:dash v{0}'.format(version),
+        'body': version_body,
+        'target_commitish': commit_sha,
+        'prerelease': True
+    })
 
-  upload_url = response.json()['upload_url']
-  upload_url = upload_url.replace('{?name}', '')
+    response = requests.post('https://api.github.com/repos/everythingme/redash/releases',
+                             data=params,
+                             auth=auth)
 
-  with open(filepath) as file_content:
-    headers = {'Content-Type': 'application/gzip'}
-    response = requests.post(upload_url, file_content, params={'name': filename}, auth=auth, headers=headers, verify=False)
+    upload_url = response.json()['upload_url']
+    upload_url = upload_url.replace('{?name}', '')
+
+    with open(filepath) as file_content:
+        headers = {'Content-Type': 'application/gzip'}
+        response = requests.post(upload_url, file_content, params={'name': filename}, auth=auth,
+                                 headers=headers, verify=False)
 

circle.yml

@@ -1,14 +1,14 @@
 machine:
   node:
     version:
-      0.10.22
+      0.10.24
   python:
     version:
       2.7.3
 dependencies:
   pre:
     - make deps
-    - pip install requests coverage nose
+    - pip install -r dev_requirements.txt
     - pip install -r requirements.txt
   cache_directories:
     - rd_ui/node_modules/
@@ -17,7 +17,7 @@ test:
   override:
     - make test
   post:
-    - make pack 
+    - make pack
 deployment:
   github:
     branch: master

dev_requirements.txt

@@ -1,2 +1,3 @@
 nose==1.3.0
 coverage==3.7.1
+mock==1.0.1

manage.py

@@ -2,18 +2,17 @@
 """
 CLI to manage redash.
 """
-import atfork
-atfork.monkeypatch_os_fork_functions()
-import atfork.stdlib_fixer
-atfork.stdlib_fixer.fix_logging_module()
+import datetime
+from flask.ext.script import Manager, prompt_pass
 
-import logging
-import time
-from redash import settings, app, db, models, data_manager, __version__
-from flask.ext.script import Manager
+from redash import settings, models, __version__
+from redash.wsgi import app
+from redash.import_export import import_manager
 
 manager = Manager(app)
 database_manager = Manager(help="Manages the database (create/drop tables).")
+users_manager = Manager(help="Users management commands.")
+data_sources_manager = Manager(help="Data sources management commands.")
 
 @manager.command
 def version():
@@ -23,38 +22,74 @@ def version():
 
 @manager.command
 def runworkers():
-    """Starts the re:dash query executors/workers."""
+    """Prints deprecation warning."""
+    print "** This command is deprecated. Please use Celery's CLI to control the workers. **"
 
-    try:
-        old_workers = data_manager.redis_connection.smembers('workers')
-        data_manager.redis_connection.delete('workers')
-
-        logging.info("Cleaning old workers: %s", old_workers)
-
-        data_manager.start_workers(settings.WORKERS_COUNT, settings.CONNECTION_ADAPTER, settings.CONNECTION_STRING)
-        logging.info("Workers started.")
-
-        while True:
-            try:
-                data_manager.refresh_queries()
-            except Exception as e:
-                logging.error("Something went wrong with refreshing queries...")
-                logging.exception(e)
-            time.sleep(60)
-    except KeyboardInterrupt:
-        logging.warning("Exiting; waiting for threads")
-        data_manager.stop_workers()
 
 @manager.shell
 def make_shell_context():
+    from redash.models import db
     return dict(app=app, db=db, models=models)
 
+@manager.command
+def check_settings():
+    from types import ModuleType
+
+    for name in dir(settings):
+        item = getattr(settings, name)
+        if not callable(item) and not name.startswith("__") and not isinstance(item, ModuleType):
+            print "{} = {}".format(name, item)
+
+@manager.command
+def import_events(events_file):
+    import json
+    from collections import Counter
+
+    count = Counter()
+
+    with open(events_file) as f:
+        for line in f:
+            try:
+                event = json.loads(line)
+
+                user = event.pop('user_id')
+                action = event.pop('action')
+                object_type = event.pop('object_type')
+                object_id = event.pop('object_id', None)
+
+                if object_id == 'dashboard' and object_type == 'dashboard':
+                    count['bad dashboard id'] += 1
+                    continue
+
+                created_at = datetime.datetime.utcfromtimestamp(event.pop('timestamp'))
+                additional_properties = json.dumps(event)
+
+                models.Event.create(user=user, action=action, object_type=object_type, object_id=object_id,
+                                    additional_properties=additional_properties, created_at=created_at)
+
+                count['imported'] += 1
+
+            except Exception as ex:
+                print "Failed importing line:"
+                print line
+                print ex.message
+                count[ex.message] += 1
+                count['failed'] += 1
+
+                models.db.close_db(None)
+
+    for k, v in count.iteritems():
+        print k
+        print v
+
+
 @database_manager.command
 def create_tables():
     """Creates the database tables."""
-    from redash.models import create_db
+    from redash.models import create_db, init_db
 
     create_db(True, False)
+    init_db()
 
 @database_manager.command
 def drop_tables():
@@ -63,11 +98,72 @@ def drop_tables():
 
     create_db(False, True)
 
+
+@users_manager.option('email', help="User's email")
+@users_manager.option('name', help="User's full name")
+@users_manager.option('--admin', dest='is_admin', action="store_true", default=False, help="set user as admin")
+@users_manager.option('--google', dest='google_auth', action="store_true", default=False, help="user uses Google Auth to login")
+@users_manager.option('--password', dest='password', default=None, help="Password for users who don't use Google Auth (leave blank for prompt).")
+@users_manager.option('--groups', dest='groups', default=models.User.DEFAULT_GROUPS, help="Comma seperated list of groups (leave blank for default).")
+def create(email, name, groups, is_admin=False, google_auth=False, password=None):
+    print "Creating user (%s, %s)..." % (email, name)
+    print "Admin: %r" % is_admin
+    print "Login with Google Auth: %r\n" % google_auth
+    if isinstance(groups, basestring):
+        groups= groups.split(',')
+        groups.remove('') # in case it was empty string
+
+    if is_admin:
+        groups += ['admin']
+
+    user = models.User(email=email, name=name, groups=groups)
+    if not google_auth:
+        password = password or prompt_pass("Password")
+        user.hash_password(password)
+
+    try:
+        user.save()
+    except Exception, e:
+        print "Failed creating user: %s" % e.message
+
+
+@users_manager.option('email', help="email address of user to delete")
+def delete(email):
+    deleted_count = models.User.delete().where(models.User.email == email).execute()
+    print "Deleted %d users." % deleted_count
+
+@data_sources_manager.command
+def import_from_settings(name=None):
+    """Import data source from settings (env variables)."""
+    name = name or "Default"
+    data_source = models.DataSource.create(name=name,
+                                           type=settings.CONNECTION_ADAPTER,
+                                           options=settings.CONNECTION_STRING)
+
+    print "Imported data source from settings (id={}).".format(data_source.id)
+
+
+@data_sources_manager.command
+def list():
+    """List currently configured data sources"""
+    for ds in models.DataSource.select():
+        print "Name: {}\nType: {}\nOptions: {}".format(ds.name, ds.type, ds.options)
+
+@data_sources_manager.command
+def new(name, type, options):
+    """Create new data source"""
+    # TODO: validate it's a valid type and in the future, validate the options.
+    print "Creating {} data source ({}) with options:\n{}".format(type, name, options)
+    data_source = models.DataSource.create(name=name,
+                                           type=type,
+                                           options=options)
+    print "Id: {}".format(data_source.id)
+
+
 manager.add_command("database", database_manager)
+manager.add_command("users", users_manager)
+manager.add_command("import", import_manager)
+manager.add_command("ds", data_sources_manager)
 
 if __name__ == '__main__':
-    channel = logging.StreamHandler()
-    logging.getLogger().addHandler(channel)
-    logging.getLogger().setLevel(settings.LOG_LEVEL)
-
     manager.run()

migrations/add_global_filters_to_dashboard.py

@@ -0,0 +1,12 @@
+from playhouse.migrate import Migrator
+from redash import models
+from redash.models import db
+
+
+if __name__ == '__main__':
+    db.connect_db()
+    migrator = Migrator(db.database)
+    with db.database.transaction():
+        migrator.add_column(models.Dashboard, models.Dashboard.dashboard_filters_enabled, 'dashboard_filters_enabled')
+
+    db.close_db(None)

migrations/add_password_to_users.py

@@ -0,0 +1,12 @@
+from playhouse.migrate import Migrator
+from redash import db
+from redash import models
+
+
+if __name__ == '__main__':
+    db.connect_db()
+    migrator = Migrator(db.database)
+    with db.database.transaction():
+        migrator.add_column(models.User, models.User.password_hash, 'password_hash')
+
+    db.close_db(None)

migrations/add_permissions_to_user.py

@@ -0,0 +1,13 @@
+from playhouse.migrate import Migrator
+from redash import db
+from redash import models
+
+
+if __name__ == '__main__':
+    db.connect_db()
+    migrator = Migrator(db.database)
+    with db.database.transaction():
+        migrator.add_column(models.User, models.User.permissions, 'permissions')
+        models.User.update(permissions=['admin'] + models.User.DEFAULT_PERMISSIONS).where(models.User.is_admin == True).execute()
+
+    db.close_db(None)

migrations/add_queue_name_to_data_source.py

@@ -0,0 +1,13 @@
+from playhouse.migrate import Migrator
+from redash.models import db
+from redash import models
+
+
+if __name__ == '__main__':
+    db.connect_db()
+    migrator = Migrator(db.database)
+    with db.database.transaction():
+        migrator.add_column(models.DataSource, models.DataSource.queue_name, 'queue_name')
+        migrator.add_column(models.DataSource, models.DataSource.scheduled_queue_name, 'scheduled_queue_name')
+
+    db.close_db(None)

migrations/add_text_to_widgets.py

@@ -0,0 +1,13 @@
+from playhouse.migrate import Migrator
+from redash.models import db
+from redash import models
+
+
+if __name__ == '__main__':
+    db.connect_db()
+    migrator = Migrator(db.database)
+    with db.database.transaction():
+        migrator.add_column(models.Widget, models.Widget.text, 'text')
+        migrator.set_nullable(models.Widget, models.Widget.visualization, True)
+
+    db.close_db(None)

migrations/add_view_query_permission.py

@@ -0,0 +1,13 @@
+import peewee
+from redash import db
+from redash import models
+
+
+if __name__ == '__main__':
+    db.connect_db()
+
+    previous_default_permissions = models.User.DEFAULT_PERMISSIONS[:]
+    previous_default_permissions.remove('view_query')
+    models.User.update(permissions=peewee.fn.array_append(models.User.permissions, 'view_query')).where(peewee.SQL("'view_source' = any(permissions)")).execute()
+
+    db.close_db(None)

migrations/create_activity_log.py

@@ -0,0 +1,11 @@
+from redash import db
+from redash import models
+
+if __name__ == '__main__':
+    db.connect_db()
+
+    if not models.ActivityLog.table_exists():
+        print "Creating activity_log table..."
+        models.ActivityLog.create_table()
+
+    db.close_db(None)

migrations/create_data_sources.py

@@ -0,0 +1,48 @@
+import logging
+import peewee
+from playhouse.migrate import Migrator
+from redash import db
+from redash import models
+from redash import settings
+
+if __name__ == '__main__':
+    db.connect_db()
+
+    if not models.DataSource.table_exists():
+        print "Creating data_sources table..."
+        models.DataSource.create_table()
+
+        default_data_source = models.DataSource.create(name="Default",
+                                                       type=settings.CONNECTION_ADAPTER,
+                                                       options=settings.CONNECTION_STRING)
+    else:
+        default_data_source = models.DataSource.select().first()
+
+    migrator = Migrator(db.database)
+    models.Query.data_source.null = True
+    models.QueryResult.data_source.null = True
+    try:
+        with db.database.transaction():
+            migrator.add_column(models.Query, models.Query.data_source, "data_source_id")
+    except peewee.ProgrammingError:
+        print "Failed to create data_source_id column -- assuming it already exists"
+
+    try:
+        with db.database.transaction():
+            migrator.add_column(models.QueryResult, models.QueryResult.data_source, "data_source_id")
+    except peewee.ProgrammingError:
+        print "Failed to create data_source_id column -- assuming it already exists"
+
+    print "Updating data source to existing one..."
+    models.Query.update(data_source=default_data_source.id).execute()
+    models.QueryResult.update(data_source=default_data_source.id).execute()
+
+    with db.database.transaction():
+        print "Setting data source to non nullable..."
+        migrator.set_nullable(models.Query, models.Query.data_source, False)
+
+    with db.database.transaction():
+        print "Setting data source to non nullable..."
+        migrator.set_nullable(models.QueryResult, models.QueryResult.data_source, False)
+
+    db.close_db(None)

migrations/create_events.py

@@ -0,0 +1,12 @@
+from redash.models import db
+from redash import models
+
+
+if __name__ == '__main__':
+    db.connect_db()
+
+    if not models.Event.table_exists():
+        print "Creating events table..."
+        models.Event.create_table()
+
+    db.close_db(None)

migrations/permissions_migration.py

@@ -0,0 +1,29 @@
+import peewee
+from playhouse.migrate import Migrator
+from redash import models
+from redash.models import db
+
+
+if __name__ == '__main__':
+    db.connect_db()
+    migrator = Migrator(db.database)
+    
+    if not models.Group.table_exists():
+        print "Creating groups table..."
+        models.Group.create_table()
+    
+    with db.database.transaction():
+        models.Group.insert(name='admin', permissions=['admin'], tables=['*']).execute()
+        models.Group.insert(name='api', permissions=['view_query'], tables=['*']).execute()
+        models.Group.insert(name='default', permissions=models.Group.DEFAULT_PERMISSIONS, tables=['*']).execute()
+
+        migrator.add_column(models.User, models.User.groups, 'groups')
+        
+        models.User.update(groups=['admin', 'default']).where(peewee.SQL("is_admin = true")).execute()
+        models.User.update(groups=['admin', 'default']).where(peewee.SQL("'admin' = any(permissions)")).execute()
+        models.User.update(groups=['default']).where(peewee.SQL("is_admin = false")).execute()
+
+        migrator.drop_column(models.User, 'permissions')
+        migrator.drop_column(models.User, 'is_admin')
+
+    db.close_db(None)

rd_ui/.travis.yml

@@ -1,6 +1,5 @@
 language: node_js
 node_js:
-  - '0.8'
   - '0.10'
 before_script:
   - 'npm install -g bower grunt-cli'

rd_ui/Gruntfile.js

@@ -1,10 +1,5 @@
-// Generated on 2013-08-25 using generator-angular 0.4.0
+// Generated on 2014-07-30 using generator-angular 0.9.2
 'use strict';
-var LIVERELOAD_PORT = 35729;
-var lrSnippet = require('connect-livereload')({ port: LIVERELOAD_PORT });
-var mountFolder = function (connect, dir) {
-  return connect.static(require('path').resolve(dir));
-};
 
 // # Globbing
 // for performance reasons we're only matching one level down:
@@ -13,48 +8,148 @@ var mountFolder = function (connect, dir) {
 // 'test/spec/**/*.js'
 
 module.exports = function (grunt) {
+
+  // Load grunt tasks automatically
   require('load-grunt-tasks')(grunt);
+
+  // Time how long tasks take. Can help when optimizing build times
   require('time-grunt')(grunt);
 
-  // configurable paths
-  var yeomanConfig = {
-    app: 'app',
+  // Configurable paths for the application
+  var appConfig = {
+    app: require('./bower.json').appPath || 'app',
     dist: 'dist'
   };
 
-  try {
-    yeomanConfig.app = require('./bower.json').appPath || yeomanConfig.app;
-  } catch (e) {}
-
+  // Define the configuration for all the tasks
   grunt.initConfig({
-    yeoman: yeomanConfig,
+
+    // Project settings
+    yeoman: appConfig,
+
+    // Watches files for changes and runs tasks based on the changed files
     watch: {
-      coffee: {
-        files: ['<%= yeoman.app %>/scripts/{,*/}*.coffee'],
-        tasks: ['coffee:dist']
+      bower: {
+        files: ['bower.json'],
+        tasks: ['wiredep']
       },
-      coffeeTest: {
-        files: ['test/spec/{,*/}*.coffee'],
-        tasks: ['coffee:test']
+      js: {
+        files: ['<%= yeoman.app %>/scripts/{,*/}*.js'],
+        tasks: ['newer:jshint:all'],
+        options: {
+          livereload: '<%= connect.options.livereload %>'
+        }
+      },
+      jsTest: {
+        files: ['test/spec/{,*/}*.js'],
+        tasks: ['newer:jshint:test', 'karma']
       },
       styles: {
         files: ['<%= yeoman.app %>/styles/{,*/}*.css'],
-        tasks: ['copy:styles', 'autoprefixer']
+        tasks: ['newer:copy:styles', 'autoprefixer']
+      },
+      gruntfile: {
+        files: ['Gruntfile.js']
       },
       livereload: {
         options: {
-          livereload: LIVERELOAD_PORT
+          livereload: '<%= connect.options.livereload %>'
         },
         files: [
           '<%= yeoman.app %>/{,*/}*.html',
           '.tmp/styles/{,*/}*.css',
-          '{.tmp,<%= yeoman.app %>}/scripts/{,*/}*.js',
           '<%= yeoman.app %>/images/{,*/}*.{png,jpg,jpeg,gif,webp,svg}'
         ]
       }
     },
+
+    // The actual grunt server settings
+    connect: {
+      options: {
+        port: 9000,
+        // Change this to '0.0.0.0' to access the server from outside.
+        hostname: 'localhost',
+        livereload: 35729
+      },
+      livereload: {
+        options: {
+          open: true,
+          middleware: function (connect) {
+            return [
+              connect.static('.tmp'),
+              connect().use(
+                '/bower_components',
+                connect.static('./bower_components')
+              ),
+              connect.static(appConfig.app)
+            ];
+          }
+        }
+      },
+      test: {
+        options: {
+          port: 9001,
+          middleware: function (connect) {
+            return [
+              connect.static('.tmp'),
+              connect.static('test'),
+              connect().use(
+                '/bower_components',
+                connect.static('./bower_components')
+              ),
+              connect.static(appConfig.app)
+            ];
+          }
+        }
+      },
+      dist: {
+        options: {
+          open: true,
+          base: '<%= yeoman.dist %>'
+        }
+      }
+    },
+
+    // Make sure code styles are up to par and there are no obvious mistakes
+    jshint: {
+      options: {
+        jshintrc: '.jshintrc',
+        reporter: require('jshint-stylish')
+      },
+      all: {
+        src: [
+          'Gruntfile.js',
+          '<%= yeoman.app %>/scripts/{,*/}*.js'
+        ]
+      },
+      test: {
+        options: {
+          jshintrc: 'test/.jshintrc'
+        },
+        src: ['test/spec/{,*/}*.js']
+      }
+    },
+
+    // Empties folders to start fresh
+    clean: {
+      dist: {
+        files: [{
+          dot: true,
+          src: [
+            '.tmp',
+            '<%= yeoman.dist %>/{,*/}*',
+            '!<%= yeoman.dist %>/.git*'
+          ]
+        }]
+      },
+      server: '.tmp'
+    },
+
+    // Add vendor prefixed styles
     autoprefixer: {
-      options: ['last 1 version'],
+      options: {
+        browsers: ['last 1 version']
+      },
       dist: {
         files: [{
           expand: true,
@@ -64,134 +159,95 @@ module.exports = function (grunt) {
         }]
       }
     },
-    connect: {
+
+    // Automatically inject Bower components into the app
+    wiredep: {
       options: {
-        port: 9000,
-        // Change this to '0.0.0.0' to access the server from outside.
-        hostname: 'localhost'
+        cwd: '<%= yeoman.app %>'
       },
-      livereload: {
-        options: {
-          middleware: function (connect) {
-            return [
-              lrSnippet,
-              mountFolder(connect, '.tmp'),
-              mountFolder(connect, yeomanConfig.app)
-            ];
-          }
-        }
-      },
-      test: {
-        options: {
-          middleware: function (connect) {
-            return [
-              mountFolder(connect, '.tmp'),
-              mountFolder(connect, 'test')
-            ];
-          }
-        }
-      },
-      dist: {
-        options: {
-          middleware: function (connect) {
-            return [
-              mountFolder(connect, yeomanConfig.dist)
-            ];
-          }
-        }
+      app: {
+        src: ['<%= yeoman.app %>/index.html'],
+        ignorePath:  /\.\.\//
       }
     },
-    open: {
-      server: {
-        url: 'http://localhost:<%= connect.options.port %>'
-      }
-    },
-    clean: {
-      dist: {
-        files: [{
-          dot: true,
-          src: [
-            '.tmp',
-            '<%= yeoman.dist %>/*',
-            '!<%= yeoman.dist %>/.git*'
-          ]
-        }]
-      },
-      server: '.tmp'
-    },
-    jshint: {
-      options: {
-        jshintrc: '.jshintrc'
-      },
-      all: [
-        'Gruntfile.js',
-        '<%= yeoman.app %>/scripts/{,*/}*.js'
-      ]
-    },
-    coffee: {
-      options: {
-        sourceMap: true,
-        sourceRoot: ''
-      },
-      dist: {
-        files: [{
-          expand: true,
-          cwd: '<%= yeoman.app %>/scripts',
-          src: '{,*/}*.coffee',
-          dest: '.tmp/scripts',
-          ext: '.js'
-        }]
-      },
-      test: {
-        files: [{
-          expand: true,
-          cwd: 'test/spec',
-          src: '{,*/}*.coffee',
-          dest: '.tmp/spec',
-          ext: '.js'
-        }]
-      }
-    },
-    // not used since Uglify task does concat,
-    // but still available if needed
-    /*concat: {
-      dist: {}
-    },*/
-    rev: {
-      dist: {
-        files: {
-          src: [
-            '<%= yeoman.dist %>/scripts/{,*/}*.js',
-            '<%= yeoman.dist %>/styles/{,*/}*.css',
-            '<%= yeoman.dist %>/images/{,*/}*.{png,jpg,jpeg,gif,webp,svg}',
-            '<%= yeoman.dist %>/styles/fonts/*'
-          ]
-        }
+
+    // Renames files for browser caching purposes
+    filerev: {
+      dist: {
+        src: [
+          '<%= yeoman.dist %>/scripts/{,*/}*.js',
+          '<%= yeoman.dist %>/styles/{,*/}*.css',
+          '<%= yeoman.dist %>/images/{,*/}*.{png,jpg,jpeg,gif,webp,svg}',
+          '<%= yeoman.dist %>/styles/fonts/*'
+        ]
       }
     },
+
+    // Reads HTML for usemin blocks to enable smart builds that automatically
+    // concat, minify and revision files. Creates configurations in memory so
+    // additional tasks can operate on them
     useminPrepare: {
-      html: '<%= yeoman.app %>/index.html',
+      html: ['<%= yeoman.app %>/index.html', '<%= yeoman.app %>/login.html'],
       options: {
-        dest: '<%= yeoman.dist %>'
+        dest: '<%= yeoman.dist %>',
+        flow: {
+          html: {
+            steps: {
+              js: ['concat', 'uglifyjs'],
+              css: ['cssmin']
+            },
+            post: {}
+          }
+        }
       }
     },
+
+    // Performs rewrites based on filerev and the useminPrepare configuration
     usemin: {
       html: ['<%= yeoman.dist %>/{,*/}*.html'],
       css: ['<%= yeoman.dist %>/styles/{,*/}*.css'],
       options: {
-        dirs: ['<%= yeoman.dist %>']
+        assetsDirs: ['<%= yeoman.dist %>','<%= yeoman.dist %>/images']
       }
     },
+
+    // The following *-min tasks will produce minified files in the dist folder
+    // By default, your `index.html`'s <!-- Usemin block --> will take care of
+    // minification. These next options are pre-configured if you do not wish
+    // to use the Usemin blocks.
+    // cssmin: {
+    //   dist: {
+    //     files: {
+    //       '<%= yeoman.dist %>/styles/main.css': [
+    //         '.tmp/styles/{,*/}*.css'
+    //       ]
+    //     }
+    //   }
+    // },
+    // uglify: {
+    //   dist: {
+    //     files: {
+    //       '<%= yeoman.dist %>/scripts/scripts.js': [
+    //         '<%= yeoman.dist %>/scripts/scripts.js'
+    //       ]
+    //     }
+    //   }
+    // },
+    // concat: {
+    //   dist: {}
+    // },
+
     imagemin: {
       dist: {
         files: [{
           expand: true,
           cwd: '<%= yeoman.app %>/images',
-          src: '{,*/}*.{png,jpg,jpeg}',
+          src: '{,*/}*.{png,jpg,jpeg,gif}',
           dest: '<%= yeoman.dist %>/images'
         }]
       }
     },
+
     svgmin: {
       dist: {
         files: [{
@@ -202,41 +258,47 @@ module.exports = function (grunt) {
         }]
       }
     },
-    cssmin: {
-      // By default, your `index.html` <!-- Usemin Block --> will take care of
-      // minification. This option is pre-configured if you do not wish to use
-      // Usemin blocks.
-      // dist: {
-      //   files: {
-      //     '<%= yeoman.dist %>/styles/main.css': [
-      //       '.tmp/styles/{,*/}*.css',
-      //       '<%= yeoman.app %>/styles/{,*/}*.css'
-      //     ]
-      //   }
-      // }
-    },
+
     htmlmin: {
       dist: {
         options: {
-          /*removeCommentsFromCDATA: true,
-          // https://github.com/yeoman/grunt-usemin/issues/44
-          //collapseWhitespace: true,
+          collapseWhitespace: true,
+          conservativeCollapse: true,
           collapseBooleanAttributes: true,
-          removeAttributeQuotes: true,
-          removeRedundantAttributes: true,
-          useShortDoctype: true,
-          removeEmptyAttributes: true,
-          removeOptionalTags: true*/
+          removeCommentsFromCDATA: true,
+          removeOptionalTags: true
         },
         files: [{
           expand: true,
-          cwd: '<%= yeoman.app %>',
-          src: ['*.html', 'views/*.html'],
+          cwd: '<%= yeoman.dist %>',
+          src: ['*.html', 'views/{,*/}*.html'],
           dest: '<%= yeoman.dist %>'
         }]
       }
     },
-    // Put files not handled in other tasks here
+
+    // ngmin tries to make the code safe for minification automatically by
+    // using the Angular long form for dependency injection. It doesn't work on
+    // things like resolve or inject so those have to be done manually.
+    ngmin: {
+      dist: {
+        files: [{
+          expand: true,
+          cwd: '.tmp/concat/scripts',
+          src: '*.js',
+          dest: '.tmp/concat/scripts'
+        }]
+      }
+    },
+
+    // Replace Google CDN references
+    cdnify: {
+      dist: {
+        html: ['<%= yeoman.dist %>/*.html']
+      }
+    },
+
+    // Copies remaining files to places other tasks can use
     copy: {
       dist: {
         files: [{
@@ -247,17 +309,21 @@ module.exports = function (grunt) {
           src: [
             '*.{ico,png,txt}',
             '.htaccess',
-            'bower_components/**/*',
-            'images/{,*/}*.{gif,webp}',
+            '*.html',
+            'views/{,*/}*.html',
+            'images/{,*/}*.{webp}',
             'fonts/*'
           ]
         }, {
           expand: true,
           cwd: '.tmp/images',
           dest: '<%= yeoman.dist %>/images',
-          src: [
-            'generated/*'
-          ]
+          src: ['generated/*']
+        }, {
+          expand: true,
+          cwd: 'bower_components/bootstrap/dist',
+          src: 'fonts/*',
+          dest: '<%= yeoman.dist %>'
         }]
       },
       styles: {
@@ -267,70 +333,52 @@ module.exports = function (grunt) {
         src: '{,*/}*.css'
       }
     },
+
+    // Run some tasks in parallel to speed up the build process
     concurrent: {
       server: [
-        'coffee:dist',
         'copy:styles'
       ],
       test: [
-        'coffee',
         'copy:styles'
       ],
       dist: [
-        'coffee',
         'copy:styles',
         'imagemin',
-        'svgmin',
-        'htmlmin'
+        'svgmin'
       ]
     },
+
+    // Test settings
     karma: {
       unit: {
-        configFile: 'karma.conf.js',
+        configFile: 'test/karma.conf.js',
         singleRun: true
       }
-    },
-    cdnify: {
-      dist: {
-        html: ['<%= yeoman.dist %>/*.html']
-      }
-    },
-    ngmin: {
-      dist: {
-        files: [{
-          expand: true,
-          cwd: '<%= yeoman.dist %>/scripts',
-          src: '*.js',
-          dest: '<%= yeoman.dist %>/scripts'
-        }]
-      }
-    },
-    uglify: {
-      dist: {
-        files: {
-          '<%= yeoman.dist %>/scripts/scripts.js': [
-            '<%= yeoman.dist %>/scripts/scripts.js'
-          ]
-        }
-      }
     }
   });
 
-  grunt.registerTask('server', function (target) {
+
+  grunt.registerTask('serve', 'Compile then start a connect web server', function (target) {
     if (target === 'dist') {
-      return grunt.task.run(['build', 'open', 'connect:dist:keepalive']);
+      return grunt.task.run(['build', 'connect:dist:keepalive']);
     }
 
     grunt.task.run([
       'clean:server',
+      'wiredep',
       'concurrent:server',
       'autoprefixer',
       'connect:livereload',
-      'open',
       'watch'
     ]);
   });
 
+  grunt.registerTask('server', 'DEPRECATED TASK. Use the "serve" task instead', function (target) {
+    grunt.log.warn('The `server` task has been deprecated. Use `grunt serve` to start a server.');
+    grunt.task.run(['serve:' + target]);
+  });
+
   grunt.registerTask('test', [
     'clean:server',
     'concurrent:test',
@@ -341,21 +389,23 @@ module.exports = function (grunt) {
 
   grunt.registerTask('build', [
     'clean:dist',
+    'wiredep',
     'useminPrepare',
     'concurrent:dist',
     'autoprefixer',
     'concat',
+    'ngmin',
     'copy:dist',
     'cdnify',
-    'ngmin',
     'cssmin',
     'uglify',
-    'rev',
-    'usemin'
+    'filerev',
+    'usemin',
+    'htmlmin'
   ]);
 
   grunt.registerTask('default', [
-    'jshint',
+    'newer:jshint',
     'test',
     'build'
   ]);

rd_ui/app/index.html

@@ -4,7 +4,7 @@
 <!--[if IE 8]>         <html class="no-js lt-ie9" ng-app="redash" ng-controller='MainCtrl'> <![endif]-->
 <!--[if gt IE 8]><!--> <html class="no-js" ng-app="redash" ng-controller='MainCtrl'> <!--<![endif]-->
 <head>
-    <title ng-bind="'re:dash | ' + pageTitle"></title>
+    <title ng-bind="'{{name}} | ' + pageTitle"></title>
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
 
@@ -12,8 +12,10 @@
     <link rel="stylesheet" href="/bower_components/bootstrap/dist/css/bootstrap.css">
     <link rel="stylesheet" href="/bower_components/codemirror/lib/codemirror.css">
     <link rel="stylesheet" href="/bower_components/gridster/dist/jquery.gridster.css">
-    <link rel="stylesheet" href="/bower_components/pivottable/examples/pivot.css">
+    <link rel="stylesheet" href="/bower_components/pivottable/dist/pivot.css">
     <link rel="stylesheet" href="/bower_components/cornelius/src/cornelius.css">
+    <link rel="stylesheet" href="/bower_components/select2/select2.css">
+    <link rel="stylesheet" href="/bower_components/pace/themes/pace-theme-minimal.css">
     <link rel="stylesheet" href="/styles/redash.css">
     <!-- endbuild -->
 </head>
@@ -29,13 +31,13 @@
             <span class="icon-bar"></span>
             <span class="icon-bar"></span>
         </button>
-        <a class="navbar-brand" href="/"><strong>re:dash</strong></a>
+        <a class="navbar-brand" href="/"><strong>{{name}}</strong></a>
     </div>
     {% raw %}
     <div class="collapse navbar-collapse navbar-ex1-collapse">
         <ul class="nav navbar-nav">
             <li class="active" ng-show="pageTitle"><a class="page-title" ng-bind="pageTitle"></a></li>
-            <li class="dropdown">
+            <li class="dropdown" ng-show="groupedDashboards.length > 0 || otherDashboards.length > 0 || currentUser.hasPermission('create_dashboard')">
                 <a href="#" class="dropdown-toggle" data-toggle="dropdown"><span class="glyphicon glyphicon-th-large"></span> <b class="caret"></b></a>
                 <ul class="dropdown-menu">
                     <span ng-repeat="(name, group) in groupedDashboards">
@@ -51,21 +53,24 @@
                     <li ng-repeat="dashboard in otherDashboards">
                         <a role="menu-item" ng-href="/dashboard/{{dashboard.slug}}" ng-bind="dashboard.name"></a>
                     </li>
-                    <li class="divider"></li>
-                    <li><a data-toggle="modal" href="#new_dashboard_dialog">New Dashboard</a></li>
+                    <li class="divider" ng-show="currentUser.hasPermission('create_dashboard') && (groupedDashboards.length > 0 || otherDashboards.length > 0)"></li>
+                    <li><a data-toggle="modal" href="#new_dashboard_dialog" ng-show="currentUser.hasPermission('create_dashboard')">New Dashboard</a></li>
                 </ul>
             </li>
-            <li class="dropdown">
+            <li class="dropdown" ng-show="currentUser.hasPermission('view_query')">
                 <a href="#" class="dropdown-toggle" data-toggle="dropdown">Queries <b class="caret"></b></a>
                 <ul class="dropdown-menu">
-                    <li><a href="/queries/new">New Query</a></li>
+                    <li ng-show="currentUser.hasPermission('create_query')"><a href="/queries/new">New Query</a></li>
                     <li><a href="/queries">Queries</a></li>
                 </ul>
             </li>
         </ul>
         <ul class="nav navbar-nav navbar-right">
-            <p class="navbar-text avatar">
-                <img ng-src="{{currentUser.gravatar_url}}" class="img-circle" alt="{{currentUser.name}}" width="40" height="40"/>
+            <p class="navbar-text avatar" ng-show="currentUser.id" ng-cloak>
+                <img ng-src="{{currentUser.gravatar_url}}" class="img-circle" alt="{{currentUser.name}}"/>
+                <a target="_self" href="/logout" id="logout" title="Logout">
+                    <span class="glyphicon glyphicon-log-out"></span>
+                </a>
             </p>
         </ul>
     </div>
@@ -97,30 +102,48 @@
 <script src="/bower_components/angular-ui-codemirror/ui-codemirror.js"></script>
 <script src="/bower_components/highcharts/highcharts.js"></script>
 <script src="/bower_components/highcharts/modules/exporting.js"></script>
-<script src="/scripts/ng-highchart.js"></script>
-<script src="/scripts/smart-table.js"></script>
-<script src="/scripts/ui-bootstrap-tpls-0.5.0.min.js"></script>
 <script src="/bower_components/gridster/dist/jquery.gridster.js"></script>
 <script src="/bower_components/angular-growl/build/angular-growl.js"></script>
-<script src="/bower_components/pivottable/examples/pivot.js"></script>
+<script src="/bower_components/pivottable/dist/pivot.js"></script>
 <script src="/bower_components/cornelius/src/cornelius.js"></script>
 <script src="/bower_components/mousetrap/mousetrap.js"></script>
 <script src="/bower_components/mousetrap/plugins/global-bind/mousetrap-global-bind.js"></script>
+<script src="/bower_components/select2/select2.js"></script>
+<script src="/bower_components/angular-ui-select2/src/select2.js"></script>
+<script src="/bower_components/underscore.string/lib/underscore.string.js"></script>
+<script src="/bower_components/marked/lib/marked.js"></script>
+<script src="/scripts/ng_highchart.js"></script>
+<script src="/scripts/ng_smart_table.js"></script>
+<script src="/scripts/ui-bootstrap-tpls-0.5.0.min.js"></script>
+<script src="/bower_components/bucky/bucky.js"></script>
+<script src="/bower_components/pace/pace.js"></script>
 <!-- endbuild -->
 
 <!-- build:js({.tmp,app}) /scripts/scripts.js -->
 <script src="/scripts/app.js"></script>
-<script src="/scripts/controllers.js"></script>
-<script src="/scripts/admin_controllers.js"></script>
-<script src="/scripts/directives.js"></script>
-<script src="/scripts/services.js"></script>
-<script src="/scripts/filters.js"></script>
+<script src="/scripts/services/services.js"></script>
+<script src="/scripts/services/resources.js"></script>
 <script src="/scripts/services/notifications.js"></script>
 <script src="/scripts/services/dashboards.js"></script>
-<script src="/scripts/query_fiddle/renderers.js"></script>
+<script src="/scripts/controllers/controllers.js"></script>
+<script src="/scripts/controllers/dashboard.js"></script>
+<script src="/scripts/controllers/admin_controllers.js"></script>
+<script src="/scripts/controllers/query_view.js"></script>
+<script src="/scripts/controllers/query_source.js"></script>
+<script src="/scripts/visualizations/base.js"></script>
+<script src="/scripts/visualizations/chart.js"></script>
+<script src="/scripts/visualizations/cohort.js"></script>
+<script src="/scripts/visualizations/table.js"></script>
+<script src="/scripts/visualizations/pivot.js"></script>
+<script src="/scripts/directives/directives.js"></script>
+<script src="/scripts/directives/query_directives.js"></script>
+<script src="/scripts/directives/dashboard_directives.js"></script>
+<script src="/scripts/filters.js"></script>
 <!-- endbuild -->
 
 <script>
+    // TODO: move currentUser & features to be an Angular service
+    var featureFlags = {{ features|safe }};
     var currentUser = {{ user|safe }};
 
     currentUser.canEdit = function(object) {
@@ -128,8 +151,12 @@
         return user_id && (user_id == currentUser.id);
     };
 
+    currentUser.hasPermission = function(permission) {
+        return this.permissions.indexOf(permission) != -1;
+    }
+
     {{ analytics|safe }}
 </script>
 
 </body>
-</html>
+</html>

rd_ui/app/login.html

@@ -0,0 +1,85 @@
+<!DOCTYPE html>
+<!--[if lt IE 7]>      <html class="no-js lt-ie9 lt-ie8 lt-ie7"> <![endif]-->
+<!--[if IE 7]>         <html class="no-js lt-ie9 lt-ie8"> <![endif]-->
+<!--[if IE 8]>         <html class="no-js lt-ie9"> <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js"> <!--<![endif]-->
+<head>
+    <title>{{name}} Login</title>
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+
+    <!-- build:css /styles/main_login.css -->
+    <link rel="stylesheet" href="/bower_components/bootstrap/dist/css/bootstrap.css">
+    <link rel="stylesheet" href="/styles/redash.css">
+    <link rel="stylesheet" href="/styles/login.css">
+    <!-- endbuild -->
+</head>
+<body>
+
+<nav class="navbar navbar-inverse navbar-fixed-top" role="navigation">
+    <div class="container">
+    <div class="navbar-header">
+        <button type="button" class="navbar-toggle" data-toggle="collapse"
+                data-target=".navbar-ex1-collapse">
+            <span class="sr-only">Toggle navigation</span>
+            <span class="icon-bar"></span>
+            <span class="icon-bar"></span>
+            <span class="icon-bar"></span>
+        </button>
+        <a class="navbar-brand" href="/"><strong>{{name}}</strong></a>
+    </div>
+    </div>
+</nav>
+
+<div class="container">
+    <div class="row">
+
+        <div class="main">
+            <form role="form" method="post" name="login">
+                <div class="form-group">
+                    <label for="inputUsernameEmail">Username or email</label>
+                    <input type="text" class="form-control" id="inputUsernameEmail" name="username" value="{{username}}">
+                </div>
+                <div class="form-group">
+                    <!--<a class="pull-right" href="#">Forgot password?</a>-->
+                    <label for="inputPassword">Password</label>
+                    <input type="password" class="form-control" id="inputPassword" name="password">
+                </div>
+                <div class="checkbox pull-right">
+                    <label>
+                        <input type="checkbox" name="remember">
+                        Remember me </label>
+                </div>
+
+                <button type="submit" class="btn btn btn-primary">
+                    Log In
+                </button>
+            </form>
+
+            {% if show_google_openid %}
+
+            <div class="login-or">
+                <hr class="hr-or">
+                <span class="span-or">or</span>
+            </div>
+
+            <div class="row">
+                <div class="col-xs-6 col-sm-6 col-md-6">
+                    <a href="/google_auth/login?next={{next}}" class="btn btn-lg btn-info btn-block">Google</a>
+                </div>
+            </div>
+
+            {% endif %}
+        </div>
+
+    </div>
+</div>
+
+<script src="/bower_components/jquery/jquery.js"></script>
+
+<script>
+    {{ analytics|safe }}
+</script>
+
+</body>
+</html>

rd_ui/app/scripts/admin_controllers.js

@@ -1,23 +0,0 @@
-(function () {
-    var AdminStatusCtrl = function ($scope, $http, $timeout) {
-        $scope.$parent.pageTitle = "System Status";
-
-        var refresh = function () {
-            $scope.refresh_time = moment().add('minutes', 1);
-            $http.get('/status.json').success(function (data) {
-                $scope.workers = data.workers;
-                delete data.workers;
-                $scope.manager = data.manager;
-                delete data.manager;
-                $scope.status = data;
-            });
-
-            $timeout(refresh, 59 * 1000);
-        };
-
-        refresh();
-    }
-
-    angular.module('redash.admin_controllers', [])
-        .controller('AdminStatusCtrl', ['$scope', '$http', '$timeout', AdminStatusCtrl])
-})();

rd_ui/app/scripts/app.js

@@ -5,55 +5,89 @@ angular.module('redash', [
     'redash.filters',
     'redash.services',
     'redash.renderers',
+    'redash.visualization',
     'ui.codemirror',
     'highchart',
+    'ui.select2',
     'angular-growl',
     'angularMoment',
     'ui.bootstrap',
     'smartTable.table',
     'ngResource',
     'ngRoute'
-]).config(['$routeProvider', '$locationProvider', '$compileProvider', 'growlProvider',
-    function($routeProvider, $locationProvider, $compileProvider, growlProvider) {
-
-        $compileProvider.aHrefSanitizationWhitelist(/^\s*(https?|http|data):/);
-        $locationProvider.html5Mode(true);
-        growlProvider.globalTimeToLive(2000);
-
-        $routeProvider.when('/dashboard/:dashboardSlug', {
-            templateUrl: '/views/dashboard.html',
-            controller: 'DashboardCtrl'
-        });
-        $routeProvider.when('/queries', {
-            templateUrl: '/views/queries.html',
-            controller: 'QueriesCtrl',
-            reloadOnSearch: false
-        });
-        $routeProvider.when('/queries/new', {
-            templateUrl: '/views/queryfiddle.html',
-            controller: 'QueryFiddleCtrl',
-            reloadOnSearch: false
-        });
-        $routeProvider.when('/queries/:queryId', {
-            templateUrl: '/views/queryfiddle.html',
-            controller: 'QueryFiddleCtrl',
-            reloadOnSearch: false
-        });
-        $routeProvider.when('/admin/status', {
-            templateUrl: '/views/admin_status.html',
-            controller: 'AdminStatusCtrl'
-        });
-        $routeProvider.when('/', {
-            templateUrl: '/views/index.html',
-            controller: 'IndexCtrl'
-        });
-        $routeProvider.otherwise({
-            redirectTo: '/'
+  ]).config(['$routeProvider', '$locationProvider', '$compileProvider', 'growlProvider',
+    function ($routeProvider, $locationProvider, $compileProvider, growlProvider) {
+      if (featureFlags.clientSideMetrics) {
+        Bucky.setOptions({
+          host: '/api/metrics'
         });
 
-        Highcharts.setOptions({
-            colors: ["#4572A7", "#AA4643", "#89A54E", "#80699B", "#3D96AE",
-                        "#DB843D", "#92A8CD", "#A47D7C", "#B5CA92"]
-        });
+        Bucky.requests.monitor('ajax_requsts');
+        Bucky.requests.transforms.enable('dashboards', /dashboard\/[\w-]+/ig, '/dashboard');
+      }
+
+      function getQuery(Query, $route) {
+        var query = Query.get({'id': $route.current.params.queryId });
+        return query.$promise;
+      };
+
+      $compileProvider.aHrefSanitizationWhitelist(/^\s*(https?|http|data):/);
+      $locationProvider.html5Mode(true);
+      growlProvider.globalTimeToLive(2000);
+
+      $routeProvider.when('/dashboard/:dashboardSlug', {
+        templateUrl: '/views/dashboard.html',
+        controller: 'DashboardCtrl',
+        reloadOnSearch: false
+      });
+      $routeProvider.when('/queries', {
+        templateUrl: '/views/queries.html',
+        controller: 'QueriesCtrl',
+        reloadOnSearch: false
+      });
+      $routeProvider.when('/queries/new', {
+        templateUrl: '/views/query.html',
+        controller: 'QuerySourceCtrl',
+        reloadOnSearch: false,
+        resolve: {
+          'query': ['Query', function newQuery(Query) {
+            return Query.newQuery();
+          }]
+        }
+      });
+      $routeProvider.when('/queries/:queryId', {
+        templateUrl: '/views/query.html',
+        controller: 'QueryViewCtrl',
+        reloadOnSearch: false,
+        resolve: {
+          'query': ['Query', '$route', getQuery]
+        }
+      });
+      $routeProvider.when('/queries/:queryId/source', {
+        templateUrl: '/views/query.html',
+        controller: 'QuerySourceCtrl',
+        reloadOnSearch: false,
+        resolve: {
+          'query': ['Query', '$route', getQuery]
+        }
+      });
+      $routeProvider.when('/admin/status', {
+        templateUrl: '/views/admin_status.html',
+        controller: 'AdminStatusCtrl'
+      });
+      $routeProvider.when('/admin/workers', {
+        templateUrl: '/views/admin_workers.html',
+        controller: 'AdminWorkersCtrl'
+      });
+
+      $routeProvider.when('/', {
+        templateUrl: '/views/index.html',
+        controller: 'IndexCtrl'
+      });
+      $routeProvider.otherwise({
+        redirectTo: '/'
+      });
+
+
     }
-]);
+  ]);

rd_ui/app/scripts/controllers.js

@@ -1,438 +0,0 @@
-(function () {
-    var DashboardCtrl = function ($scope, $routeParams, $http, $timeout, Dashboard) {
-        $scope.refreshEnabled = false;
-        $scope.refreshRate = 60;
-        $scope.dashboard = Dashboard.get({slug: $routeParams.dashboardSlug}, function(dashboard) {
-            $scope.$parent.pageTitle = dashboard.name;
-        });
-
-        var autoRefresh = function() {
-            if ($scope.refreshEnabled) {
-                $timeout(function() {
-                    Dashboard.get({slug: $routeParams.dashboardSlug}, function(dashboard) {
-                        var newWidgets = _.groupBy(_.flatten(dashboard.widgets), 'id');
-
-                        _.each($scope.dashboard.widgets, function(row) {
-                            _.each(row, function(widget, i) {
-                                var newWidget = newWidgets[widget.id];
-                                if (newWidget && newWidget[0].visualization.query.latest_query_data_id != widget.visualization.query.latest_query_data_id ) {
-                                    row[i] = newWidget[0];
-                                }
-                            });
-                        });
-
-                        autoRefresh();
-                    });
-
-                }, $scope.refreshRate);
-            };
-        }
-
-        $scope.triggerRefresh = function(){
-            $scope.refreshEnabled = !$scope.refreshEnabled;
-
-            if ($scope.refreshEnabled) {
-                var refreshRate = _.min(_.flatten($scope.dashboard.widgets), function(widget) {
-                    return widget.visualization.query.ttl;
-                }).visualization.query.ttl;
-
-                $scope.refreshRate = _.max([120, refreshRate * 2])*1000;
-
-                autoRefresh();
-            }
-        };
-    };
-
-    var WidgetCtrl = function ($scope, $http, $location, Query) {
-        $scope.deleteWidget = function() {
-            if (!confirm('Are you sure you want to remove "' + $scope.widget.visualization.name + '" from the dashboard?')) {
-                return;
-            }
-
-            $http.delete('/api/widgets/' + $scope.widget.id).success(function() {
-                $scope.dashboard.widgets = _.map($scope.dashboard.widgets, function(row) {
-                    return _.filter(row, function(widget) {
-                        return widget.id != $scope.widget.id;
-                    })
-                });
-            });
-        };
-
-        $scope.open = function(query, visualization) {
-            $location.path('/queries/' + query.id);
-            $location.hash(visualization.id);
-        }
-
-        $scope.query = new Query($scope.widget.visualization.query);
-        $scope.queryResult = $scope.query.getQueryResult();
-
-        $scope.updateTime = (new Date($scope.queryResult.getUpdatedAt())).toISOString();
-        $scope.nextUpdateTime = moment(new Date(($scope.query.updated_at + $scope.query.ttl + $scope.query.runtime + 300) * 1000)).fromNow();
-
-        $scope.updateTime = '';
-    }
-
-    var QueryFiddleCtrl = function ($scope, $window, $location, $routeParams, $http, $location, growl, notifications, Query, Visualization) {
-        var DEFAULT_TAB = 'table';
-        var pristineHash = null;
-        var leavingPageText = "You will lose your changes if you leave";
-
-        $scope.dirty = undefined;
-        $scope.newVisualization = undefined;
-
-        $window.onbeforeunload = function(){
-            if (currentUser.canEdit($scope.query) && $scope.dirty) {
-                return leavingPageText;
-            }
-        }
-
-        Mousetrap.bindGlobal("meta+s", function(e) {
-            e.preventDefault();
-
-            if (currentUser.canEdit($scope.query)) {
-                $scope.saveQuery();
-            }
-        });
-
-        $scope.$on('$locationChangeStart', function(event, next, current) {
-            if (next.split("#")[0] == current.split("#")[0]) {
-                return;
-            }
-
-            if (!currentUser.canEdit($scope.query)) {
-                return;
-            }
-
-            if($scope.dirty &&
-                !confirm(leavingPageText + "\n\nAre you sure you want to leave this page?")) {
-                event.preventDefault();
-            } else {
-                Mousetrap.unbind("meta+s");
-            }
-        });
-
-        $scope.$parent.pageTitle = "Query Fiddle";
-
-        $scope.$watch(function() {return $location.hash()}, function(hash) {
-            $scope.selectedTab = hash || DEFAULT_TAB;
-        });
-
-        $scope.lockButton = function (lock) {
-            $scope.queryExecuting = lock;
-        };
-
-        $scope.formatQuery = function() {
-            $scope.editorOptions.readOnly = 'nocursor';
-
-            $http.post('/api/queries/format', {'query': $scope.query.query}).success(function(response) {
-                $scope.query.query = response;
-                $scope.editorOptions.readOnly = false;
-            })
-        }
-
-        $scope.saveQuery = function (duplicate, oldId) {
-            if (!oldId) {
-                oldId = $scope.query.id;
-            }
-            delete $scope.query.latest_query_data;
-            $scope.query.$save(function (q) {
-                pristineHash = q.getHash();
-                $scope.dirty = false;
-
-                if (duplicate) {
-                    growl.addInfoMessage("Query duplicated.", {ttl: 2000});
-                } else{
-                    growl.addSuccessMessage("Query saved.", {ttl: 2000});
-                }
-
-                if (oldId != q.id) {
-                    if (oldId == undefined) {
-                        $location.path($location.path().replace('new', q.id)).replace();
-                    } else {
-                        // TODO: replace this with a safer method
-                        $location.path($location.path().replace(oldId, q.id)).replace();
-
-                        // Reset visualizations tab to table after duplicating a query:
-                        $location.hash('table');
-                    }
-                }
-            }, function(httpResponse) {
-                growl.addErrorMessage("Query could not be saved");
-            });
-        };
-
-        $scope.duplicateQuery = function () {
-            var oldId = $scope.query.id;
-            $scope.query.id = null;
-            $scope.query.ttl = -1;
-
-            $scope.saveQuery(true, oldId);
-        };
-
-        // Query Editor:
-        $scope.editorOptions = {
-            mode: 'text/x-sql',
-            lineWrapping: true,
-            lineNumbers: true,
-            readOnly: false,
-            matchBrackets: true,
-            autoCloseBrackets: true
-        };
-
-        $scope.refreshOptions = [
-            {value: -1, name: 'No Refresh'},
-            {value: 60, name: 'Every minute'},
-        ]
-
-        _.each(_.range(1, 13), function(i) {
-            $scope.refreshOptions.push({value: i*3600, name: 'Every ' + i + 'h'});
-        })
-
-        $scope.refreshOptions.push({value: 24*3600, name: 'Every 24h'});
-        $scope.refreshOptions.push({value: 7*24*3600, name: 'Once a week'});
-
-        $scope.$watch('queryResult && queryResult.getError()', function (newError, oldError) {
-            if (newError == undefined) {
-                return;
-            }
-
-            if (oldError == undefined && newError != undefined) {
-                $scope.lockButton(false);
-            }
-        });
-
-        $scope.$watch('queryResult && queryResult.getData()', function (data, oldData) {
-            if (!data) {
-                return;
-            }
-
-            if ($scope.queryResult.getId() == null) {
-                $scope.dataUri = "";
-            } else {
-                $scope.dataUri = '/api/queries/' + $scope.query.id + '/results/' + $scope.queryResult.getId() + '.csv';
-                $scope.dataFilename = $scope.query.name.replace(" ", "_") + moment($scope.queryResult.getUpdatedAt()).format("_YYYY_MM_DD") + ".csv";
-            }
-        });
-
-        $scope.$watch("queryResult && queryResult.getStatus()", function (status) {
-            if (!status) {
-                return;
-            }
-
-            if (status == "done") {
-                if ($scope.query.id && $scope.query.latest_query_data_id != $scope.queryResult.getId() &&
-                    $scope.query.query_hash == $scope.queryResult.query_result.query_hash) {
-                    Query.save({'id': $scope.query.id, 'latest_query_data_id': $scope.queryResult.getId()})
-                }
-                $scope.query.latest_query_data_id = $scope.queryResult.getId();
-
-                notifications.showNotification("re:dash", $scope.query.name + " updated.");
-
-                $scope.lockButton(false);
-            }
-        });
-
-        if ($routeParams.queryId != undefined) {
-            $scope.query = Query.get({id: $routeParams.queryId}, function(q) {
-                pristineHash = q.getHash();
-                $scope.dirty = false;
-                $scope.queryResult = $scope.query.getQueryResult();
-            });
-        } else {
-            $scope.query = new Query({query: "", name: "New Query", ttl: -1, user: currentUser});
-            $scope.lockButton(false);
-        }
-
-        $scope.$watch('query.name', function() {
-            $scope.$parent.pageTitle = $scope.query.name;
-        });
-
-        $scope.$watch(function() {
-            return $scope.query.getHash();
-        }, function(newHash) {
-            $scope.dirty = (newHash !== pristineHash);
-        });
-
-        $scope.executeQuery = function() {
-            $scope.queryResult = $scope.query.getQueryResult(0);
-            $scope.lockButton(true);
-            $scope.cancelling = false;
-        };
-
-        $scope.cancelExecution = function() {
-            $scope.cancelling = true;
-            $scope.queryResult.cancelExecution();
-        };
-
-        $scope.deleteVisualization = function($e, vis) {
-            $e.preventDefault();
-            if (confirm('Are you sure you want to delete ' + vis.name + ' ?')) {
-                Visualization.delete(vis);
-                if ($scope.selectedTab == vis.id) {
-                    $scope.selectedTab = DEFAULT_TAB;
-                }
-                $scope.query.visualizations =
-                    $scope.query.visualizations.filter(function(v) {
-                        return vis.id !== v.id;
-                    });
-            }
-        };
-    }
-
-    var QueriesCtrl = function($scope, $http, $location, $filter, Query) {
-        $scope.$parent.pageTitle = "All Queries";
-        $scope.gridConfig = {
-            isPaginationEnabled: true,
-            itemsByPage: 50,
-            maxSize: 8,
-            isGlobalSearchActivated: true
-        }
-
-        $scope.allQueries = [];
-        $scope.queries = [];
-
-        var dateFormatter = function (value) {
-            if (!value) return "-";
-            return value.format("DD/MM/YY HH:mm");
-        }
-
-        var filterQueries = function() {
-            $scope.queries = _.filter($scope.allQueries, function(query) {
-                if (!$scope.selectedTab) {
-                    return false;
-                }
-
-                if ($scope.selectedTab.key == 'my') {
-                    return query.user.id == currentUser.id && query.name != 'New Query';
-                } else if ($scope.selectedTab.key == 'drafts') {
-                    return query.user.id == currentUser.id && query.name == 'New Query';
-                }
-
-                return query.name != 'New Query';
-            });
-        }
-
-        Query.query(function(queries) {
-            $scope.allQueries = _.map(queries, function(query) {
-                query.created_at = moment(query.created_at);
-                query.last_retrieved_at = moment(query.last_retrieved_at);
-                return query;
-            });
-
-            filterQueries();
-        });
-
-        $scope.gridColumns = [
-            {
-                "label": "Name",
-                "map": "name",
-                "cellTemplateUrl": "/views/queries_query_name_cell.html"
-            },
-            {
-                'label': 'Created By',
-                'map': 'user.name'
-            },
-            {
-                'label': 'Created At',
-                'map': 'created_at',
-                'formatFunction': dateFormatter
-            },
-            {
-                'label': 'Runtime (avg)',
-                'map': 'avg_runtime',
-                'formatFunction': function(value) {
-                    return $filter('durationHumanize')(value);
-                }
-            },
-            {
-                'label': 'Runtime (min)',
-                'map': 'min_runtime',
-                'formatFunction': function(value) {
-                    return $filter('durationHumanize')(value);
-                }
-            },
-            {
-                'label': 'Runtime (max)',
-                'map': 'max_runtime',
-                'formatFunction': function(value) {
-                    return $filter('durationHumanize')(value);
-                }
-            },
-            {
-                'label': 'Last Executed At',
-                'map': 'last_retrieved_at',
-                'formatFunction': dateFormatter
-            },
-            {
-                'label': 'Times Executed',
-                'map': 'times_retrieved'
-            },
-            {
-                'label': 'Update Schedule',
-                'map': 'ttl',
-                'formatFunction': function(value) {
-                    return $filter('refreshRateHumanize')(value);
-                }
-            }
-        ]
-        $scope.tabs = [{"name": "My Queries", "key": "my"}, {"key": "all", "name": "All Queries"}, {"key": "drafts", "name": "Drafts"}];
-
-        $scope.$watch('selectedTab', function(tab) {
-            if (tab) {
-                $scope.$parent.pageTitle =  tab.name;
-            }
-
-            filterQueries();
-        });
-    }
-
-    var MainCtrl = function ($scope, Dashboard, notifications) {
-        $scope.dashboards = [];
-        $scope.reloadDashboards = function() {
-            Dashboard.query(function (dashboards) {
-                $scope.dashboards = _.sortBy(dashboards, "name");
-                $scope.allDashboards = _.groupBy($scope.dashboards, function(d) {
-                    parts = d.name.split(":");
-                    if (parts.length == 1) {
-                        return "Other";
-                    }
-                    return parts[0];
-                });
-                $scope.otherDashboards = $scope.allDashboards['Other'] || [];
-                $scope.groupedDashboards = _.omit($scope.allDashboards, 'Other');
-            });
-        }
-
-        $scope.reloadDashboards();
-
-        $scope.currentUser = currentUser;
-        $scope.newDashboard = {
-            'name': null,
-            'layout': null
-        }
-
-        $(window).click(function () {
-            notifications.getPermissions();
-        });
-    }
-
-    var IndexCtrl = function($scope, Dashboard) {
-        $scope.$parent.pageTitle = "Home";
-
-        $scope.archiveDashboard = function(dashboard) {
-            if (confirm('Are you sure you want to delete "' + dashboard.name + '" dashboard?')) {
-                dashboard.$delete(function() {
-                    $scope.$parent.reloadDashboards();
-                });
-            }
-        }
-    }
-
-    angular.module('redash.controllers', [])
-        .controller('DashboardCtrl', ['$scope', '$routeParams', '$http', '$timeout', 'Dashboard', DashboardCtrl])
-        .controller('WidgetCtrl', ['$scope', '$http', '$location', 'Query', WidgetCtrl])
-        .controller('QueriesCtrl', ['$scope', '$http', '$location', '$filter', 'Query', QueriesCtrl])
-        .controller('QueryFiddleCtrl', ['$scope', '$window', '$location', '$routeParams', '$http', '$location', 'growl', 'notifications', 'Query', 'Visualization', QueryFiddleCtrl])
-        .controller('IndexCtrl', ['$scope', 'Dashboard', IndexCtrl])
-        .controller('MainCtrl', ['$scope', 'Dashboard', 'notifications', MainCtrl]);
-})();

rd_ui/app/scripts/controllers/admin_controllers.js

@@ -0,0 +1,31 @@
+(function () {
+  var AdminStatusCtrl = function ($scope, Events, $http, $timeout) {
+    Events.record(currentUser, "view", "page", "admin/status");
+    $scope.$parent.pageTitle = "System Status";
+
+    var refresh = function () {
+      $scope.refresh_time = moment().add('minutes', 1);
+      $http.get('/status.json').success(function (data) {
+        $scope.workers = data.workers;
+        delete data.workers;
+        $scope.manager = data.manager;
+        delete data.manager;
+        $scope.status = data;
+      });
+
+      $timeout(refresh, 59 * 1000);
+    };
+
+    $scope.flowerUrl = featureFlags.flowerUrl;
+
+    refresh();
+  }
+
+  var AdminWorkersCtrl = function ($scope, $sce) {
+    $scope.flowerUrl = $sce.trustAsResourceUrl(featureFlags.flowerUrl);
+  };
+
+  angular.module('redash.admin_controllers', [])
+         .controller('AdminStatusCtrl', ['$scope', 'Events', '$http', '$timeout', AdminStatusCtrl])
+         .controller('AdminWorkersCtrl', ['$scope', '$sce', AdminWorkersCtrl])
+})();

rd_ui/app/scripts/controllers/controllers.js

@@ -0,0 +1,169 @@
+(function () {
+  var QueriesCtrl = function ($scope, $http, $location, $filter, Query) {
+    $scope.$parent.pageTitle = "All Queries";
+    $scope.gridConfig = {
+      isPaginationEnabled: true,
+      itemsByPage: 50,
+      maxSize: 8,
+      isGlobalSearchActivated: true
+    }
+
+    $scope.allQueries = [];
+    $scope.queries = [];
+
+    var dateFormatter = function (value) {
+      if (!value) return "-";
+      return value.format("DD/MM/YY HH:mm");
+    }
+
+    var filterQueries = function () {
+      $scope.queries = _.filter($scope.allQueries, function (query) {
+        if (!$scope.selectedTab) {
+          return false;
+        }
+
+        if ($scope.selectedTab.key == 'my') {
+          return query.user.id == currentUser.id && query.name != 'New Query';
+        } else if ($scope.selectedTab.key == 'drafts') {
+          return query.user.id == currentUser.id && query.name == 'New Query';
+        }
+
+        return query.name != 'New Query';
+      });
+    }
+
+    Query.query(function (queries) {
+      $scope.allQueries = _.map(queries, function (query) {
+        query.created_at = moment(query.created_at);
+        query.last_retrieved_at = moment(query.last_retrieved_at);
+        return query;
+      });
+
+      filterQueries();
+    });
+
+    $scope.gridColumns = [
+      {
+        "label": "Name",
+        "map": "name",
+        "cellTemplateUrl": "/views/queries_query_name_cell.html"
+      },
+      {
+        'label': 'Created By',
+        'map': 'user.name'
+      },
+      {
+        'label': 'Created At',
+        'map': 'created_at',
+        'formatFunction': dateFormatter
+      },
+      {
+        'label': 'Runtime (avg)',
+        'map': 'avg_runtime',
+        'formatFunction': function (value) {
+          return $filter('durationHumanize')(value);
+        }
+      },
+      {
+        'label': 'Runtime (min)',
+        'map': 'min_runtime',
+        'formatFunction': function (value) {
+          return $filter('durationHumanize')(value);
+        }
+      },
+      {
+        'label': 'Runtime (max)',
+        'map': 'max_runtime',
+        'formatFunction': function (value) {
+          return $filter('durationHumanize')(value);
+        }
+      },
+      {
+        'label': 'Last Executed At',
+        'map': 'last_retrieved_at',
+        'formatFunction': dateFormatter
+      },
+      {
+        'label': 'Times Executed',
+        'map': 'times_retrieved'
+      },
+      {
+        'label': 'Update Schedule',
+        'map': 'ttl',
+        'formatFunction': function (value) {
+          return $filter('refreshRateHumanize')(value);
+        }
+      }
+    ]
+    $scope.tabs = [
+      {"name": "My Queries", "key": "my"},
+      {"key": "all", "name": "All Queries"},
+      {"key": "drafts", "name": "Drafts"}
+    ];
+
+    $scope.$watch('selectedTab', function (tab) {
+      if (tab) {
+        $scope.$parent.pageTitle = tab.name;
+      }
+
+      filterQueries();
+    });
+  }
+
+  var MainCtrl = function ($scope, Dashboard, notifications) {
+    if (featureFlags.clientSideMetrics) {
+      $scope.$on('$locationChangeSuccess', function(event, newLocation, oldLocation) {
+        // This will be called once per actual page load.
+        Bucky.sendPagePerformance();
+      });
+    }
+
+
+    $scope.dashboards = [];
+    $scope.reloadDashboards = function () {
+      Dashboard.query(function (dashboards) {
+        $scope.dashboards = _.sortBy(dashboards, "name");
+        $scope.allDashboards = _.groupBy($scope.dashboards, function (d) {
+          parts = d.name.split(":");
+          if (parts.length == 1) {
+            return "Other";
+          }
+          return parts[0];
+        });
+        $scope.otherDashboards = $scope.allDashboards['Other'] || [];
+        $scope.groupedDashboards = _.omit($scope.allDashboards, 'Other');
+      });
+    }
+
+    $scope.reloadDashboards();
+
+    $scope.currentUser = currentUser;
+    $scope.newDashboard = {
+      'name': null,
+      'layout': null
+    }
+
+    $(window).click(function () {
+      notifications.getPermissions();
+    });
+  }
+
+  var IndexCtrl = function ($scope, Events, Dashboard) {
+    Events.record(currentUser, "view", "page", "homepage");
+    $scope.$parent.pageTitle = "Home";
+
+    $scope.archiveDashboard = function (dashboard) {
+      if (confirm('Are you sure you want to delete "' + dashboard.name + '" dashboard?')) {
+        Events.record(currentUser, "archive", "dashboard", dashboard.id);
+        dashboard.$delete(function () {
+          $scope.$parent.reloadDashboards();
+        });
+      }
+    }
+  }
+
+  angular.module('redash.controllers', [])
+    .controller('QueriesCtrl', ['$scope', '$http', '$location', '$filter', 'Query', QueriesCtrl])
+    .controller('IndexCtrl', ['$scope', 'Events', 'Dashboard', IndexCtrl])
+    .controller('MainCtrl', ['$scope', 'Dashboard', 'notifications', MainCtrl]);
+})();

rd_ui/app/scripts/controllers/dashboard.js

@@ -0,0 +1,143 @@
+(function() {
+  var DashboardCtrl = function($scope, Events, Widget, $routeParams, $http, $timeout, $q, Dashboard) {
+    $scope.refreshEnabled = false;
+    $scope.refreshRate = 60;
+
+    var loadDashboard = _.throttle(function() {
+      $scope.dashboard = Dashboard.get({ slug: $routeParams.dashboardSlug }, function (dashboard) {
+        Events.record(currentUser, "view", "dashboard", dashboard.id);
+
+        $scope.$parent.pageTitle = dashboard.name;
+
+        var promises = [];
+
+        $scope.dashboard.widgets = _.map($scope.dashboard.widgets, function (row) {
+          return _.map(row, function (widget) {
+            var w = new Widget(widget);
+
+            if (w.visualization && dashboard.dashboard_filters_enabled) {
+              promises.push(w.getQuery().getQueryResultPromise());
+            }
+
+            return w;
+          });
+        });
+
+        $q.all(promises).then(function(queryResults) {
+          var filters = {};
+          _.each(queryResults, function(queryResult) {
+            var queryFilters = queryResult.getFilters();
+            _.each(queryFilters, function (filter) {
+              if (!_.has(filters, filter.name)) {
+                // TODO: first object should be a copy, otherwise one of the chart filters behaves different than the others.
+                filters[filter.name] = filter;
+                filters[filter.name].originFilters = [];
+
+                $scope.$watch(function () { return filter.current }, function (value) {
+                  _.each(filter.originFilters, function (originFilter) {
+                    originFilter.current = value;
+                  });
+                });
+              };
+
+              // TODO: merge values.
+              filters[filter.name].originFilters.push(filter);
+            });
+          });
+
+          if (dashboard.dashboard_filters_enabled) {
+            $scope.filters = _.values(filters);
+          }
+        });
+
+
+      }, function () {
+        // error...
+        // try again. we wrap loadDashboard with throttle so it doesn't happen too often.\
+        // we might want to consider exponential backoff and also move this as a general solution in $http/$resource for
+        // all AJAX calls.
+        loadDashboard();
+      });
+    }, 1000);
+
+    loadDashboard();
+
+    var autoRefresh = function() {
+      if ($scope.refreshEnabled) {
+        $timeout(function() {
+          Dashboard.get({
+            slug: $routeParams.dashboardSlug
+          }, function(dashboard) {
+            var newWidgets = _.groupBy(_.flatten(dashboard.widgets), 'id');
+
+            _.each($scope.dashboard.widgets, function(row) {
+              _.each(row, function(widget, i) {
+                var newWidget = newWidgets[widget.id];
+                if (newWidget && newWidget[0].visualization.query.latest_query_data_id != widget.visualization.query.latest_query_data_id) {
+                  row[i] = newWidget[0];
+                }
+              });
+            });
+
+            autoRefresh();
+          });
+
+        }, $scope.refreshRate);
+      };
+    }
+
+    $scope.triggerRefresh = function() {
+      $scope.refreshEnabled = !$scope.refreshEnabled;
+
+      Events.record(currentUser, "autorefresh", "dashboard", dashboard.id, {'enable': $scope.refreshEnabled});
+
+      if ($scope.refreshEnabled) {
+        var refreshRate = _.min(_.flatten($scope.dashboard.widgets), function(widget) {
+          return widget.visualization.query.ttl;
+        }).visualization.query.ttl;
+
+        $scope.refreshRate = _.max([120, refreshRate * 2]) * 1000;
+
+        autoRefresh();
+      }
+    };
+  };
+
+  var WidgetCtrl = function($scope, Events, Query) {
+    $scope.deleteWidget = function() {
+      if (!confirm('Are you sure you want to remove "' + $scope.widget.getName() + '" from the dashboard?')) {
+        return;
+      }
+
+      Events.record(currentUser, "delete", "widget", $scope.widget.id);
+
+      $scope.widget.$delete(function() {
+        $scope.dashboard.widgets = _.map($scope.dashboard.widgets, function(row) {
+          return _.filter(row, function(widget) {
+            return widget.id != undefined;
+          })
+        });
+      });
+    };
+
+    Events.record(currentUser, "view", "widget", $scope.widget.id);
+
+    if ($scope.widget.visualization) {
+      Events.record(currentUser, "view", "query", $scope.widget.visualization.query.id);
+      Events.record(currentUser, "view", "visualization", $scope.widget.visualization.id);
+
+      $scope.query = $scope.widget.getQuery();
+      $scope.queryResult = $scope.query.getQueryResult();
+      $scope.nextUpdateTime = moment(new Date(($scope.query.updated_at + $scope.query.ttl + $scope.query.runtime + 300) * 1000)).fromNow();
+
+      $scope.type = 'visualization';
+    } else {
+      $scope.type = 'textbox';
+    }
+  };
+
+  angular.module('redash.controllers')
+    .controller('DashboardCtrl', ['$scope', 'Events', 'Widget', '$routeParams', '$http', '$timeout', '$q', 'Dashboard', DashboardCtrl])
+    .controller('WidgetCtrl', ['$scope', 'Events', 'Query', WidgetCtrl])
+
+})();

rd_ui/app/scripts/controllers/query_source.js

@@ -0,0 +1,120 @@
+(function() {
+  'use strict';
+
+  function QuerySourceCtrl(Events, growl, $controller, $scope, $location, Query, Visualization, KeyboardShortcuts) {
+    // extends QueryViewCtrl
+    $controller('QueryViewCtrl', {$scope: $scope});
+    // TODO:
+    // This doesn't get inherited. Setting it on this didn't work either (which is weird).
+    // Obviously it shouldn't be repeated, but we got bigger fish to fry.
+    var DEFAULT_TAB = 'table';
+
+    Events.record(currentUser, 'view_source', 'query', $scope.query.id);
+
+    var isNewQuery = !$scope.query.id,
+        queryText = $scope.query.query,
+        // ref to QueryViewCtrl.saveQuery
+        saveQuery = $scope.saveQuery,
+        shortcuts = {
+          'meta+s': function () {
+            if ($scope.canEdit) {
+              $scope.saveQuery();
+            }
+          },
+          'meta+enter': function () {
+            $scope.executeQuery();
+          }
+        };
+
+    $scope.sourceMode = true;
+    $scope.canEdit = currentUser.canEdit($scope.query);
+    $scope.isDirty = false;
+
+    $scope.newVisualization = undefined;
+
+    // @override
+    Object.defineProperty($scope, 'showDataset', {
+      get: function() {
+        return $scope.queryResult && $scope.queryResult.getStatus() == 'done';
+      }
+    });
+
+
+    KeyboardShortcuts.bind(shortcuts);
+
+    // @override
+    $scope.saveQuery = function(options, data) {
+      var savePromise = saveQuery(options, data);
+
+      savePromise.then(function(savedQuery) {
+        queryText = savedQuery.query;
+        $scope.isDirty = $scope.query.query !== queryText;
+
+        if (isNewQuery) {
+          // redirect to new created query (keep hash)
+          $location.path(savedQuery.getSourceLink()).replace();
+        }
+      });
+
+      return savePromise;
+    };
+
+    $scope.duplicateQuery = function() {
+      Events.record(currentUser, 'fork', 'query', $scope.query.id);
+      $scope.query.id = null;
+      $scope.query.ttl = -1;
+
+      $scope.saveQuery({
+        successMessage: 'Query forked',
+        errorMessage: 'Query could not be forked'
+      }).then(function redirect(savedQuery) {
+        // redirect to forked query (clear hash)
+        $location.url(savedQuery.getSourceLink()).replace()
+      });
+    };
+
+    $scope.deleteVisualization = function($e, vis) {
+      $e.preventDefault();
+      if (confirm('Are you sure you want to delete ' + vis.name + ' ?')) {
+        Events.record(currentUser, 'delete', 'visualization', vis.id);
+
+        Visualization.delete(vis, function() {
+          if ($scope.selectedTab == vis.id) {
+            $scope.selectedTab = DEFAULT_TAB;
+            $location.hash($scope.selectedTab);
+          }
+          $scope.query.visualizations =
+            $scope.query.visualizations.filter(function (v) {
+              return vis.id !== v.id;
+            });
+        }, function () {
+          growl.addErrorMessage("Error deleting visualization. Maybe it's used in a dashboard?");
+        });
+
+      }
+    };
+
+    $scope.$watch('query.query', function(newQueryText) {
+      $scope.isDirty = (newQueryText !== queryText);
+    });
+
+    $scope.$on('$destroy', function destroy() {
+      KeyboardShortcuts.unbind(shortcuts);
+    });
+
+    if (isNewQuery) {
+      // save new query when creating a visualization
+      var unbind = $scope.$watch('selectedTab == "add"', function(triggerSave) {
+        if (triggerSave) {
+          unbind();
+          $scope.saveQuery();
+        }
+      });
+    }
+  }
+
+  angular.module('redash.controllers').controller('QuerySourceCtrl', [
+    'Events', 'growl', '$controller', '$scope', '$location', 'Query',
+    'Visualization', 'KeyboardShortcuts', QuerySourceCtrl
+    ]);
+})();

rd_ui/app/scripts/controllers/query_view.js

@@ -0,0 +1,144 @@
+(function() {
+  'use strict';
+
+  function QueryViewCtrl($scope, Events, $route, $location, notifications, growl, Query, DataSource) {
+    var DEFAULT_TAB = 'table';
+
+    $scope.query = $route.current.locals.query;
+    Events.record(currentUser, 'view', 'query', $scope.query.id);
+    $scope.queryResult = $scope.query.getQueryResult();
+    $scope.queryExecuting = false;
+
+    $scope.isQueryOwner = currentUser.id === $scope.query.user.id;
+    $scope.canViewSource = currentUser.hasPermission('view_source');
+
+    $scope.dataSources = DataSource.get(function(dataSources) {
+      $scope.query.data_source_id = $scope.query.data_source_id || dataSources[0].id;
+    });
+
+    // in view mode, latest dataset is always visible
+    // source mode changes this behavior
+    $scope.showDataset = true;
+
+    $scope.lockButton = function(lock) {
+      $scope.queryExecuting = lock;
+    };
+
+    $scope.saveQuery = function(options, data) {
+      if (data) {
+        data.id = $scope.query.id;
+      } else {
+        data = _.clone($scope.query);
+      }
+
+      options = _.extend({}, {
+        successMessage: 'Query saved',
+        errorMessage: 'Query could not be saved'
+      }, options);
+
+      delete data.latest_query_data;
+      delete data.queryResult;
+
+      return Query.save(data, function() {
+        growl.addSuccessMessage(options.successMessage);
+      }, function(httpResponse) {
+        growl.addErrorMessage(options.errorMessage);
+      }).$promise;
+    }
+
+    $scope.saveDescription = function() {
+      Events.record(currentUser, 'edit_description', 'query', $scope.query.id);
+      $scope.saveQuery(undefined, {'description': $scope.query.description});
+    };
+
+    $scope.saveName = function() {
+      Events.record(currentUser, 'edit_name', 'query', $scope.query.id);
+      $scope.saveQuery(undefined, {'name': $scope.query.name});
+    };
+
+    $scope.executeQuery = function() {
+      $scope.queryResult = $scope.query.getQueryResult(0);
+      $scope.lockButton(true);
+      $scope.cancelling = false;
+      Events.record(currentUser, 'execute', 'query', $scope.query.id);
+    };
+
+    $scope.cancelExecution = function() {
+      $scope.cancelling = true;
+      $scope.queryResult.cancelExecution();
+      Events.record(currentUser, 'cancel_execute', 'query', $scope.query.id);
+    };
+
+    $scope.updateDataSource = function() {
+      Events.record(currentUser, 'update_data_source', 'query', $scope.query.id);
+
+      $scope.query.latest_query_data = null;
+      $scope.query.latest_query_data_id = null;
+
+      if ($scope.query.id) {
+        Query.save({
+          'id': $scope.query.id,
+          'data_source_id': $scope.query.data_source_id,
+          'latest_query_data_id': null
+        });
+      }
+
+      $scope.executeQuery();
+    };
+
+    $scope.setVisualizationTab = function (visualization) {
+      $scope.selectedTab = visualization.id;
+      $location.hash(visualization.id);
+    };
+
+    $scope.$watch('query.name', function() {
+      $scope.$parent.pageTitle = $scope.query.name;
+    });
+
+    $scope.$watch('queryResult && queryResult.getData()', function(data, oldData) {
+      if (!data) {
+        return;
+      }
+
+      $scope.filters = $scope.queryResult.getFilters();
+    });
+
+    $scope.$watch("queryResult && queryResult.getStatus()", function(status) {
+      if (!status) {
+        return;
+      }
+
+      if (status == 'done') {
+        if ($scope.query.id &&
+          $scope.query.latest_query_data_id != $scope.queryResult.getId() &&
+          $scope.query.query_hash == $scope.queryResult.query_result.query_hash) {
+          Query.save({
+            'id': $scope.query.id,
+            'latest_query_data_id': $scope.queryResult.getId()
+          })
+        }
+        $scope.query.latest_query_data_id = $scope.queryResult.getId();
+        $scope.query.queryResult = $scope.queryResult;
+
+        notifications.showNotification("re:dash", $scope.query.name + " updated.");
+      }
+
+      if (status === 'done' || status === 'failed') {
+        $scope.lockButton(false);
+      }
+    });
+
+    $scope.$watch(function() {
+      return $location.hash()
+    }, function(hash) {
+      if (hash == 'pivot') {
+        Events.record(currentUser, 'pivot', 'query', $scope.query && $scope.query.id);
+      }
+      $scope.selectedTab = hash || DEFAULT_TAB;
+    });
+  };
+
+  angular.module('redash.controllers')
+    .controller('QueryViewCtrl',
+      ['$scope', 'Events', '$route', '$location', 'notifications', 'growl', 'Query', 'DataSource', QueryViewCtrl]);
+})();

rd_ui/app/scripts/directives.js

@@ -1,436 +0,0 @@
-(function() {
-    'use strict';
-
-    var directives = angular.module('redash.directives', []);
-
-    directives.directive('rdTab', ['$location', function($location) {
-        return {
-            restrict: 'E',
-            scope: {
-                'id': '@',
-                'name': '@'
-            },
-            transclude: true,
-            template: '<li class="rd-tab" ng-class="{active: id==selectedTab}"><a href="#{{id}}">{{name}}<span ng-transclude></span></a></li>',
-            replace: true,
-            link: function(scope) {
-                scope.$watch(function(){return scope.$parent.selectedTab}, function(tab) {
-                    scope.selectedTab = tab;
-                });
-            }
-        }
-    }]);
-
-    directives.directive('rdTabs', ['$location', '$rootScope', function($location, $rootScope) {
-        return {
-            restrict: 'E',
-            scope: {
-                tabsCollection: '=',
-                selectedTab: '='
-            },
-            template: '<ul class="nav nav-tabs"><li ng-class="{active: tab==selectedTab}" ng-repeat="tab in tabsCollection"><a href="#{{tab.key}}">{{tab.name}}</a></li></ul>',
-            replace: true,
-            link: function($scope, element, attrs) {
-                $scope.selectTab = function(tabKey) {
-                    $scope.selectedTab  = _.find($scope.tabsCollection, function(tab) { return tab.key == tabKey; });
-                }
-
-                $scope.$watch(function() { return $location.hash()}, function(hash) {
-                    if (hash) {
-                        $scope.selectTab($location.hash());
-                    } else {
-                        $scope.selectTab($scope.tabsCollection[0].key);
-                    }
-                });
-            }
-        }
-    }]);
-
-    directives.directive('editVisulatizationForm', ['Visualization', 'growl', function(Visualization, growl) {
-        return {
-            restrict: 'E',
-            templateUrl: '/views/edit_visualization.html',
-            replace: true,
-            scope: {
-                query: '=',
-                vis: '=?'
-            },
-            link: function(scope, element, attrs) {
-                scope.advancedMode = false;
-                scope.visTypes = {
-                    'Chart': Visualization.prototype.TYPES.CHART,
-                    'Cohort': Visualization.prototype.TYPES.COHORT
-                };
-                scope.seriesTypes = {
-                    'Line': 'line',
-                    'Column': 'column',
-                    'Area': 'area',
-                    'Scatter': 'scatter',
-                    'Pie': 'pie'
-                };
-
-                scope.stackingOptions = {
-                    "None": "none",
-                    "Normal": "normal",
-                    "Percent": "percent"
-                };
-
-                scope.stacking = "none";
-
-                if (!scope.vis) {
-                    // create new visualization
-                    // wait for query to load to populate with defaults
-                    var unwatch = scope.$watch('query', function(q) {
-                        if (q && q.id) {
-                            unwatch();
-                            scope.vis = {
-                                'query_id': q.id,
-                                'type': Visualization.prototype.TYPES.CHART,
-                                'name': '',
-                                'description': q.description || '',
-                                'options': newOptions(Visualization.prototype.TYPES.CHART)
-                            };
-                        }
-                    }, true);
-                }
-
-                function newOptions(chartType) {
-                    if (chartType === Visualization.prototype.TYPES.CHART) {
-                        return {
-                            'series': {
-                                'type': 'column',
-                                'stacking': null
-                            }
-                        };
-                    };
-
-                    return {};
-                }
-
-                var chartOptionsUnwatch = null;
-
-                scope.$watch('vis.type', function(type) {
-                    // if not edited by user, set name to match type
-                    if (type && scope.vis && !scope.visForm.name.$dirty) {
-                        // poor man's titlecase
-                        scope.vis.name = scope.vis.type[0] + scope.vis.type.slice(1).toLowerCase();
-                    }
-
-                    if (type && type == Visualization.prototype.TYPES.CHART) {
-                        if (scope.vis.options.series.stacking === null) {
-                            scope.stacking = "none";
-                        } else if (scope.vis.options.series.stacking === undefined) {
-                            scope.stacking = "normal";
-                        } else {
-                            scope.stacking = scope.vis.options.series.stacking ;
-                        }
-
-                        chartOptionsUnwatch = scope.$watch("stacking", function(stacking) {
-                            if (stacking == "none") {
-                                scope.vis.options.series.stacking = null;
-                            } else {
-                                scope.vis.options.series.stacking = stacking;
-                            }
-                        });
-                    } else {
-                        if (chartOptionsUnwatch) {
-                            chartOptionsUnwatch();
-                            chartOptionsUnwatch = null;
-                        }
-                    }
-                });
-
-                scope.toggleAdvancedMode = function() {
-                    scope.advancedMode = !scope.advancedMode;
-                };
-
-                scope.typeChanged = function() {
-                    scope.vis.options = newOptions(scope.vis.type);
-                };
-
-                scope.submit = function() {
-                    Visualization.save(scope.vis, function success(result) {
-                        growl.addSuccessMessage("Visualization saved");
-
-                        scope.vis = result;
-
-                        var visIds = _.pluck(scope.query.visualizations, 'id');
-                        var index = visIds.indexOf(result.id);
-                        if (index > -1) {
-                            scope.query.visualizations[index] = result;
-                        } else {
-                            scope.query.visualizations.push(result);
-                        }
-                    }, function error() {
-                        growl.addErrorMessage("Visualization could not be saved");
-                    });
-                };
-            }
-        }
-    }]);
-
-    directives.directive('editDashboardForm', ['$http', '$location', '$timeout', 'Dashboard', function($http, $location, $timeout, Dashboard) {
-        return {
-            restrict: 'E',
-            scope: {
-                dashboard: '='
-            },
-            templateUrl: '/views/edit_dashboard.html',
-            replace: true,
-            link: function($scope, element, attrs) {
-                var gridster = element.find(".gridster ul").gridster({
-                    widget_margins: [5, 5],
-                    widget_base_dimensions: [260, 100],
-                    min_cols: 2,
-                    max_cols: 2,
-                    serialize_params: function($w, wgd) {
-                        return {
-                            col: wgd.col,
-                            row: wgd.row,
-                            id: $w.data('widget-id')
-                        }
-                    }
-                }).data('gridster');
-
-                var gsItemTemplate = '<li data-widget-id="{id}" class="widget panel panel-default gs-w">' +
-                                 '<div class="panel-heading">{name}' +
-                                 '</div></li>';
-
-                $scope.$watch('dashboard.widgets', function(widgets) {
-                    $timeout(function () {
-                        gridster.remove_all_widgets();
-
-                        if (widgets && widgets.length) {
-                            var layout = [];
-
-                            _.each(widgets, function(row, rowIndex) {
-                                _.each(row, function(widget, colIndex) {
-                                    layout.push({
-                                        id: widget.id,
-                                        col: colIndex+1,
-                                        row: rowIndex+1,
-                                        ySize: 1,
-                                        xSize: widget.width,
-                                        name: widget.visualization.query.name
-                                    });
-                                });
-                            });
-
-                            _.each(layout, function(item) {
-                                var el = gsItemTemplate.replace('{id}', item.id).replace('{name}', item.name);
-                                gridster.add_widget(el, item.xSize, item.ySize, item.col, item.row);
-
-                            });
-                        }
-                    });
-                }, true);
-
-                $scope.saveDashboard = function() {
-                    $scope.saveInProgress = true;
-                    // TODO: we should use the dashboard service here.
-                    if ($scope.dashboard.id) {
-                        var positions = $(element).find('.gridster ul').data('gridster').serialize();
-                        var layout = [];
-                        _.each(_.sortBy(positions, function (pos) {
-                            return pos.row * 10 + pos.col;
-                        }), function (pos) {
-                            var row = pos.row - 1;
-                            var col = pos.col - 1;
-                            layout[row] = layout[row] || [];
-                            if (col > 0 && layout[row][col - 1] == undefined) {
-                                layout[row][col - 1] = pos.id;
-                            } else {
-                                layout[row][col] = pos.id;
-                            }
-
-                        });
-                        $scope.dashboard.layout = layout;
-
-                        layout = JSON.stringify(layout);
-                        $http.post('/api/dashboards/' + $scope.dashboard.id, {'name': $scope.dashboard.name, 'layout': layout}).success(function(response) {
-                            $scope.dashboard = new Dashboard(response);
-                            $scope.saveInProgress = false;
-                            $(element).modal('hide');
-                        })
-                    } else {
-                        $http.post('/api/dashboards', {'name': $scope.dashboard.name}).success(function(response) {
-                            $(element).modal('hide');
-                            $location.path('/dashboard/' + response.slug).replace();
-                        })
-                    }
-                }
-
-            }
-        }
-    }]);
-
-    directives.directive('newWidgetForm', ['$http', 'Query', function($http, Query) {
-        return {
-            restrict: 'E',
-            scope: {
-                dashboard: '='
-            },
-            templateUrl: '/views/new_widget_form.html',
-            replace: true,
-            link: function($scope, element, attrs) {
-                $scope.widgetSizes = [{name: 'Regular', value: 1}, {name: 'Double', value: 2}];
-
-                var reset = function() {
-                    $scope.saveInProgress = false;
-                    $scope.widgetSize = 1;
-                    $scope.queryId = null;
-                    $scope.selectedVis = null;
-
-                }
-
-                reset();
-
-                $scope.loadVisualizations = function() {
-                    if (!$scope.queryId) {
-                        return;
-                    }
-
-                    Query.get({
-                        id: $scope.queryId
-                    }, function(query) {
-                        if (query) {
-                            $scope.query = query;
-                            if(query.visualizations.length) {
-                                $scope.selectedVis = query.visualizations[0];
-                            }
-                        }
-                    });
-                };
-
-                $scope.saveWidget = function() {
-                    $scope.saveInProgress = true;
-
-                    var widget = {
-                        'visualization_id': $scope.selectedVis.id,
-                        'dashboard_id': $scope.dashboard.id,
-                        'options': {},
-                        'width': $scope.widgetSize
-                    }
-
-                    $http.post('/api/widgets', widget).success(function(response) {
-                        // update dashboard layout
-                        $scope.dashboard.layout = response['layout'];
-                        if (response['new_row']) {
-                            $scope.dashboard.widgets.push([response['widget']]);
-                        } else {
-                            $scope.dashboard.widgets[$scope.dashboard.widgets.length-1].push(response['widget']);
-                        }
-
-                        // close the dialog
-                        $('#add_query_dialog').modal('hide');
-                        reset();
-                    })
-                }
-
-            }
-        }
-    }])
-
-    // From: http://jsfiddle.net/joshdmiller/NDFHg/
-    directives.directive('editInPlace', function () {
-        return {
-            restrict: 'E',
-            scope: {
-                value: '=',
-                ignoreBlanks: '=',
-                editable: '='
-            },
-            template: function(tElement, tAttrs) {
-                var elType = tAttrs.editor || 'input';
-                var placeholder = tAttrs.placeholder || 'Click to edit';
-                return '<span ng-click="editable && edit()" ng-bind="value" ng-class="{editable: editable}"></span>' +
-                       '<span ng-click="editable && edit()" ng-show="editable && !value" ng-class="{editable: editable}">' + placeholder + '</span>' +
-                       '<{elType} ng-model="value" class="form-control" rows="2"></{elType}>'.replace('{elType}', elType);
-            },
-            link: function ($scope, element, attrs) {
-                // Let's get a reference to the input element, as we'll want to reference it.
-                var inputElement = angular.element(element.children()[2]);
-
-                // This directive should have a set class so we can style it.
-                element.addClass('edit-in-place');
-
-                // Initially, we're not editing.
-                $scope.editing = false;
-
-                // ng-click handler to activate edit-in-place
-                $scope.edit = function () {
-                    if ($scope.ignoreBlanks) {
-                        $scope.oldValue = $scope.value;
-                    }
-
-                    $scope.editing = true;
-
-                    // We control display through a class on the directive itself. See the CSS.
-                    element.addClass('active');
-
-                    // And we must focus the element.
-                    // `angular.element()` provides a chainable array, like jQuery so to access a native DOM function,
-                    // we have to reference the first element in the array.
-                    inputElement[0].focus();
-                };
-
-                $(inputElement).blur(function() {
-                    if ($scope.ignoreBlanks && _.isEmpty($scope.value)) {
-                        $scope.value = $scope.oldValue;
-                    }
-                    $scope.editing = false;
-                    element.removeClass('active');
-                })
-            }
-        };
-    });
-
-    // http://stackoverflow.com/a/17904092/1559840
-    directives.directive('jsonText', function() {
-        return {
-            restrict: 'A',
-            require: 'ngModel',
-            link: function(scope, element, attr, ngModel) {
-              function into(input) {
-                return JSON.parse(input);
-              }
-              function out(data) {
-                return JSON.stringify(data, undefined, 2);
-              }
-              ngModel.$parsers.push(into);
-              ngModel.$formatters.push(out);
-
-            }
-        };
-    });
-
-    directives.directive('rdTimer', ['$timeout', function ($timeout) {
-        return {
-            restrict: 'E',
-            scope: { timestamp: '=' },
-            template: '{{currentTime}}',
-            controller: ['$scope' ,function ($scope) {
-                $scope.currentTime = "00:00:00";
-                var currentTimeout = null;
-
-                var updateTime = function() {
-                    $scope.currentTime = moment(moment() - moment($scope.timestamp)).utc().format("HH:mm:ss")
-                    currentTimeout = $timeout(updateTime, 1000);
-                }
-
-                var cancelTimer = function() {
-                    if (currentTimeout) {
-                        $timeout.cancel(currentTimeout);
-                        currentTimeout = null;
-                    }
-                }
-
-                updateTime();
-
-                $scope.$on('$destroy', function () {
-                    cancelTimer();
-                });
-            }]
-        };
-    }]);
-})();

rd_ui/app/scripts/directives/dashboard_directives.js

@@ -0,0 +1,208 @@
+(function() {
+  'use strict'
+
+  var directives = angular.module('redash.directives');
+
+  directives.directive('editDashboardForm', ['Events', '$http', '$location', '$timeout', 'Dashboard',
+    function(Events, $http, $location, $timeout, Dashboard) {
+      return {
+        restrict: 'E',
+        scope: {
+          dashboard: '='
+        },
+        templateUrl: '/views/edit_dashboard.html',
+        replace: true,
+        link: function($scope, element, attrs) {
+          var gridster = element.find(".gridster ul").gridster({
+            widget_margins: [5, 5],
+            widget_base_dimensions: [260, 100],
+            min_cols: 2,
+            max_cols: 2,
+            serialize_params: function($w, wgd) {
+              return {
+                col: wgd.col,
+                row: wgd.row,
+                id: $w.data('widget-id')
+              }
+            }
+          }).data('gridster');
+
+          var gsItemTemplate = '<li data-widget-id="{id}" class="widget panel panel-default gs-w">' +
+            '<div class="panel-heading">{name}' +
+            '</div></li>';
+
+          $scope.$watch('dashboard.widgets && dashboard.widgets.length', function(widgets_length) {
+            $timeout(function() {
+              gridster.remove_all_widgets();
+
+              if ($scope.dashboard.widgets && $scope.dashboard.widgets.length) {
+                var layout = [];
+
+                _.each($scope.dashboard.widgets, function(row, rowIndex) {
+                  _.each(row, function(widget, colIndex) {
+                    layout.push({
+                      id: widget.id,
+                      col: colIndex + 1,
+                      row: rowIndex + 1,
+                      ySize: 1,
+                      xSize: widget.width,
+                      name: widget.getName()//visualization.query.name
+                    });
+                  });
+                });
+
+                _.each(layout, function(item) {
+                  var el = gsItemTemplate.replace('{id}', item.id).replace('{name}', item.name);
+                  gridster.add_widget(el, item.xSize, item.ySize, item.col, item.row);
+                });
+              }
+            });
+          });
+
+          $scope.saveDashboard = function() {
+            $scope.saveInProgress = true;
+            // TODO: we should use the dashboard service here.
+            if ($scope.dashboard.id) {
+              var positions = $(element).find('.gridster ul').data('gridster').serialize();
+              var layout = [];
+              _.each(_.sortBy(positions, function(pos) {
+                return pos.row * 10 + pos.col;
+              }), function(pos) {
+                var row = pos.row - 1;
+                var col = pos.col - 1;
+                layout[row] = layout[row] || [];
+                if (col > 0 && layout[row][col - 1] == undefined) {
+                  layout[row][col - 1] = pos.id;
+                } else {
+                  layout[row][col] = pos.id;
+                }
+
+              });
+              $scope.dashboard.layout = layout;
+
+              layout = JSON.stringify(layout);
+              $http.post('/api/dashboards/' + $scope.dashboard.id, {
+                'name': $scope.dashboard.name,
+                'layout': layout
+              }).success(function(response) {
+                $scope.dashboard = new Dashboard(response);
+                $scope.saveInProgress = false;
+                $(element).modal('hide');
+              });
+              Events.record(currentUser, 'edit', 'dashboard', $scope.dashboard.id);
+            } else {
+
+              $http.post('/api/dashboards', {
+                'name': $scope.dashboard.name
+              }).success(function(response) {
+                $(element).modal('hide');
+                $scope.dashboard = {
+                  'name': null,
+                  'layout': null
+                };
+                $scope.saveInProgress = false;
+                $location.path('/dashboard/' + response.slug).replace();
+              });
+              Events.record(currentUser, 'create', 'dashboard');
+            }
+          }
+
+        }
+      }
+    }
+  ]);
+
+  directives.directive('newWidgetForm', ['Query', 'Widget', 'growl',
+    function(Query, Widget, growl) {
+      return {
+        restrict: 'E',
+        scope: {
+          dashboard: '='
+        },
+        templateUrl: '/views/new_widget_form.html',
+        replace: true,
+        link: function($scope, element, attrs) {
+          $scope.widgetSizes = [{
+            name: 'Regular',
+            value: 1
+          }, {
+            name: 'Double',
+            value: 2
+          }];
+
+          $scope.type = 'visualization';
+
+          $scope.isVisualization = function () {
+            return $scope.type == 'visualization';
+          };
+
+          $scope.isTextBox = function () {
+            return $scope.type == 'textbox';
+          };
+
+          $scope.setType = function (type) {
+            $scope.type = type;
+          };
+
+          var reset = function() {
+            $scope.saveInProgress = false;
+            $scope.widgetSize = 1;
+            $scope.queryId = null;
+            $scope.selectedVis = null;
+            $scope.query = null;
+            $scope.text = "";
+          };
+
+          reset();
+
+          $scope.loadVisualizations = function () {
+            if (!$scope.queryId) {
+              return;
+            }
+
+            Query.get({ id: $scope.queryId }, function(query) {
+              if (query) {
+                $scope.query = query;
+                if (query.visualizations.length) {
+                  $scope.selectedVis = query.visualizations[0];
+                }
+              }
+            });
+          };
+
+          $scope.saveWidget = function() {
+            $scope.saveInProgress = true;
+
+            var widget = new Widget({
+              'visualization_id': $scope.selectedVis && $scope.selectedVis.id,
+              'dashboard_id': $scope.dashboard.id,
+              'options': {},
+              'width': $scope.widgetSize,
+              'text': $scope.text
+            });
+
+            widget.$save().then(function(response) {
+              // update dashboard layout
+              $scope.dashboard.layout = response['layout'];
+              var newWidget = new Widget(response['widget']);
+              if (response['new_row']) {
+                $scope.dashboard.widgets.push([newWidget]);
+              } else {
+                $scope.dashboard.widgets[$scope.dashboard.widgets.length - 1].push(newWidget);
+              }
+
+              // close the dialog
+              $('#add_query_dialog').modal('hide');
+              reset();
+            }).catch(function() {
+              growl.addErrorMessage("Widget can not be added");
+            }).finally(function() {
+              $scope.saveInProgress = false;
+            });
+          }
+
+        }
+      }
+    }
+  ])
+})();

rd_ui/app/scripts/directives/directives.js

@@ -0,0 +1,227 @@
+(function () {
+  'use strict';
+
+  var directives = angular.module('redash.directives', []);
+
+  directives.directive('alertUnsavedChanges', ['$window', function ($window) {
+    return {
+      restrict: 'E',
+      replace: true,
+      scope: {
+        'isDirty': '='
+      },
+      link: function ($scope) {
+        var
+
+          unloadMessage = "You will lose your changes if you leave",
+          confirmMessage = unloadMessage + "\n\nAre you sure you want to leave this page?",
+
+        // store original handler (if any)
+          _onbeforeunload = $window.onbeforeunload;
+
+        $window.onbeforeunload = function () {
+          return $scope.isDirty ? unloadMessage : null;
+        }
+
+        $scope.$on('$locationChangeStart', function (event, next, current) {
+          if (next.split("#")[0] == current.split("#")[0]) {
+            return;
+          }
+
+          if ($scope.isDirty && !confirm(confirmMessage)) {
+            event.preventDefault();
+          }
+        });
+
+        $scope.$on('$destroy', function () {
+          $window.onbeforeunload = _onbeforeunload;
+        });
+      }
+    }
+  }]);
+
+  directives.directive('rdTab', function () {
+    return {
+      restrict: 'E',
+      scope: {
+        'tabId': '@',
+        'name': '@'
+      },
+      transclude: true,
+      template: '<li class="rd-tab" ng-class="{active: tabId==selectedTab}"><a href="#{{tabId}}">{{name}}<span ng-transclude></span></a></li>',
+      replace: true,
+      link: function (scope) {
+        scope.$watch(function () {
+          return scope.$parent.selectedTab
+        }, function (tab) {
+          scope.selectedTab = tab;
+        });
+      }
+    }
+  });
+
+  directives.directive('rdTabs', ['$location', function ($location) {
+    return {
+      restrict: 'E',
+      scope: {
+        tabsCollection: '=',
+        selectedTab: '='
+      },
+      template: '<ul class="nav nav-tabs"><li ng-class="{active: tab==selectedTab}" ng-repeat="tab in tabsCollection"><a href="#{{tab.key}}">{{tab.name}}</a></li></ul>',
+      replace: true,
+      link: function ($scope, element, attrs) {
+        $scope.selectTab = function (tabKey) {
+          $scope.selectedTab = _.find($scope.tabsCollection, function (tab) {
+            return tab.key == tabKey;
+          });
+        }
+
+        $scope.$watch(function () {
+          return $location.hash()
+        }, function (hash) {
+          if (hash) {
+            $scope.selectTab($location.hash());
+          } else {
+            $scope.selectTab($scope.tabsCollection[0].key);
+          }
+        });
+      }
+    }
+  }]);
+
+  // From: http://jsfiddle.net/joshdmiller/NDFHg/
+  directives.directive('editInPlace', function () {
+    return {
+      restrict: 'E',
+      scope: {
+        value: '=',
+        ignoreBlanks: '=',
+        editable: '=',
+        done: '='
+      },
+      template: function (tElement, tAttrs) {
+        var elType = tAttrs.editor || 'input';
+        var placeholder = tAttrs.placeholder || 'Click to edit';
+        return '<span ng-click="editable && edit()" ng-bind="value" ng-class="{editable: editable}"></span>' +
+          '<span ng-click="editable && edit()" ng-show="editable && !value" ng-class="{editable: editable}">' + placeholder + '</span>' +
+          '<{elType} ng-model="value" class="rd-form-control"></{elType}>'.replace('{elType}', elType);
+      },
+      link: function ($scope, element, attrs) {
+        // Let's get a reference to the input element, as we'll want to reference it.
+        var inputElement = angular.element(element.children()[2]);
+
+        // This directive should have a set class so we can style it.
+        element.addClass('edit-in-place');
+
+        // Initially, we're not editing.
+        $scope.editing = false;
+
+        // ng-click handler to activate edit-in-place
+        $scope.edit = function () {
+          $scope.oldValue = $scope.value;
+
+          $scope.editing = true;
+
+          // We control display through a class on the directive itself. See the CSS.
+          element.addClass('active');
+
+          // And we must focus the element.
+          // `angular.element()` provides a chainable array, like jQuery so to access a native DOM function,
+          // we have to reference the first element in the array.
+          inputElement[0].focus();
+        };
+
+        function save() {
+          if ($scope.editing) {
+            if ($scope.ignoreBlanks && _.isEmpty($scope.value)) {
+              $scope.value = $scope.oldValue;
+            }
+            $scope.editing = false;
+            element.removeClass('active');
+
+            if ($scope.value !== $scope.oldValue) {
+              $scope.done && $scope.done();
+            }
+          }
+        }
+
+        $(inputElement).keydown(function (e) {
+          // 'return' or 'enter' key pressed
+          // allow 'shift' to break lines
+          if (e.which === 13 && !e.shiftKey) {
+            save();
+          } else if (e.which === 27) {
+            $scope.value = $scope.oldValue;
+            $scope.$apply(function () {
+              $(inputElement[0]).blur();
+            });
+          }
+        }).blur(function () {
+          save();
+        });
+      }
+    };
+  });
+
+  // http://stackoverflow.com/a/17904092/1559840
+  directives.directive('jsonText', function () {
+    return {
+      restrict: 'A',
+      require: 'ngModel',
+      link: function (scope, element, attr, ngModel) {
+        function into(input) {
+          return JSON.parse(input);
+        }
+
+        function out(data) {
+          return JSON.stringify(data, undefined, 2);
+        }
+
+        ngModel.$parsers.push(into);
+        ngModel.$formatters.push(out);
+
+        scope.$watch(attr.ngModel, function (newValue) {
+          element[0].value = out(newValue);
+        }, true);
+      }
+    };
+  });
+
+  directives.directive('rdTimer', [function () {
+    return {
+      restrict: 'E',
+      scope: { timestamp: '=' },
+      template: '{{currentTime}}',
+      controller: ['$scope' , function ($scope) {
+        $scope.currentTime = "00:00:00";
+
+        // We're using setInterval directly instead of $timeout, to avoid using $apply, to
+        // prevent the digest loop being run every second.
+        var currentTimer = setInterval(function () {
+          $scope.currentTime = moment(moment() - moment($scope.timestamp)).utc().format("HH:mm:ss");
+          $scope.$digest();
+        }, 1000);
+
+        $scope.$on('$destroy', function () {
+          if (currentTimer) {
+            clearInterval(currentTimer);
+            currentTimer = null;
+          }
+        });
+      }]
+    };
+  }]);
+
+  directives.directive('rdTimeAgo', function () {
+    return {
+      restrict: 'E',
+      scope: {
+        value: '='
+      },
+      template: '<span>' +
+        '<span ng-show="value" am-time-ago="value"></span>' +
+        '<span ng-hide="value">-</span>' +
+        '</span>'
+    }
+  });
+})();

rd_ui/app/scripts/directives/query_directives.js

@@ -0,0 +1,162 @@
+(function() {
+  'use strict'
+
+  function queryLink() {
+    return {
+      restrict: 'E',
+      scope: {
+        'query': '=',
+        'visualization': '=?'
+      },
+      template: '<a ng-href="{{link}}" class="query-link">{{query.name}}</a>',
+      link: function(scope, element) {
+        scope.link = '/queries/' + scope.query.id;
+        if (scope.visualization) {
+          if (scope.visualization.type === 'TABLE') {
+            // link to hard-coded table tab instead of the (hidden) visualization tab
+            scope.link += '#table';
+          } else {
+            scope.link += '#' + scope.visualization.id;
+          }
+        }
+        // element.find('a').attr('href', link);
+      }
+    }
+  }
+
+  function querySourceLink() {
+    return {
+      restrict: 'E',
+      template: '<span ng-show="query.id && canViewSource">\
+                    <a ng-show="!sourceMode"\
+                      ng-href="{{query.id}}/source#{{selectedTab}}">Show Source\
+                    </a>\
+                    <a ng-show="sourceMode"\
+                      ng-href="/queries/{{query.id}}#{{selectedTab}}">Hide Source\
+                    </a>\
+                </span>'
+    }
+  }
+
+  function queryResultCSVLink() {
+    return {
+      restrict: 'A',
+      link: function (scope, element) {
+        scope.$watch('queryResult && queryResult.getData()', function(data) {
+          if (!data) {
+            return;
+          }
+
+          if (scope.queryResult.getId() == null) {
+            element.attr('href', '');
+          } else {
+            element.attr('href', '/api/queries/' + scope.query.id + '/results/' + scope.queryResult.getId() + '.csv');
+            element.attr('download', scope.query.name.replace(" ", "_") + moment(scope.queryResult.getUpdatedAt()).format("_YYYY_MM_DD") + ".csv");
+          }
+        });
+      }
+    }
+  }
+
+  function queryEditor() {
+    return {
+      restrict: 'E',
+      scope: {
+        'query': '=',
+        'lock': '='
+      },
+      template: '<textarea\
+                  ui-codemirror="editorOptions"\
+                  ng-model="query.query">',
+      link: function($scope) {
+        $scope.editorOptions = {
+            mode: 'text/x-sql',
+            lineWrapping: true,
+            lineNumbers: true,
+            readOnly: false,
+            matchBrackets: true,
+            autoCloseBrackets: true
+        };
+
+        $scope.$watch('lock', function(locked) {
+          $scope.editorOptions.readOnly = locked ? 'nocursor' : false;
+        });
+      }
+    }
+  }
+
+  function queryFormatter($http) {
+    return {
+      restrict: 'E',
+      // don't create new scope to avoid ui-codemirror bug
+      // seehttps://github.com/angular-ui/ui-codemirror/pull/37
+      scope: false,
+      template: '<button type="button" class="btn btn-default btn-xs"\
+                   ng-click="formatQuery()">\
+                    <span class="glyphicon glyphicon-indent-left"></span>\
+                     Format SQL\
+                </button>',
+      link: function($scope) {
+        $scope.formatQuery = function formatQuery() {
+            $scope.queryExecuting = true;
+            $http.post('/api/queries/format', {
+                'query': $scope.query.query
+            }).success(function (response) {
+                $scope.query.query = response;
+            }).finally(function () {
+              $scope.queryExecuting = false;
+            });
+        };
+      }
+    }
+  }
+
+  function queryRefreshSelect() {
+    return {
+      restrict: 'E',
+      template: '<select\
+                  ng-disabled="!isQueryOwner"\
+                  ng-model="query.ttl"\
+                  ng-change="saveQuery()"\
+                  ng-options="c.value as c.name for c in refreshOptions">\
+                  </select>',
+      link: function($scope) {
+        $scope.refreshOptions = [
+            {
+                value: -1,
+                name: 'No Refresh'
+            },
+            {
+                value: 60,
+                name: 'Every minute'
+            },
+        ]
+
+        _.each(_.range(1, 13), function (i) {
+            $scope.refreshOptions.push({
+                value: i * 3600,
+                name: 'Every ' + i + 'h'
+            });
+        })
+
+        $scope.refreshOptions.push({
+            value: 24 * 3600,
+            name: 'Every 24h'
+        });
+        $scope.refreshOptions.push({
+            value: 7 * 24 * 3600,
+            name: 'Once a week'
+        });
+      }
+
+    }
+  }
+
+  angular.module('redash.directives')
+  .directive('queryLink', queryLink)
+  .directive('querySourceLink', querySourceLink)
+  .directive('queryResultLink', queryResultCSVLink)
+  .directive('queryEditor', queryEditor)
+  .directive('queryRefreshSelect', queryRefreshSelect)
+  .directive('queryFormatter', ['$http', queryFormatter]);
+})();

rd_ui/app/scripts/filters.js

@@ -1,50 +1,75 @@
 var durationHumanize = function (duration) {
-    var humanized = "";
-    if (duration == undefined) {
-        humanized = "-";
-    } else if (duration < 60) {
-        humanized = Math.round(duration) + "s";
-    } else if (duration > 3600*24) {
-        var days = Math.round(parseFloat(duration) / 60.0 / 60.0 / 24.0);
-        humanized = days + "days";
-    } else if (duration >= 3600) {
-        var hours = Math.round(parseFloat(duration) / 60.0 / 60.0);
-        humanized = hours + "h";
-    } else {
-        var minutes = Math.round(parseFloat(duration) / 60.0);
-        humanized = minutes + "m";
-    }
-    return humanized;
-}
+  var humanized = "";
+  if (duration == undefined) {
+    humanized = "-";
+  } else if (duration < 60) {
+    humanized = Math.round(duration) + "s";
+  } else if (duration > 3600 * 24) {
+    var days = Math.round(parseFloat(duration) / 60.0 / 60.0 / 24.0);
+    humanized = days + "days";
+  } else if (duration >= 3600) {
+    var hours = Math.round(parseFloat(duration) / 60.0 / 60.0);
+    humanized = hours + "h";
+  } else {
+    var minutes = Math.round(parseFloat(duration) / 60.0);
+    humanized = minutes + "m";
+  }
+  return humanized;
+};
+
+var urlPattern = /(^|[\s\n]|<br\/?>)((?:https?|ftp):\/\/[\-A-Z0-9+\u0026\u2019@#\/%?=()~_|!:,.;]*[\-A-Z0-9+\u0026@#\/%=~()_|])/gi;
 
 angular.module('redash.filters', []).
-    filter('durationHumanize', function () {
-        return durationHumanize;
-    })
+  filter('durationHumanize', function () {
+    return durationHumanize;
+  })
 
-    .filter('refreshRateHumanize', function () {
-        return function (ttl) {
-            if (ttl==-1) {
-                return "Never";
-            } else {
-                return "Every " + durationHumanize(ttl);
-            }
-        }
-    })
+  .filter('refreshRateHumanize', function () {
+    return function (ttl) {
+      if (ttl == -1) {
+        return "Never";
+      } else {
+        return "Every " + durationHumanize(ttl);
+      }
+    }
+  })
 
-    .filter('toHuman', function() {
-        return function(text) {
-            return text.replace(/_/g, ' ').replace(/(?:^|\s)\S/g, function (a) {
-                return a.toUpperCase();
-            });
-        }
-    })
+  .filter('toHuman', function () {
+    return function (text) {
+      return text.replace(/_/g, ' ').replace(/(?:^|\s)\S/g, function (a) {
+        return a.toUpperCase();
+      });
+    }
+  })
 
-    .filter('colWidth', function () {
-        return function (widgetWidth) {
-            if (widgetWidth == 1) {
-                return 6;
-            }
-            return 12;
-        }
-    });
+  .filter('colWidth', function () {
+    return function (widgetWidth) {
+      if (widgetWidth == 1) {
+        return 6;
+      }
+      return 12;
+    }
+  })
+
+  .filter('capitalize', function () {
+    return function (text) {
+      if (text) {
+        return _.str.capitalize(text);
+      } else {
+        return null;
+      }
+
+    }
+  })
+
+  .filter('linkify', function () {
+    return function (text) {
+      return text.replace(urlPattern, "$1<a href='$2' target='_blank'>$2</a>");
+    };
+  })
+
+  .filter('markdown', ['$sce', function ($sce) {
+    return function (text) {
+      return $sce.trustAsHtml(marked(text));
+    }
+  }]);

rd_ui/app/scripts/ng-highchart.js

@@ -1,272 +0,0 @@
-(function () {
-    'use strict';
-
-    var defaultOptions = {
-        title: {
-            "text": null
-        },
-        xAxis: {
-            type: 'datetime'
-        },
-        yAxis: {
-            title: {
-                text: null
-            }
-        },
-        tooltip: {
-            valueDecimals: 2,
-            formatter: function () {
-                if (!this.points) {
-                    this.points = [this.point];
-                };
-
-                if (moment.isMoment(this.x)) {
-                    var s = '<b>' + moment(this.x).format("DD/MM/YY HH:mm") + '</b>',
-                        pointsCount = this.points.length;
-
-                    $.each(this.points, function (i, point) {
-                        s += '<br/><span style="color:' + point.series.color + '">' + point.series.name + '</span>: ' +
-                            Highcharts.numberFormat(point.y);
-
-                        if (pointsCount > 1 && point.percentage) {
-                            s += " (" + Highcharts.numberFormat(point.percentage) + "%)";
-                        }
-                    });
-                } else {
-                    var points = this.points;
-                    var name = points[0].key || points[0].name;
-
-                    var s = "<b>" + name + "</b>";
-
-                    $.each(points, function (i, point) {
-                        if (points.length > 1) {
-                            s += '<br/><span style="color:' + point.series.color + '">' + point.series.name + '</span>: ' + Highcharts.numberFormat(point.y);
-                        } else {
-                            s += ": " + Highcharts.numberFormat(point.y);
-                            if (point.percentage < 100) {
-                                s += ' (' +Highcharts.numberFormat(point.percentage) +  '%)';
-                            }
-                        }
-                    });
-                }
-
-                return s;
-            },
-            shared: true
-        },
-        exporting: {
-            chartOptions: {
-                title: {
-                    text: ''
-                }
-            },
-            buttons: {
-                contextButton: {
-                    menuItems: [
-                        {
-                            text: 'Toggle % Stacking',
-                            onclick: function () {
-                                var newStacking = "normal";
-                                if (this.series[0].options.stacking == "normal") {
-                                    newStacking = "percent";
-                                }
-
-                                _.each(this.series, function (series) {
-                                    series.update({stacking: newStacking}, true);
-                                });
-                            }
-                        }
-                    ]
-                }
-            }
-        },
-        credits: {
-            enabled: false
-        },
-        plotOptions: {
-            area: {
-                marker: {
-                    enabled: false,
-                    symbol: 'circle',
-                    radius: 2,
-                    states: {
-                        hover: {
-                            enabled: true
-                        }
-                    }
-                }
-            },
-            column: {
-                stacking: "normal",
-                pointPadding: 0,
-                borderWidth: 1,
-                groupPadding: 0,
-                shadow: false
-            },
-            line: {
-                marker: {
-                    radius: 1
-                },
-                lineWidth: 2,
-                states: {
-                    hover: {
-                        lineWidth: 2,
-                        marker: {
-                            radius: 3
-                        }
-                    }
-                }
-            },
-            pie: {
-                allowPointSelect: true,
-                cursor: 'pointer',
-                dataLabels: {
-                    enabled: true,
-                    color: '#000000',
-                    connectorColor: '#000000',
-                    format: '<b>{point.name}</b>: {point.percentage:.1f} %'
-                }
-            },
-            scatter: {
-                marker: {
-                    radius: 5,
-                    states: {
-                        hover: {
-                            enabled: true,
-                            lineColor: 'rgb(100,100,100)'
-                        }
-                    }
-                },
-                tooltip: {
-                    headerFormat: '<b>{series.name}</b><br>',
-                    pointFormat: '{point.x}, {point.y}'
-                }
-            }
-        },
-        series: []
-    };
-
-    angular.module('highchart', [])
-        .directive('chart', ['$timeout', function ($timeout) {
-            return {
-                restrict: 'E',
-                template: '<div></div>',
-                scope: {
-                    options: "=options",
-                    series: "=series"
-                },
-                transclude: true,
-                replace: true,
-
-                link: function (scope, element, attrs) {
-                    var chartsDefaults = {
-                        chart: {
-                            renderTo: element[0],
-                            type: attrs.type || null,
-                            height: attrs.height || null,
-                            width: attrs.width || null
-                        }
-                    };
-
-                    var chartOptions = $.extend(true, {}, defaultOptions, chartsDefaults);
-
-                    // $timeout makes sure that this function invoked after the DOM ready. When draw/init
-                    // invoked after the DOM is ready, we see first an empty HighCharts objects and later
-                    // they get filled up. Which gives the feeling that the charts loading faster (otherwise
-                    // we stare at an empty screen until the HighCharts object is ready).
-                    $timeout(function(){
-                        // Update when options change
-                        scope.$watch('options', function (newOptions) {
-                            initChart(newOptions);
-                        }, true);
-
-                        //Update when charts data changes
-                        scope.$watch(function () {
-                            // TODO: this might be an issue in case the series change, but they stay
-                            // with the same length
-                            return (scope.series && scope.series.length) || 0;
-                        }, function (length) {
-                            if (!length || length == 0) {
-                                scope.chart.showLoading();
-                            } else {
-                                drawChart();
-                            };
-                        }, true);
-                    });
-
-                    function initChart(options) {
-                        if (scope.chart) {
-                            scope.chart.destroy();
-                        };
-
-                        $.extend(true, chartOptions, options);
-
-                        scope.chart = new Highcharts.Chart(chartOptions);
-                        drawChart();
-                    }
-
-                    function drawChart() {
-                        while (scope.chart.series.length > 0) {
-                            scope.chart.series[0].remove(false);
-                        };
-
-                        if (_.some(scope.series[0].data, function (p) {
-                            return angular.isString(p.x)
-                        })) {
-                            scope.chart.xAxis[0].update({type: 'category'});
-
-                            // We need to make sure that for each category, each series has a value.
-                            var categories = _.union.apply(this, _.map(scope.series, function (s) {
-                                return _.pluck(s.data, 'x')
-                            }));
-
-                            _.each(scope.series, function (s) {
-                                // TODO: move this logic to Query#getChartData
-                                var yValues = _.groupBy(s.data, 'x');
-
-                                var newData = _.sortBy(_.map(categories, function (category) {
-                                    return {
-                                        name: category,
-                                        y: yValues[category] && yValues[category][0].y
-                                    }
-                                }), 'name');
-
-                                s.data = newData;
-                            });
-                        } else {
-                            scope.chart.xAxis[0].update({type: 'datetime'});
-                        }
-
-                        scope.chart.counters.color = 0;
-
-                        _.each(scope.series, function (s) {
-                            // here we override the series with the visualization config
-                            s = _.extend(s, chartOptions['series']);
-
-                            if (s.type == 'area') {
-                                _.each(s.data, function (p) {
-                                    // This is an insane hack: somewhere deep in HighChart's code,
-                                    // when you stack areas, it tries to convert the string representation
-                                    // of point's x into a number. With the default implementation of toString
-                                    // it fails....
-
-                                    if (moment.isMoment(p.x)) {
-                                        p.x.toString = function () {
-                                            return String(this.toDate().getTime());
-                                        };
-                                    }
-                                });
-                            };
-
-                            scope.chart.addSeries(s, false);
-                        });
-
-                        scope.chart.redraw();
-                        scope.chart.hideLoading();
-                    }
-
-                }
-            };
-
-        }]);
-})();

rd_ui/app/scripts/ng_highchart.js

@@ -0,0 +1,341 @@
+(function () {
+  'use strict';
+
+  Highcharts.setOptions({
+    colors: ["#4572A7", "#AA4643", "#89A54E", "#80699B", "#3D96AE",
+      "#DB843D", "#92A8CD", "#A47D7C", "#B5CA92"]
+  });
+
+  var defaultOptions = {
+    title: {
+      "text": null
+    },
+    xAxis: {
+      type: 'datetime'
+    },
+    yAxis: [
+      {
+        title: {
+          text: null
+        },
+        // showEmpty: true // by default
+      },
+      {
+        title: {
+          text: null
+        },
+        opposite: true,
+        showEmpty: false
+      }
+    ],
+
+
+    tooltip: {
+      valueDecimals: 2,
+      formatter: function () {
+        if (!this.points) {
+          this.points = [this.point];
+        }
+        ;
+
+        if (moment.isMoment(this.x)) {
+          var s = '<b>' + moment(this.x).format("DD/MM/YY HH:mm") + '</b>',
+              pointsCount = this.points.length;
+
+          $.each(this.points, function (i, point) {
+            s += '<br/><span style="color:' + point.series.color + '">' + point.series.name + '</span>: ' +
+                Highcharts.numberFormat(point.y);
+
+            if (pointsCount > 1 && point.percentage) {
+              s += " (" + Highcharts.numberFormat(point.percentage) + "%)";
+            }
+          });
+        } else {
+          var points = this.points;
+          var name = points[0].key || points[0].name;
+
+          var s = "<b>" + name + "</b>";
+
+          $.each(points, function (i, point) {
+            if (points.length > 1) {
+              s += '<br/><span style="color:' + point.series.color + '">' + point.series.name + '</span>: ' + Highcharts.numberFormat(point.y);
+            } else {
+              s += ": " + Highcharts.numberFormat(point.y);
+              if (point.percentage < 100) {
+                s += ' (' + Highcharts.numberFormat(point.percentage) + '%)';
+              }
+            }
+          });
+        }
+
+        return s;
+      },
+      shared: true
+    },
+    exporting: {
+      chartOptions: {
+        title: {
+          text: ''
+        }
+      },
+      buttons: {
+        contextButton: {
+          menuItems: [
+            {
+              text: 'Toggle % Stacking',
+              onclick: function () {
+                var newStacking = "normal";
+                if (this.series[0].options.stacking == "normal") {
+                  newStacking = "percent";
+                }
+
+                _.each(this.series, function (series) {
+                  series.update({stacking: newStacking}, true);
+                });
+              }
+            },
+            {
+              text: 'Select All',
+              onclick: function () {
+                _.each(this.series, function (s) {
+                  s.setVisible(true, false);
+                });
+                this.redraw();
+              }
+            },
+            {
+              text: 'Unselect All',
+              onclick: function () {
+                _.each(this.series, function (s) {
+                  s.setVisible(false, false);
+                });
+                this.redraw();
+              }
+            },
+            {
+              text: 'Show Total',
+              onclick: function () {
+                var data = {};
+                _.each(this.series, function (s) {
+                  s.setVisible(false, false);
+                  _.each(s.data, function (p) {
+                    data[p.x] = data[p.x] || {'x': p.x, 'y': 0};
+                    data[p.x].y = data[p.x].y + p.y;
+                  });
+                });
+
+                this.addSeries({
+                  data: _.values(data),
+                  type: 'line',
+                  name: 'Total'
+                }, false)
+
+                this.redraw();
+              }
+            }
+          ]
+        }
+      }
+    },
+    credits: {
+      enabled: false
+    },
+    plotOptions: {
+      area: {
+        marker: {
+          enabled: false,
+          symbol: 'circle',
+          radius: 2,
+          states: {
+            hover: {
+              enabled: true
+            }
+          }
+        }
+      },
+      column: {
+        stacking: "normal",
+        pointPadding: 0,
+        borderWidth: 1,
+        groupPadding: 0,
+        shadow: false
+      },
+      line: {
+        marker: {
+          radius: 1
+        },
+        lineWidth: 2,
+        states: {
+          hover: {
+            lineWidth: 2,
+            marker: {
+              radius: 3
+            }
+          }
+        }
+      },
+      pie: {
+        allowPointSelect: true,
+        cursor: 'pointer',
+        dataLabels: {
+          enabled: true,
+          color: '#000000',
+          connectorColor: '#000000',
+          format: '<b>{point.name}</b>: {point.y} ({point.percentage:.1f} %)'
+        }
+      },
+      scatter: {
+        marker: {
+          radius: 5,
+          states: {
+            hover: {
+              enabled: true,
+              lineColor: 'rgb(100,100,100)'
+            }
+          }
+        },
+        tooltip: {
+          headerFormat: '<b>{series.name}</b><br>',
+          pointFormat: '{point.x}, {point.y}'
+        }
+      }
+    },
+    series: []
+  };
+
+  angular.module('highchart', [])
+      .directive('chart', ['$timeout', function ($timeout) {
+        return {
+          restrict: 'E',
+          template: '<div></div>',
+          scope: {
+            options: "=options",
+            series: "=series"
+          },
+          transclude: true,
+          replace: true,
+
+          link: function (scope, element, attrs) {
+            var chartsDefaults = {
+              chart: {
+                renderTo: element[0],
+                type: attrs.type || null,
+                height: attrs.height || null,
+                width: attrs.width || null
+              }
+            };
+
+            var chartOptions = $.extend(true, {}, defaultOptions, chartsDefaults);
+
+            // $timeout makes sure that this function invoked after the DOM ready. When draw/init
+            // invoked after the DOM is ready, we see first an empty HighCharts objects and later
+            // they get filled up. Which gives the feeling that the charts loading faster (otherwise
+            // we stare at an empty screen until the HighCharts object is ready).
+            $timeout(function () {
+              // Update when options change
+              scope.$watch('options', function (newOptions) {
+                initChart(newOptions);
+              }, true);
+
+              //Update when charts data changes
+              scope.$watchCollection('series', function (series) {
+                if (!series || series.length == 0) {
+                  scope.chart.showLoading();
+                } else {
+                  drawChart();
+                }
+                ;
+              });
+            });
+
+            function initChart(options) {
+              if (scope.chart) {
+                scope.chart.destroy();
+              }
+              ;
+
+              $.extend(true, chartOptions, options);
+
+              scope.chart = new Highcharts.Chart(chartOptions);
+              drawChart();
+            }
+
+            function drawChart() {
+              while (scope.chart.series.length > 0) {
+                scope.chart.series[0].remove(false);
+              };
+
+              if (!('xAxis' in chartOptions && 'type' in chartOptions['xAxis'])) {
+                if (scope.series.length > 0 && _.some(scope.series[0].data, function (p) {
+                  return (angular.isString(p.x) || angular.isDefined(p.name));
+                })) {
+                  chartOptions['xAxis'] = chartOptions['xAxis'] || {};
+                  chartOptions['xAxis']['type'] = 'category';
+                } else {
+                  chartOptions['xAxis'] = chartOptions['xAxis'] || {};
+                  chartOptions['xAxis']['type'] = 'datetime';
+                }
+              }
+
+              if (chartOptions['xAxis']['type'] == 'category' || chartOptions['series']['type']=='pie') {
+                if (!angular.isDefined(scope.series[0].data[0].name)) {
+                  // We need to make sure that for each category, each series has a value.
+                  var categories = _.union.apply(this, _.map(scope.series, function (s) {
+                    return _.pluck(s.data, 'x')
+                  }));
+
+                  _.each(scope.series, function (s) {
+                    // TODO: move this logic to Query#getChartData
+                    var yValues = _.groupBy(s.data, 'x');
+
+                    var newData = _.map(categories, function (category) {
+                      return {
+                        name: category,
+                        y: (yValues[category] && yValues[category][0].y) || 0
+                      }
+                    });
+
+                    if (categories.length == 1) {
+                      newData = _.sortBy(newData, 'y').reverse();
+                    }
+                    ;
+
+                    s.data = newData;
+                  });
+                }
+              }
+
+              scope.chart.counters.color = 0;
+
+              _.each(scope.series, function (s) {
+                // here we override the series with the visualization config
+                s = _.extend(s, chartOptions['series']);
+
+                if (s.type == 'area') {
+                  _.each(s.data, function (p) {
+                    // This is an insane hack: somewhere deep in HighChart's code,
+                    // when you stack areas, it tries to convert the string representation
+                    // of point's x into a number. With the default implementation of toString
+                    // it fails....
+
+                    if (moment.isMoment(p.x)) {
+                      p.x.toString = function () {
+                        return String(this.toDate().getTime());
+                      };
+                    }
+                  });
+                }
+                ;
+
+                scope.chart.addSeries(s, false);
+              });
+
+              scope.chart.redraw();
+              scope.chart.hideLoading();
+            }
+
+          }
+        };
+
+      }]);
+})();

rd_ui/app/scripts/ng_smart_table.js

@@ -217,7 +217,7 @@
                             element.html('<div editable-cell="" row="dataRow" column="column" type="column.type"></div>');
                             compile(element.contents())(scope);
                         } else {
-                            element.text(scope.formatedValue);
+                            element.html(scope.formatedValue);
                         }
                     }
 

rd_ui/app/scripts/query_fiddle/renderers.js

@@ -1,203 +0,0 @@
-var renderers = angular.module('redash.renderers', []);
-
-renderers.directive('visualizationRenderer', function() {
-    return {
-        restrict: 'E',
-        scope: {
-            visualization: '=',
-            queryResult: '='
-        },
-        template: '<div ng-switch on="visualization.type">' +
-                    '<grid-renderer ng-switch-when="TABLE" options="visualization.options" query-result="queryResult"></grid-renderer>' +
-                    '<chart-renderer ng-switch-when="CHART" options="visualization.options" query-result="queryResult"></chart-renderer>' +
-                    '<cohort-renderer ng-switch-when="COHORT" options="visualization.options" query-result="queryResult"></cohort-renderer>' +
-                   '</div>',
-        replace: false
-    }
-});
-
-renderers.directive('chartRenderer', function () {
-    return {
-        restrict: 'E',
-        scope: {
-            queryResult: '=',
-            options: '=?'
-        },
-        template: "<chart options='chartOptions' series='chartSeries' class='graph'></chart>",
-        replace: false,
-        controller: ['$scope', function ($scope) {
-            $scope.chartSeries = [];
-            $scope.chartOptions = {};
-
-            $scope.$watch('options', function(chartOptions) {
-                if (chartOptions) {
-                    $scope.chartOptions = chartOptions;
-                }
-            });
-            $scope.$watch('queryResult && queryResult.getData()', function (data) {
-                if (!data || $scope.queryResult.getData() == null) {
-                    $scope.chartSeries.splice(0, $scope.chartSeries.length);
-                } else {
-                    $scope.chartSeries.splice(0, $scope.chartSeries.length);
-
-                    _.each($scope.queryResult.getChartData(), function (s) {
-                        $scope.chartSeries.push(_.extend(s, {'stacking': 'normal'}));
-                    });
-                }
-            });
-        }]
-    }
-})
-
-renderers.directive('gridRenderer', function () {
-    return {
-        restrict: 'E',
-        scope: {
-            queryResult: '=',
-            itemsPerPage: '='
-        },
-        templateUrl: "/views/grid_renderer.html",
-        replace: false,
-        controller: ['$scope', function ($scope) {
-            $scope.gridColumns = [];
-            $scope.gridData = [];
-            $scope.gridConfig = {
-                isPaginationEnabled: true,
-                itemsByPage: $scope.itemsPerPage || 15,
-                maxSize: 8
-            };
-
-            $scope.$watch('queryResult && queryResult.getData()', function (data) {
-                if (!data) {
-                    return;
-                }
-
-                if ($scope.queryResult.getData() == null) {
-                    $scope.gridColumns = [];
-                    $scope.gridData = [];
-                    $scope.filters = [];
-                } else {
-
-
-                    $scope.filters = $scope.queryResult.getFilters();
-
-                    var gridData = _.map($scope.queryResult.getData(), function (row) {
-                        var newRow = {};
-                        _.each(row, function (val, key) {
-                            newRow[$scope.queryResult.getColumnCleanName(key)] = val;
-                        })
-                        return newRow;
-                    });
-
-                    $scope.gridColumns = _.map($scope.queryResult.getColumnCleanNames(), function (col, i) {
-                        var columnDefinition = {
-                            'label': $scope.queryResult.getColumnFriendlyNames()[i],
-                            'map': col
-                        };
-
-                        if (gridData.length > 0) {
-                            var exampleData = gridData[0][col];
-                            if (angular.isNumber(exampleData)) {
-                                columnDefinition['formatFunction'] = 'number';
-                                columnDefinition['formatParameter'] = 2;
-                            } else if (moment.isMoment(exampleData)) {
-                                columnDefinition['formatFunction'] = function(value) {
-                                    return value.format("DD/MM/YY HH:mm");
-                                }
-                            }
-                        }
-
-                        return columnDefinition;
-                    });
-
-                    $scope.gridData = _.clone(gridData);
-
-                    $scope.$watch('filters', function (filters) {
-                        $scope.gridData = _.filter(gridData, function (row) {
-                            return _.reduce(filters, function (memo, filter) {
-                                if (filter.current == 'All') {
-                                    return memo && true;
-                                }
-
-                                return (memo && row[$scope.queryResult.getColumnCleanName(filter.name)] == filter.current);
-                            }, true);
-                        });
-                    }, true);
-                }
-            });
-        }]
-    }
-})
-
-renderers.directive('pivotTableRenderer', function () {
-    return {
-        restrict: 'E',
-        scope: {
-            queryResult: '='
-        },
-        template: "",
-        replace: false,
-        link: function($scope, element, attrs) {
-            $scope.$watch('queryResult && queryResult.getData()', function (data) {
-                if (!data) {
-                    return;
-                }
-
-                if ($scope.queryResult.getData() == null) {
-                } else {
-                    $(element).pivotUI($scope.queryResult.getData(), {
-                         renderers: $.pivotUtilities.renderers
-                    }, true);
-                }
-            });
-        }
-    }
-})
-
-renderers.directive('cohortRenderer', function() {
-    return {
-        restrict: 'E',
-        scope: {
-            queryResult: '='
-        },
-        template: "",
-        replace: false,
-        link: function($scope, element, attrs) {
-            $scope.$watch('queryResult && queryResult.getData()', function (data) {
-                if (!data) {
-                    return;
-                }
-
-                if ($scope.queryResult.getData() == null) {
-
-                } else {
-                    var sortedData = _.sortBy($scope.queryResult.getData(), "date");
-                    var grouped = _.groupBy(sortedData, "date");
-                    var data = _.map(grouped, function(values, date) {
-                       var row = [values[0].total];
-                        _.each(values, function(value) { row.push(value.value); });
-                        return row;
-                    });
-
-                    var initialDate = moment(sortedData[0].date).toDate(),
-                        container = angular.element(element)[0];
-
-                    Cornelius.draw({
-                        initialDate: initialDate,
-                        container: container,
-                        cohort: data,
-                        title: null,
-                        timeInterval: 'daily',
-                        labels: {
-                            time: 'Activation Day',
-                            people: 'Users'
-                        },
-                        formatHeaderLabel: function (i) {
-                            return "Day " + (i - 1);
-                        }
-                    });
-                }
-            });
-        }
-    }
-})

rd_ui/app/scripts/services.js

@@ -1,305 +0,0 @@
-(function () {
-    var QueryResult = function($resource, $timeout) {
-        var QueryResultResource = $resource('/api/query_results/:id', {id: '@id'}, {'post': {'method': 'POST'}});
-        var Job = $resource('/api/jobs/:id', {id: '@id'});
-
-        var updateFunction = function (props) {
-            angular.extend(this, props);
-            if ('query_result' in props) {
-                this.status = "done";
-
-                _.each(this.query_result.data.rows, function (row) {
-                    _.each(row, function (v, k) {
-                        if (_.isString(v) && v.match(/^\d{4}-\d{2}-\d{2}/)) {
-                            row[k] = moment(v);
-                        }
-                    });
-                });
-            } else if (this.job.status == 3) {
-                this.status = "processing";
-            } else {
-                this.status = undefined;
-            }
-        }
-
-        function QueryResult(props) {
-            this.job = {};
-            this.query_result = {};
-            this.status = "waiting";
-
-            this.updatedAt = moment();
-
-            if (props) {
-                updateFunction.apply(this, [props]);
-            }
-        }
-
-        var statuses = {
-            1: "waiting",
-            2: "processing",
-            3: "done",
-            4: "failed"
-        }
-
-        QueryResult.prototype.update = updateFunction;
-
-        QueryResult.prototype.getId = function() {
-            var id = null;
-            if ('query_result' in this) {
-                id = this.query_result.id;
-            }
-            return id;
-        }
-
-        QueryResult.prototype.cancelExecution = function() {
-            Job.delete({id: this.job.id});
-        }
-
-        QueryResult.prototype.getStatus = function() {
-            return this.status || statuses[this.job.status];
-        }
-
-        QueryResult.prototype.getError = function() {
-            // TODO: move this logic to the server...
-            if (this.job.error == "None") {
-                return undefined;
-            }
-
-            return this.job.error;
-        }
-
-        QueryResult.prototype.getUpdatedAt = function() {
-            return this.query_result.retrieved_at || this.job.updated_at*1000.0 || this.updatedAt;
-        }
-
-        QueryResult.prototype.getRuntime = function() {
-            return this.query_result.runtime;
-        }
-
-        QueryResult.prototype.getData = function() {
-            if (!this.query_result.data) {
-                return null;
-            }
-
-            var data = this.query_result.data.rows;
-            return data;
-        }
-
-        QueryResult.prototype.getChartData = function () {
-            var series = {};
-
-            _.each(this.getData(), function (row) {
-                var point = {};
-                var seriesName = undefined;
-                var xValue = 0;
-                var yValues = {};
-
-                _.each(row, function (value, definition) {
-                    var type = definition.split("::")[1];
-                    var name = definition.split("::")[0];
-
-                    if (type == 'x') {
-                        xValue = value;
-                        point[type] = value;
-                    }
-
-                    if (type == 'y') {
-                        yValues[name] = value;
-                        point[type] = value;
-                    }
-
-                    if (type == 'series') {
-                        seriesName = String(value);
-                    }
-                });
-
-                var addPointToSeries = function(seriesName, point) {
-                    if (series[seriesName] == undefined) {
-                        series[seriesName] = {
-                            name: seriesName,
-                            type: 'column',
-                            data: []
-                        }
-                    }
-
-                    series[seriesName]['data'].push(point);
-                }
-
-                if (seriesName === undefined) {
-                    _.each(yValues, function(yValue, seriesName) {
-                        addPointToSeries(seriesName, {'x': xValue, 'y': yValue});
-                    });
-                } else {
-                    addPointToSeries(seriesName, point);
-                }
-            });
-
-            _.each(series, function(series) {
-                series.data = _.sortBy(series.data, 'x');
-            });
-
-            return _.values(series);
-        };
-
-        QueryResult.prototype.getColumns = function () {
-            if (this.columns == undefined) {
-                this.columns = _.map(this.query_result.data.columns, function(v) {
-                    return v.name;
-                })
-            }
-
-            return this.columns;
-        }
-
-        QueryResult.prototype.getColumnCleanName = function (column) {
-            var parts = column.split('::');
-            var name = parts[1];
-            if (parts[0] != '') {
-                // TODO: it's probably time to generalize this.
-                // see also getColumnFriendlyName
-                name = parts[0].replace(/%/g, '__pct').replace(/ /g, '_').replace(/\?/g,'');
-            }
-            return name;
-        }
-
-        QueryResult.prototype.getColumnFriendlyName = function (column) {
-            return this.getColumnCleanName(column).replace('__pct', '%').replace(/_/g, ' ').replace(/(?:^|\s)\S/g, function (a) {
-                return a.toUpperCase();
-            });
-        }
-
-        QueryResult.prototype.getColumnCleanNames = function () {
-            return _.map(this.getColumns(), function (col) {
-                return this.getColumnCleanName(col);
-            }, this);
-        }
-
-        QueryResult.prototype.getColumnFriendlyNames = function () {
-            return _.map(this.getColumns(), function (col) {
-                return this.getColumnFriendlyName(col);
-            }, this);
-        }
-
-        QueryResult.prototype.getFilters = function () {
-            var filterNames = [];
-            _.each(this.getColumns(), function (col) {
-                if (col.split('::')[1] == 'filter') {
-                    filterNames.push(col);
-                }
-            });
-
-            var filterValues = [];
-            _.each(this.getData(), function (row) {
-                _.each(filterNames, function (filter, i) {
-                    if (filterValues[i] == undefined) {
-                        filterValues[i] = [];
-                    }
-                    filterValues[i].push(row[filter]);
-                })
-            });
-
-            var filters = _.map(filterNames, function (filter, i) {
-                var f = {
-                    name: filter,
-                    friendlyName: this.getColumnFriendlyName(filter),
-                    values: _.uniq(filterValues[i])
-                };
-
-                f.current = f.values[0];
-                return f;
-            }, this);
-
-            return filters;
-        };
-
-        var refreshStatus = function(queryResult, query, ttl) {
-            Job.get({'id': queryResult.job.id}, function(response) {
-                queryResult.update(response);
-
-                if (queryResult.getStatus() == "processing" && queryResult.job.query_result_id && queryResult.job.query_result_id != "None") {
-                    QueryResultResource.get({'id': queryResult.job.query_result_id}, function(response) {
-                        queryResult.update(response);
-                    });
-                } else if (queryResult.getStatus() != "failed") {
-                    $timeout(function () {
-                        refreshStatus(queryResult, query, ttl);
-                    }, 3000);
-                }
-            })
-        }
-
-        QueryResult.getById = function (id) {
-            var queryResult = new QueryResult();
-
-            QueryResultResource.get({'id': id}, function (response) {
-                queryResult.update(response);
-            });
-
-            return queryResult;
-        }
-
-        QueryResult.get = function (query, ttl) {
-            var queryResult = new QueryResult();
-
-            QueryResultResource.post({'query': query, 'ttl': ttl}, function (response) {
-                queryResult.update(response);
-
-                if ('job' in response) {
-                    refreshStatus(queryResult, query, ttl);
-                }
-            });
-
-            return queryResult;
-        }
-
-        return QueryResult;
-    };
-
-    var Query = function ($resource, QueryResult) {
-        var Query = $resource('/api/queries/:id', {id: '@id'});
-
-        Query.prototype.getQueryResult = function(ttl) {
-            if (ttl == undefined) {
-                ttl = this.ttl;
-            }
-
-
-            var queryResult = null;
-            if (this.latest_query_data && ttl != 0) {
-                queryResult = new QueryResult({'query_result': this.latest_query_data});
-            } else if (this.latest_query_data_id && ttl != 0) {
-                queryResult = QueryResult.getById(this.latest_query_data_id);
-            } else {
-                queryResult = QueryResult.get(this.query, ttl);
-            }
-
-            return queryResult;
-        };
-
-        Query.prototype.getHash = function() {
-            return [this.name, this.description, this.query].join('!#');
-        };
-
-        return Query;
-    };
-
-    var Visualization = function($resource) {
-        var Visualization = $resource('/api/visualizations/:id', {id: '@id'});
-
-        Visualization.prototype = {
-            TYPES: {
-                'CHART': 'CHART',
-                'COHORT': 'COHORT',
-                'TABLE': 'TABLE'
-            }
-        };
-
-        return Visualization;
-    };
-
-    angular.module('redash.services', [])
-        .factory('QueryResult', ['$resource', '$timeout', QueryResult])
-        .factory('Query', ['$resource', 'QueryResult', Query])
-        .factory('Visualization', ['$resource', Visualization])
-
-})();

rd_ui/app/scripts/services/dashboards.js

@@ -2,7 +2,7 @@
     var Dashboard = function($resource) {
         var resource = $resource('/api/dashboards/:slug', {slug: '@slug'});
         resource.prototype.canEdit = function() {
-            return currentUser.is_admin || currentUser.canEdit(this);
+            return currentUser.hasPermission('admin') || currentUser.canEdit(this);
         }
         return resource;
     }

rd_ui/app/scripts/services/notifications.js

@@ -1,5 +1,5 @@
 (function () {
-    var notifications = function () {
+    var notifications = function (Events) {
         var notificationService = {};
         var lastNotification = null;
 
@@ -40,6 +40,7 @@
             notification.onclick = function () {
                 window.focus();
                 this.cancel();
+                Events.record(currentUser, 'click', 'notification');
             };
 
             notification.show()
@@ -49,5 +50,5 @@
     }
 
     angular.module('redash.services')
-        .factory('notifications', notifications);
+        .factory('notifications', ['Events', notifications]);
 })();

rd_ui/app/scripts/services/resources.js

@@ -0,0 +1,454 @@
+(function () {
+  var QueryResult = function ($resource, $timeout, $q) {
+    var QueryResultResource = $resource('/api/query_results/:id', {id: '@id'}, {'post': {'method': 'POST'}});
+    var Job = $resource('/api/jobs/:id', {id: '@id'});
+
+    var updateFunction = function (props) {
+      angular.extend(this, props);
+      if ('query_result' in props) {
+        this.status = "done";
+        this.filters = undefined;
+        this.filterFreeze = undefined;
+
+        var columnTypes = {};
+
+        _.each(this.query_result.data.rows, function (row) {
+          _.each(row, function (v, k) {
+            if (angular.isNumber(v)) {
+              columnTypes[k] = 'float';
+            } else if (_.isString(v) && v.match(/^\d{4}-\d{2}-\d{2}T/)) {
+              row[k] = moment(v);
+              columnTypes[k] = 'datetime';
+            } else if (_.isString(v) && v.match(/^\d{4}-\d{2}-\d{2}/)) {
+              row[k] = moment(v);
+              columnTypes[k] = 'date';
+            }
+          }, this);
+        }, this);
+
+        _.each(this.query_result.data.columns, function(column) {
+          if (columnTypes[column.name]) {
+            column.type = columnTypes[column.name];
+          }
+        });
+
+        this.deferred.resolve(this);
+      } else if (this.job.status == 3) {
+        this.status = "processing";
+      } else {
+        this.status = undefined;
+      }
+    }
+
+    function QueryResult(props) {
+      this.deferred = $q.defer();
+      this.job = {};
+      this.query_result = {};
+      this.status = "waiting";
+      this.filters = undefined;
+      this.filterFreeze = undefined;
+
+      this.updatedAt = moment();
+
+      if (props) {
+        updateFunction.apply(this, [props]);
+      }
+    }
+
+    var statuses = {
+      1: "waiting",
+      2: "processing",
+      3: "done",
+      4: "failed"
+    }
+
+    QueryResult.prototype.update = updateFunction;
+
+    QueryResult.prototype.getId = function () {
+      var id = null;
+      if ('query_result' in this) {
+        id = this.query_result.id;
+      }
+      return id;
+    }
+
+    QueryResult.prototype.cancelExecution = function () {
+      Job.delete({id: this.job.id});
+    }
+
+    QueryResult.prototype.getStatus = function () {
+      return this.status || statuses[this.job.status];
+    }
+
+    QueryResult.prototype.getError = function () {
+      // TODO: move this logic to the server...
+      if (this.job.error == "None") {
+        return undefined;
+      }
+
+      return this.job.error;
+    }
+
+    QueryResult.prototype.getUpdatedAt = function () {
+      return this.query_result.retrieved_at || this.job.updated_at * 1000.0 || this.updatedAt;
+    }
+
+    QueryResult.prototype.getRuntime = function () {
+      return this.query_result.runtime;
+    }
+
+    QueryResult.prototype.getRawData = function () {
+      if (!this.query_result.data) {
+        return null;
+      }
+
+      var data = this.query_result.data.rows;
+
+      return data;
+    }
+
+    QueryResult.prototype.getData = function () {
+      if (!this.query_result.data) {
+        return null;
+      }
+
+      var filterValues = function (filters) {
+        if (!filters) {
+          return null;
+        }
+
+        return _.reduce(filters, function (str, filter) {
+          return str + filter.current;
+        }, "")
+      }
+
+      var filters = this.getFilters();
+      var filterFreeze = filterValues(filters);
+
+      if (this.filterFreeze != filterFreeze) {
+        this.filterFreeze = filterFreeze;
+
+        if (filters) {
+          this.filteredData = _.filter(this.query_result.data.rows, function (row) {
+            return _.reduce(filters, function (memo, filter) {
+              if (!_.isArray(filter.current)) {
+                filter.current = [filter.current];
+              };
+
+              return (memo && _.some(filter.current, function(v) {
+                // We compare with either the value or the String representation of the value,
+                // because Select2 casts true/false to "true"/"false".
+                return v == row[filter.name] || String(row[filter.name]) == v
+              }));
+            }, true);
+          });
+        } else {
+          this.filteredData = this.query_result.data.rows;
+        }
+      }
+
+      return this.filteredData;
+    }
+
+    QueryResult.prototype.getChartData = function (mapping) {
+      var series = {};
+
+      _.each(this.getData(), function (row) {
+        var point = {};
+        var seriesName = undefined;
+        var xValue = 0;
+        var yValues = {};
+
+        _.each(row, function (value, definition) {
+          var name = definition.split("::")[0];
+          var type = definition.split("::")[1];
+          if (mapping) {
+            type = mapping[definition];
+          }
+
+          if (type == 'unused') {
+            return;
+          }
+
+          if (type == 'x') {
+            xValue = value;
+            point[type] = value;
+          }
+          if (type == 'y') {
+            if (value == null) {
+              value = 0;
+            }
+            yValues[name] = value;
+            point[type] = value;
+          }
+
+          if (type == 'series') {
+            seriesName = String(value);
+          }
+
+          if (type == 'multi-filter') {
+            seriesName = String(value);
+          }
+        });
+
+        var addPointToSeries = function (seriesName, point) {
+          if (series[seriesName] == undefined) {
+            series[seriesName] = {
+              name: seriesName,
+              type: 'column',
+              data: []
+            }
+          }
+
+          series[seriesName]['data'].push(point);
+        }
+
+        if (seriesName === undefined) {
+          _.each(yValues, function (yValue, seriesName) {
+            addPointToSeries(seriesName, {'x': xValue, 'y': yValue});
+          });
+        } else {
+          addPointToSeries(seriesName, point);
+        }
+      });
+
+      _.each(series, function (series) {
+        series.data = _.sortBy(series.data, 'x');
+      });
+
+      return _.values(series);
+    };
+
+    QueryResult.prototype.getColumns = function () {
+      if (this.columns == undefined && this.query_result.data) {
+        this.columns = this.query_result.data.columns;
+      }
+
+      return this.columns;
+    }
+
+    QueryResult.prototype.getColumnNames = function () {
+      if (this.columnNames == undefined && this.query_result.data) {
+        this.columnNames = _.map(this.query_result.data.columns, function (v) {
+          return v.name;
+        });
+      }
+
+      return this.columnNames;
+    }
+
+    QueryResult.prototype.getColumnNameWithoutType = function (column) {
+      var parts = column.split('::');
+      if (parts[0] == "" && parts.length == 2) {
+        return parts[1];
+      }
+      return parts[0];
+    };
+
+    var charConversionMap = {
+      '__pct': /%/g,
+      '_': / /g,
+      '__qm': /\?/g,
+      '__brkt': /[\(\)\[\]]/g,
+      '__dash': /-/g,
+      '__amp': /&/g,
+      '__sl': /\//g,
+      '__fsl': /\\/g,
+    };
+
+    QueryResult.prototype.getColumnCleanName = function (column) {
+      var name = this.getColumnNameWithoutType(column);
+
+      if (name != '') {
+        _.each(charConversionMap, function(regex, replacement) {
+          name = name.replace(regex, replacement);
+        });
+      }
+
+      return name;
+    }
+
+    QueryResult.prototype.getColumnFriendlyName = function (column) {
+      return this.getColumnNameWithoutType(column).replace(/(?:^|\s)\S/g, function (a) {
+        return a.toUpperCase();
+      });
+    }
+
+    QueryResult.prototype.getColumnCleanNames = function () {
+      return _.map(this.getColumnNames(), function (col) {
+        return this.getColumnCleanName(col);
+      }, this);
+    }
+
+    QueryResult.prototype.getColumnFriendlyNames = function () {
+      return _.map(this.getColumnNames(), function (col) {
+        return this.getColumnFriendlyName(col);
+      }, this);
+    }
+
+    QueryResult.prototype.getFilters = function () {
+      if (!this.filters) {
+        this.prepareFilters();
+      }
+
+      return this.filters;
+    };
+
+    QueryResult.prototype.prepareFilters = function () {
+      var filters = [];
+      var filterTypes = ['filter', 'multi-filter'];
+      _.each(this.getColumnNames(), function (col) {
+        var type = col.split('::')[1]
+        if (_.contains(filterTypes, type)) {
+          // filter found
+          var filter = {
+            name: col,
+            friendlyName: this.getColumnFriendlyName(col),
+            values: [],
+            multiple: (type=='multi-filter')
+          }
+          filters.push(filter);
+        }
+      }, this);
+
+      _.each(this.getRawData(), function (row) {
+        _.each(filters, function (filter) {
+          filter.values.push(row[filter.name]);
+          if (filter.values.length == 1) {
+            filter.current = row[filter.name];
+          }
+        })
+      });
+
+      _.each(filters, function(filter) {
+        filter.values = _.uniq(filter.values);
+      });
+
+      this.filters = filters;
+    }
+
+    var refreshStatus = function (queryResult, query, ttl) {
+      Job.get({'id': queryResult.job.id}, function (response) {
+        queryResult.update(response);
+
+        if (queryResult.getStatus() == "processing" && queryResult.job.query_result_id && queryResult.job.query_result_id != "None") {
+          QueryResultResource.get({'id': queryResult.job.query_result_id}, function (response) {
+            queryResult.update(response);
+          });
+        } else if (queryResult.getStatus() != "failed") {
+          $timeout(function () {
+            refreshStatus(queryResult, query, ttl);
+          }, 3000);
+        }
+      })
+    }
+
+    QueryResult.getById = function (id) {
+      var queryResult = new QueryResult();
+
+      QueryResultResource.get({'id': id}, function (response) {
+        queryResult.update(response);
+      });
+
+      return queryResult;
+    };
+
+    QueryResult.prototype.toPromise = function() {
+      return this.deferred.promise;
+    }
+
+    QueryResult.get = function (data_source_id, query, ttl) {
+      var queryResult = new QueryResult();
+
+      QueryResultResource.post({'data_source_id': data_source_id, 'query': query, 'ttl': ttl}, function (response) {
+        queryResult.update(response);
+
+        if ('job' in response) {
+          refreshStatus(queryResult, query, ttl);
+        }
+      });
+
+      return queryResult;
+    }
+
+    return QueryResult;
+  };
+
+  var Query = function ($resource, QueryResult, DataSource) {
+    var Query = $resource('/api/queries/:id', {id: '@id'});
+
+    Query.newQuery = function () {
+      return new Query({
+        query: "",
+        name: "New Query",
+        ttl: -1,
+        user: currentUser
+      });
+    };
+
+    Query.prototype.getSourceLink = function () {
+      return '/queries/' + this.id + '/source';
+    };
+
+    Query.prototype.getQueryResult = function (ttl) {
+      if (ttl == undefined) {
+        ttl = this.ttl;
+      }
+
+      if (this.latest_query_data && ttl != 0) {
+        if (!this.queryResult) {
+          this.queryResult = new QueryResult({'query_result': this.latest_query_data});
+        }
+      } else if (this.latest_query_data_id && ttl != 0) {
+        if (!this.queryResult) {
+          this.queryResult = QueryResult.getById(this.latest_query_data_id);
+        }
+      } else if (this.data_source_id) {
+        this.queryResult = QueryResult.get(this.data_source_id, this.query, ttl);
+      }
+
+      return this.queryResult;
+    };
+
+    Query.prototype.getQueryResultPromise = function() {
+      return this.getQueryResult().toPromise();
+    }
+
+    return Query;
+  };
+
+
+
+  var DataSource = function ($resource) {
+    var DataSourceResource = $resource('/api/data_sources/:id', {id: '@id'}, {'get': {'method': 'GET', 'cache': true, 'isArray': true}});
+
+    return DataSourceResource;
+  }
+
+  var Widget = function ($resource, Query) {
+    var WidgetResource = $resource('/api/widgets/:id', {id: '@id'});
+
+    WidgetResource.prototype.getQuery = function () {
+      if (!this.query && this.visualization) {
+         this.query = new Query(this.visualization.query);
+      }
+
+      return this.query;
+    };
+
+    WidgetResource.prototype.getName = function () {
+      if (this.visualization) {
+        return this.visualization.query.name + ' (' + this.visualization.name + ')';
+      }
+      return _.str.truncate(this.text, 20);
+    };
+
+    return WidgetResource;
+  }
+
+  angular.module('redash.services')
+      .factory('QueryResult', ['$resource', '$timeout', '$q', QueryResult])
+      .factory('Query', ['$resource', 'QueryResult', 'DataSource', Query])
+      .factory('DataSource', ['$resource', DataSource])
+      .factory('Widget', ['$resource', 'Query', Widget]);
+})();

rd_ui/app/scripts/services/services.js

@@ -0,0 +1,52 @@
+(function () {
+  'use strict'
+
+  function KeyboardShortcuts() {
+    this.bind = function bind(keymap) {
+      _.forEach(keymap, function (fn, key) {
+        Mousetrap.bindGlobal(key, function (e) {
+          e.preventDefault();
+          fn();
+        });
+      });
+
+    }
+
+    this.unbind = function unbind(keymap) {
+      _.forEach(keymap, function (fn, key) {
+        Mousetrap.unbind(key);
+      });
+    }
+  }
+
+  function Events($http) {
+    this.events = [];
+
+    this.post = _.debounce(function() {
+      var events = this.events;
+      this.events = [];
+
+      $http.post('/api/events', events);
+
+    }, 1000);
+
+    this.record = function (user, action, object_type, object_id, additional_properties) {
+
+      var event = {
+        "user_id": user.id,
+        "action": action,
+        "object_type": object_type,
+        "object_id": object_id,
+        "timestamp": Date.now()/1000.0
+      };
+      _.extend(event, additional_properties);
+      this.events.push(event);
+
+      this.post();
+    };
+  }
+
+  angular.module('redash.services', [])
+      .service('KeyboardShortcuts', [KeyboardShortcuts])
+      .service('Events', ['$http', Events])
+})();

rd_ui/app/scripts/visualizations/base.js

@@ -0,0 +1,213 @@
+(function () {
+  var VisualizationProvider = function () {
+    this.visualizations = {};
+    this.visualizationTypes = {};
+    var defaultConfig = {
+      defaultOptions: {},
+      skipTypes: false,
+      editorTemplate: null
+    }
+
+    this.registerVisualization = function (config) {
+      var visualization = _.extend({}, defaultConfig, config);
+
+      // TODO: this is prone to errors; better refactor.
+      if (_.isEmpty(this.visualizations)) {
+        this.defaultVisualization = visualization;
+      }
+
+      this.visualizations[config.type] = visualization;
+
+      if (!config.skipTypes) {
+        this.visualizationTypes[config.name] = config.type;
+      }
+      ;
+    };
+
+    this.getSwitchTemplate = function (property) {
+      var pattern = /(<[a-zA-Z0-9-]*?)( |>)/
+
+      var mergedTemplates = _.reduce(this.visualizations, function (templates, visualization) {
+        if (visualization[property]) {
+          var ngSwitch = '$1 ng-switch-when="' + visualization.type + '" $2';
+          var template = visualization[property].replace(pattern, ngSwitch);
+
+          return templates + "\n" + template;
+        }
+
+        return templates;
+      }, "");
+
+      mergedTemplates = '<div ng-switch on="visualization.type">' + mergedTemplates + "</div>";
+
+      return mergedTemplates;
+    }
+
+    this.$get = ['$resource', function ($resource) {
+      var Visualization = $resource('/api/visualizations/:id', {id: '@id'});
+      Visualization.visualizations = this.visualizations;
+      Visualization.visualizationTypes = this.visualizationTypes;
+      Visualization.renderVisualizationsTemplate = this.getSwitchTemplate('renderTemplate');
+      Visualization.editorTemplate = this.getSwitchTemplate('editorTemplate');
+      Visualization.defaultVisualization = this.defaultVisualization;
+
+      return Visualization;
+    }];
+  };
+
+  var VisualizationRenderer = function ($location, Visualization) {
+    return {
+      restrict: 'E',
+      scope: {
+        visualization: '=',
+        queryResult: '='
+      },
+      // TODO: using switch here (and in the options editor) might introduce errors and bad
+      // performance wise. It's better to eventually show the correct template based on the
+      // visualization type and not make the browser render all of them.
+      template: '<filters></filters>\n' + Visualization.renderVisualizationsTemplate,
+      replace: false,
+      link: function (scope) {
+        scope.select2Options = {
+          width: '50%'
+        };
+
+        function readURL() {
+          var searchFilters = angular.fromJson($location.search().filters);
+          if (searchFilters) {
+            _.forEach(scope.filters, function(filter) {
+              var value = searchFilters[filter.friendlyName];
+              if (value) {
+                filter.current = value;
+              }
+            });
+          }
+        }
+
+        function updateURL(filters) {
+          var current = {};
+          _.each(filters, function(filter) {
+            if (filter.current) {
+              current[filter.friendlyName] = filter.current;
+            }
+          });
+
+          var newSearch = angular.extend($location.search(), {
+            filters: angular.toJson(current)
+          });
+          $location.search(newSearch);
+        }
+
+        scope.$watch('queryResult && queryResult.getFilters()', function (filters) {
+          if (filters) {
+            scope.filters = filters;
+
+            if (filters.length && false) {
+              readURL();
+
+              // start watching for changes and update URL
+              scope.$watch('filters', updateURL, true);
+            }
+          }
+        });
+      }
+    }
+  };
+
+  var VisualizationOptionsEditor = function (Visualization) {
+    return {
+      restrict: 'E',
+      template: Visualization.editorTemplate,
+      replace: false
+    }
+  };
+
+  var Filters = function () {
+    return {
+      restrict: 'E',
+      templateUrl: '/views/visualizations/filters.html'
+    }
+  }
+
+  var EditVisualizationForm = function (Events, Visualization, growl) {
+    return {
+      restrict: 'E',
+      templateUrl: '/views/visualizations/edit_visualization.html',
+      replace: true,
+      scope: {
+        query: '=',
+        queryResult: '=',
+        visualization: '=?',
+        openEditor: '=?',
+        onNewSuccess: '=?'
+      },
+      link: function (scope, element, attrs) {
+        scope.editRawOptions = currentUser.hasPermission('edit_raw_chart');
+        scope.visTypes = Visualization.visualizationTypes;
+
+        scope.newVisualization = function () {
+          return {
+            'type': Visualization.defaultVisualization.type,
+            'name': Visualization.defaultVisualization.name,
+            'description': '',
+            'options': Visualization.defaultVisualization.defaultOptions
+          };
+        }
+
+        if (!scope.visualization) {
+          var unwatch = scope.$watch('query.id', function (queryId) {
+            if (queryId) {
+              unwatch();
+
+              scope.visualization = scope.newVisualization();
+            }
+          });
+        }
+
+        scope.$watch('visualization.type', function (type, oldType) {
+          // if not edited by user, set name to match type
+          if (type && oldType != type && scope.visualization && !scope.visForm.name.$dirty) {
+            // poor man's titlecase
+            scope.visualization.name = scope.visualization.type[0] + scope.visualization.type.slice(1).toLowerCase();
+          }
+        });
+
+        scope.submit = function () {
+          if (scope.visualization.id) {
+            Events.record(currentUser, "update", "visualization", scope.visualization.id, {'type': scope.visualization.type});
+          } else {
+            Events.record(currentUser, "create", "visualization", null, {'type': scope.visualization.type});
+          }
+
+          scope.visualization.query_id = scope.query.id;
+
+          Visualization.save(scope.visualization, function success(result) {
+            growl.addSuccessMessage("Visualization saved");
+
+            scope.visualization = scope.newVisualization(scope.query);
+
+            var visIds = _.pluck(scope.query.visualizations, 'id');
+            var index = visIds.indexOf(result.id);
+            if (index > -1) {
+              scope.query.visualizations[index] = result;
+            } else {
+              // new visualization
+              scope.query.visualizations.push(result);
+              scope.onNewSuccess && scope.onNewSuccess(result);
+            }
+          }, function error() {
+            growl.addErrorMessage("Visualization could not be saved");
+          });
+        };
+      }
+    }
+  };
+
+
+  angular.module('redash.visualization', [])
+      .provider('Visualization', VisualizationProvider)
+      .directive('visualizationRenderer', ['$location', 'Visualization', VisualizationRenderer])
+      .directive('visualizationOptionsEditor', ['Visualization', VisualizationOptionsEditor])
+      .directive('filters', Filters)
+      .directive('editVisulatizationForm', ['Events', 'Visualization', 'growl', EditVisualizationForm])
+})();

rd_ui/app/scripts/visualizations/chart.js

@@ -0,0 +1,244 @@
+(function () {
+  var chartVisualization = angular.module('redash.visualization');
+
+  chartVisualization.config(['VisualizationProvider', function (VisualizationProvider) {
+    var renderTemplate = '<chart-renderer options="visualization.options" query-result="queryResult"></chart-renderer>';
+    var editTemplate = '<chart-editor></chart-editor>';
+    var defaultOptions = {
+      'series': {
+//        'type': 'column',
+        'stacking': null
+      }
+    };
+
+    VisualizationProvider.registerVisualization({
+      type: 'CHART',
+      name: 'Chart',
+      renderTemplate: renderTemplate,
+      editorTemplate: editTemplate,
+      defaultOptions: defaultOptions
+    });
+  }]);
+
+  chartVisualization.directive('chartRenderer', function () {
+    return {
+      restrict: 'E',
+      scope: {
+        queryResult: '=',
+        options: '=?'
+      },
+      template: "<chart options='chartOptions' series='chartSeries' class='graph'></chart>",
+      replace: false,
+      controller: ['$scope', function ($scope) {
+        $scope.chartSeries = [];
+        $scope.chartOptions = {};
+
+        var reloadData = function(data) {
+          if (!data || ($scope.queryResult && $scope.queryResult.getData()) == null) {
+            $scope.chartSeries.splice(0, $scope.chartSeries.length);
+          } else {
+            $scope.chartSeries.splice(0, $scope.chartSeries.length);
+
+            _.each($scope.queryResult.getChartData($scope.options.columnMapping), function (s) {
+              var additional = {'stacking': 'normal'};
+              if ($scope.options.seriesOptions && $scope.options.seriesOptions[s.name]) {
+                additional = $scope.options.seriesOptions[s.name];
+                if (!additional.name || additional.name == "") {
+                  additional.name = s.name;
+                }
+              }
+              $scope.chartSeries.push(_.extend(s, additional));
+            });
+          };
+        };
+
+        $scope.$watch('options', function (chartOptions) {
+          if (chartOptions) {
+            $scope.chartOptions = chartOptions;
+          }
+        });
+
+        $scope.$watch('options.seriesOptions', function () {
+          reloadData(true);
+        }, true);
+
+
+        $scope.$watchCollection('options.columnMapping', function (chartOptions) {
+          reloadData(true);
+        });
+
+        $scope.$watch('queryResult && queryResult.getData()', function (data) {
+          reloadData(data);
+        });
+      }]
+    };
+  });
+
+  chartVisualization.directive('chartEditor', function () {
+    return {
+      restrict: 'E',
+      templateUrl: '/views/visualizations/chart_editor.html',
+      link: function (scope, element, attrs) {
+        scope.seriesTypes = {
+          'Line': 'line',
+          'Column': 'column',
+          'Area': 'area',
+          'Scatter': 'scatter',
+          'Pie': 'pie'
+        };
+
+        scope.globalSeriesType = 'column';
+
+        scope.stackingOptions = {
+          "None": "none",
+          "Normal": "normal",
+          "Percent": "percent"
+        };
+
+        scope.xAxisOptions = {
+          "Date/Time": "datetime",
+          "Linear": "linear",
+          "Category": "category"
+        };
+
+        scope.xAxisType = "datetime";
+        scope.stacking = "none";
+
+
+        scope.columnTypes = {
+          "X": "x",
+//          "X (Date time)": "x",
+//          "X (Linear)": "x-linear",
+//          "X (Category)": "x-category",
+          "Y": "y",
+          "Series": "series",
+          "Unused": "unused"
+        };
+
+        scope.series = [];
+
+        scope.columnTypeSelection = {};
+
+        var chartOptionsUnwatch = null,
+            columnsWatch = null;
+
+        scope.$watch('globalSeriesType', function(type, old) {
+          if (type && old && type !== old && scope.visualization.options.seriesOptions) {
+            _.each(scope.visualization.options.seriesOptions, function(sOptions) {
+              sOptions.type = type;
+            });
+          }
+        });
+        scope.$watch('visualization.type', function (visualizationType) {
+          if (visualizationType == 'CHART') {
+            if (scope.visualization.options.series.stacking === null) {
+              scope.stacking = "none";
+            } else if (scope.visualization.options.series.stacking === undefined) {
+              scope.stacking = "normal";
+            } else {
+              scope.stacking = scope.visualization.options.series.stacking;
+            }
+
+            var refreshSeries = function() {
+              scope.series = _.map(scope.queryResult.getChartData(scope.visualization.options.columnMapping), function (s) { return s.name; });
+
+              // TODO: remove uneeded ones?
+              if (scope.visualization.options.seriesOptions == undefined) {
+                scope.visualization.options.seriesOptions = {
+                  type: scope.globalSeriesType
+                };
+              };
+
+              _.each(scope.series, function(s, i) {
+                if (scope.visualization.options.seriesOptions[s] == undefined) {
+                  scope.visualization.options.seriesOptions[s] = {'type': 'column', 'yAxis': 0};
+                }
+                scope.visualization.options.seriesOptions[s].zIndex = i;
+
+              });
+              scope.zIndexes = _.range(scope.series.length);
+              scope.yAxes = [[0, 'left'], [1, 'right']];
+            };
+
+            var initColumnMapping = function() {
+              scope.columns = scope.queryResult.getColumns();
+
+              if (scope.visualization.options.columnMapping == undefined) {
+                scope.visualization.options.columnMapping = {};
+              }
+
+              scope.columnTypeSelection = scope.visualization.options.columnMapping;
+
+              _.each(scope.columns, function(column) {
+                var definition = column.name.split("::"),
+                    definedColumns = _.keys(scope.visualization.options.columnMapping);
+
+                if (_.indexOf(definedColumns, column.name) != -1) {
+                  // Skip already defined columns.
+                  return;
+                };
+
+                if (definition.length == 1) {
+                  scope.columnTypeSelection[column.name] = scope.visualization.options.columnMapping[column.name] = 'unused';
+                } else if (definition == 'multi-filter') {
+                  scope.columnTypeSelection[column.name] = scope.visualization.options.columnMapping[column.name] = 'series';
+                } else if (_.indexOf(_.values(scope.columnTypes), definition[1]) != -1) {
+                  scope.columnTypeSelection[column.name] = scope.visualization.options.columnMapping[column.name] = definition[1];
+                } else {
+                  scope.columnTypeSelection[column.name] = scope.visualization.options.columnMapping[column.name] = 'unused';
+                }
+              });
+            };
+
+            columnsWatch = scope.$watch('queryResult.getId()', function(id) {
+              if (!id) {
+                return;
+              }
+
+              initColumnMapping();
+              refreshSeries();
+            });
+
+            scope.$watchCollection('columnTypeSelection', function(selections) {
+              _.each(scope.columnTypeSelection, function(type, name) {
+                scope.visualization.options.columnMapping[name] = type;
+              });
+
+              refreshSeries();
+            });
+
+            chartOptionsUnwatch = scope.$watch("stacking", function (stacking) {
+              if (stacking == "none") {
+                scope.visualization.options.series.stacking = null;
+              } else {
+                scope.visualization.options.series.stacking = stacking;
+              }
+            });
+
+            scope.xAxisType = (scope.visualization.options.xAxis && scope.visualization.options.xAxis.type) || scope.xAxisType;
+
+            xAxisUnwatch = scope.$watch("xAxisType", function (xAxisType) {
+              scope.visualization.options.xAxis = scope.visualization.options.xAxis || {};
+              scope.visualization.options.xAxis.type = xAxisType;
+            });
+          } else {
+            if (chartOptionsUnwatch) {
+              chartOptionsUnwatch();
+              chartOptionsUnwatch = null;
+            }
+
+            if (columnsWatch) {
+              columnWatch();
+              columnWatch = null;
+            }
+
+            if (xAxisUnwatch) {
+              xAxisUnwatch();
+              xAxisUnwatch = null;
+            }
+          }
+        });
+      }
+    }
+  });
+}());

rd_ui/app/scripts/visualizations/cohort.js

@@ -0,0 +1,64 @@
+(function () {
+    var cohortVisualization = angular.module('redash.visualization');
+
+    cohortVisualization.config(['VisualizationProvider', function(VisualizationProvider) {
+        VisualizationProvider.registerVisualization({
+            type: 'COHORT',
+            name: 'Cohort',
+            renderTemplate: '<cohort-renderer options="visualization.options" query-result="queryResult"></cohort-renderer>'
+        });
+    }]);
+
+    cohortVisualization.directive('cohortRenderer', function() {
+        return {
+            restrict: 'E',
+            scope: {
+                queryResult: '='
+            },
+            template: "",
+            replace: false,
+            link: function($scope, element, attrs) {
+                $scope.$watch('queryResult && queryResult.getData()', function (data) {
+                    if (!data) {
+                        return;
+                    }
+
+                    if ($scope.queryResult.getData() == null) {
+
+                    } else {
+                        var sortedData = _.sortBy($scope.queryResult.getData(), "date");
+                        var grouped = _.groupBy(sortedData, "date");
+                        var maxColumns = _.reduce(grouped, function(memo, data){ 
+                            return (data.length > memo)? data.length : memo;
+                        }, 0);
+                        var data = _.map(grouped, function(values, date) {
+                           var row = [values[0].total];
+                            _.each(values, function(value) { row.push(value.value); });
+                            _.each(_.range(values.length, maxColumns), function() { row.push(null); });
+                            return row;
+                        });
+
+                        var initialDate = moment(sortedData[0].date).toDate(),
+                            container = angular.element(element)[0];
+
+                        Cornelius.draw({
+                            initialDate: initialDate,
+                            container: container,
+                            cohort: data,
+                            title: null,
+                            timeInterval: 'daily',
+                            labels: {
+                                time: 'Activation Day',
+                                people: 'Users'
+                            },
+                            formatHeaderLabel: function (i) {
+                                return "Day " + (i - 1);
+                            }
+                        });
+                    }
+                });
+            }
+        }
+    });
+
+}());

rd_ui/app/scripts/visualizations/pivot.js

@@ -0,0 +1,29 @@
+var renderers = angular.module('redash.renderers', []);
+
+renderers.directive('pivotTableRenderer', function () {
+    return {
+        restrict: 'E',
+        scope: {
+            queryResult: '='
+        },
+        template: "",
+        replace: false,
+        link: function($scope, element, attrs) {
+            $scope.$watch('queryResult && queryResult.getData()', function (data) {
+                if (!data) {
+                    return;
+                }
+
+                if ($scope.queryResult.getData() == null) {
+                } else {
+                    // We need to give the pivot table its own copy of the data, because its change
+                    // it which interferes with other visualizations.
+                    var data = $.extend(true, [], $scope.queryResult.getData());
+                    $(element).pivotUI(data, {
+                         renderers: $.pivotUtilities.renderers
+                    }, true);
+                }
+            });
+        }
+    }
+});

rd_ui/app/scripts/visualizations/table.js

@@ -0,0 +1,109 @@
+(function () {
+  var tableVisualization = angular.module('redash.visualization');
+
+  tableVisualization.config(['VisualizationProvider', function (VisualizationProvider) {
+    VisualizationProvider.registerVisualization({
+      type: 'TABLE',
+      name: 'Table',
+      renderTemplate: '<grid-renderer options="visualization.options" query-result="queryResult"></grid-renderer>',
+      skipTypes: true
+    });
+  }]);
+
+  tableVisualization.directive('gridRenderer', function () {
+    return {
+      restrict: 'E',
+      scope: {
+        queryResult: '=',
+        itemsPerPage: '='
+      },
+      templateUrl: "/views/grid_renderer.html",
+      replace: false,
+      controller: ['$scope', '$filter', function ($scope, $filter) {
+        $scope.gridColumns = [];
+        $scope.gridData = [];
+        $scope.gridConfig = {
+          isPaginationEnabled: true,
+          itemsByPage: $scope.itemsPerPage || 15,
+          maxSize: 8
+        };
+
+        $scope.$watch('queryResult && queryResult.getData()', function (data) {
+          if (!data) {
+            return;
+          }
+
+          if ($scope.queryResult.getData() == null) {
+            $scope.gridColumns = [];
+            $scope.gridData = [];
+            $scope.filters = [];
+          } else {
+            $scope.filters = $scope.queryResult.getFilters();
+
+            var prepareGridData = function (data) {
+              var gridData = _.map(data, function (row) {
+                var newRow = {};
+                _.each(row, function (val, key) {
+                  newRow[$scope.queryResult.getColumnCleanName(key)] = val;
+                })
+                return newRow;
+              });
+
+              return gridData;
+            };
+
+            $scope.gridData = prepareGridData($scope.queryResult.getData());
+
+            var columns = $scope.queryResult.getColumns();
+            $scope.gridColumns = _.map($scope.queryResult.getColumnCleanNames(), function (col, i) {
+              var columnDefinition = {
+                'label': $scope.queryResult.getColumnFriendlyNames()[i],
+                'map': col
+              };
+
+              var columnType = columns[i].type;
+
+              if (columnType === 'integer') {
+                columnDefinition.formatFunction = 'number';
+                columnDefinition.formatParameter = 0;
+              } else if (columnType === 'float') {
+                columnDefinition.formatFunction = 'number';
+                columnDefinition.formatParameter = 2;
+              } else if (columnType === 'boolean') {
+                columnDefinition.formatFunction = function (value) {
+                  if (value !== undefined) {
+                    return "" + value;
+                  }
+                  return value;
+                };
+              } else if (columnType === 'date') {
+                columnDefinition.formatFunction = function (value) {
+                  if (value) {
+                    return value.format("DD/MM/YY");
+                  }
+                  return value;
+                };
+              } else if (columnType === 'datetime') {
+                columnDefinition.formatFunction = function (value) {
+                  if (value) {
+                    return value.format("DD/MM/YY HH:mm");
+                  }
+                  return value;
+                };
+              } else {
+                columnDefinition.formatFunction = function (value) {
+                  if (angular.isString(value)) {
+                    value = $filter('linkify')(value);
+                  }
+                  return value;
+                }
+              }
+
+              return columnDefinition;
+            });
+          }
+        });
+      }]
+    }
+  })
+}());

rd_ui/app/styles/login.css

@@ -0,0 +1,37 @@
+.main {
+    max-width: 320px;
+    margin: 0 auto;
+}
+
+.login-or {
+    position: relative;
+    font-size: 18px;
+    color: #aaa;
+    margin-top: 10px;
+            margin-bottom: 10px;
+    padding-top: 10px;
+    padding-bottom: 10px;
+}
+
+.span-or {
+    display: block;
+    position: absolute;
+    left: 50%;
+    top: -2px;
+    margin-left: -25px;
+    background-color: #fff;
+    width: 50px;
+    text-align: center;
+}
+
+.hr-or {
+    background-color: #cdcdcd;
+    height: 1px;
+    margin-top: 0px !important;
+    margin-bottom: 0px !important;
+}
+
+/*h3 {*/
+    /*text-align: center;*/
+    /*line-height: 300%;*/
+/*}*/

rd_ui/app/styles/redash.css

@@ -25,7 +25,33 @@ a.navbar-brand {
     margin-top: 5px;
     margin-bottom: 5px;
 }
+.avatar img {
+    width: 40px;
+    height: 40px;
+}
 
+#logout {
+    color: white;
+    position: relative;
+    left: -9px;
+    bottom: -11px;
+}
+
+.details-toggle {
+    cursor: pointer;
+}
+.details-toggle::before {
+    content: '▸';
+    margin-right: 5px;
+}
+.details-toggle.open::before {
+    content: '▾';
+    margin-right: 5px;
+}
+
+.edit-in-place span {
+    white-space: pre-line;
+}
 .edit-in-place span.editable {
     cursor: pointer;
 }
@@ -60,10 +86,15 @@ a.navbar-brand {
     margin-bottom: 0px;
 }
 
-.panel-heading > a {
+.panel-heading > a,
+.panel-heading .query-link {
     color: inherit;
 }
 
+.panel-heading .query-link:hover {
+    text-decoration: none;
+}
+
 /* angular-growl */
 .growl {
     position: fixed;
@@ -214,7 +245,46 @@ to add those CSS styles here. */
     background-color: #FF8080;
     border-radius: 50%;
 }
+.nav-tabs > li.rd-tab-btn {
+    float: right;
+}
 
-visualization-renderer > div {
-    overflow: scroll;
-}
+/* light version of bootstrap's form-control */
+.rd-form-control {
+    display: block;
+    padding: 6px 12px;
+    line-height: 1.428571429;
+    color: #555555;
+    vertical-align: middle;
+    background-color: #ffffff;
+    border: 1px solid #cccccc;
+    border-radius: 4px;
+    -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+    box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+    -webkit-transition: border-color ease-in-out 0.15s, box-shadow ease-in-out 0.15s;
+    transition: border-color ease-in-out 0.15s, box-shadow ease-in-out 0.15s;
+}
+.rd-form-control {
+     width: 100%;
+}
+pivot-table-renderer > table, grid-renderer > div, visualization-renderer > div {
+    overflow: auto;
+}
+
+.rd-widget-textbox p {
+    margin-bottom: 0;
+}
+
+.iframe-container {
+    height: 100%;
+}
+
+/*
+bootstrap's hidden-xs class adds display:block when not hidden
+use this class when you need to keep the original display value
+*/
+@media (max-width: 767px) {
+    .rd-hidden-xs {
+        display: none !important;
+    }
+}

rd_ui/app/views/admin_status.html

@@ -21,30 +21,20 @@
                     Started
                 </li>
                 <li class="list-group-item">
-                    <span class="badge">{{manager.queue_size}}</span>
-                    Queue Size
+                    <span class="badge">{{manager.outdated_queries_count}}</span>
+                    Outdated Queries Count
+                </li>
+
+                <li class="list-group-item" ng-if="flowerUrl">
+                    <a href="/admin/workers">Workers' Status</a>
                 </li>
             </ul>
             <ul class="list-group col-lg-4">
-                <div ng-repeat="worker in workers">
-                    <li class="list-group-item active">Worker {{$index+1}}</li>
-                    <li class="list-group-item">
-                    <span class="badge" am-time-ago="worker.updated_at*1000.0"></span>
-                    Updated
+                <li class="list-group-item active">Queues</li>
+                <li class="list-group-item" ng-repeat="(name, value) in manager.queues">
+                    <span class="badge">{{value.size}}</span>
+                    {{name}} ({{value.data_sources}})
                 </li>
-                <li class="list-group-item">
-                    <span class="badge" am-time-ago="worker.started_at*1000.0"></span>
-                    Started
-                </li>
-                <li class="list-group-item">
-                    <span class="badge">{{worker.jobs_count}}</span>
-                    Jobs Received
-                </li>
-                <li class="list-group-item">
-                    <span class="badge">{{worker.done_jobs_count}}</span>
-                    Jobs Done
-                </li>
-                </div>
             </ul>
         </div>
         <div class="panel-footer">Next refresh: <span am-time-ago="refresh_time"></span></div>

rd_ui/app/views/admin_workers.html

@@ -0,0 +1,3 @@
+<div class="container-fluid iframe-container">
+    <iframe src="{{flowerUrl}}" style="width:100%; height:100%; background-color:transparent;"></iframe>
+</div>

rd_ui/app/views/dashboard.html

@@ -14,6 +14,7 @@
                 </button>
             </span>
     </h2>
+    <filters></filters>
 </div>
 
 <div class="container" id="dashboard">
@@ -21,11 +22,12 @@
         <div ng-repeat="widget in row" class="col-lg-{{widget.width | colWidth}}"
              ng-controller='WidgetCtrl'>
 
-            <div class="panel panel-default">
+            <div class="panel panel-default" ng-if="type=='visualization'">
                 <div class="panel-heading">
-                    <h3 class="panel-title" style="cursor: pointer;" ng-click="open(query, widget.visualization)">
+                    <h3 class="panel-title">
                         <p>
-                            <span ng-bind="query.name"></span>
+                            <span ng-hide="currentUser.hasPermission('view_query')">{{query.name}}</span>
+                            <query-link query="query" visualization="widget.visualization" ng-show="currentUser.hasPermission('view_query')"></query-link>
                         </p>
                         <div class="text-muted" ng-bind="query.description"></div>
                     </h3>
@@ -39,12 +41,32 @@
                           tooltip-placement="bottom">Updated: <span am-time-ago="queryResult.getUpdatedAt()"></span></span>
 
                     <span class="pull-right">
-                        <a class="btn btn-default btn-xs" ng-href="/queries/{{query.id}}#{{widget.visualization.id}}"><span class="glyphicon glyphicon-link"></span></a>
+                        <a class="btn btn-default btn-xs" ng-href="/queries/{{query.id}}#{{widget.visualization.id}}" ng-show="currentUser.hasPermission('view_query')"><span class="glyphicon glyphicon-link"></span></a>
                         <button type="button" class="btn btn-default btn-xs" ng-show="dashboard.canEdit()" ng-click="deleteWidget()" title="Remove Widget"><span class="glyphicon glyphicon-trash"></span></button>
                     </span>
 
+                    <span class="pull-right">
+                        <a class="btn btn-default btn-xs" ng-disabled="!queryResult.getData()" query-result-link target="_self">
+                            <span class="glyphicon glyphicon-cloud-download"></span>
+                        </a>
+                    </span>
+                </div>
+            </div>
+
+            <div class="panel panel-default rd-widget-textbox" ng-if="type=='textbox'" ng-mouseenter="showControls = true" ng-mouseleave="showControls = false">
+                <div class="panel-body">
+                    <div class="row">
+                        <div class="col-lg-11">
+                            <p ng-bind-html="widget.text | markdown"></p>
+                        </div>
+                        <div class="col-lg-1">
+                          <span class="pull-right" ng-show="showControls">
+                              <button type="button" class="btn btn-default btn-xs" ng-show="dashboard.canEdit()" ng-click="deleteWidget()" title="Remove Widget"><span class="glyphicon glyphicon-trash"></span></button>
+                          </span>
+                        </div>
+                    </div>
                 </div>
             </div>
         </div>
     </div>
-</div>
+</div>

rd_ui/app/views/edit_visualization.html

@@ -1,24 +0,0 @@
-<form role="form" name="visForm" ng-submit="submit()">
-  <div class="form-group">
-    <label class="control-label">Name</label>
-    <input name="name" type="text" class="form-control" ng-model="vis.name" placeholder="{{vis.type}}">
-  </div>
-
-  <div class="form-group">
-    <label class="control-label">Visualization Type</label>
-    <select required ng-model="vis.type" ng-options="value as key for (key, value) in visTypes" class="form-control" ng-change="typeChanged()"></select>
-  </div>
-
-  <div class="form-group" ng-show="vis.type == visTypes.Chart">
-    <label class="control-label">Chart Type</label>
-    <select required ng-model="vis.options.series.type" ng-options="value as key for (key, value) in seriesTypes" class="form-control"></select>
-
-    <label class="control-label">Stacking</label>
-    <select required ng-model="stacking" ng-options="value as key for (key, value) in stackingOptions" class="form-control"></select>
-  </div>
-
-  <div class="form-group">
-    <button type="submit" class="btn btn-primary">Save</button>
-  </div>
-
-</form>

rd_ui/app/views/grid_renderer.html

@@ -1,17 +1,4 @@
 <div>
-    <div class="btn-group pull-right" ng-repeat="filter in filters">
-        <button type="button" class="btn btn-default dropdown-toggle" data-toggle="dropdown">
-            {{filter.friendlyName}}: {{filter.current}}<span class="caret"></span>
-        </button>
-        <ul class="dropdown-menu" role="menu">
-            <li ng-repeat="value in filter.values">
-                <a href="#" ng-click="filter.current = value">{{value}}</a>
-            </li>
-            <li class="divider"></li>
-            <li><a href="#" ng-click="filter.current = 'All'">All</a></li>
-        </ul>
-    </div>
-
     <smart-table rows="gridData" columns="gridColumns"
                  config="gridConfig"
                  class="table table-condensed table-hover"></smart-table>

rd_ui/app/views/index.html

@@ -3,7 +3,7 @@
     <div class="list-group" ng-repeat="(name, dashboards) in allDashboards">
         <div class="list-group-item active">
             {{name}}
-            <button type="button" class="btn btn-sm btn-link" data-toggle="modal" href="#new_dashboard_dialog" tooltip="New Dashboard"><span class="glyphicon glyphicon-plus-sign"></span></button>
+            <button ng-show="currentUser.hasPermission('create_dashboard')" type="button" class="btn btn-sm btn-link" data-toggle="modal" href="#new_dashboard_dialog" tooltip="New Dashboard"><span class="glyphicon glyphicon-plus-sign"></span></button>
         </div>
         <div class="list-group-item" ng-repeat="dashboard in dashboards" >
             <button type="button" class="close delete-button" aria-hidden="true" ng-show="dashboard.canEdit()" ng-click="archiveDashboard(dashboard)" tooltip="Delete Dashboard">&times;</button>
@@ -11,7 +11,7 @@
         </div>
     </div>
 
-    <div ng-show="currentUser.is_admin">
+    <div ng-show="currentUser.hasPermission('admin')">
         <div class="list-group">
             <div class="list-group-item active">Admin</div>
             <a href="/admin/status" class="list-group-item">Status</a>

rd_ui/app/views/new_widget_form.html

@@ -6,35 +6,54 @@
                 <h4 class="modal-title">Add Widget</h4>
             </div>
             <div class="modal-body">
-                <p>
-                    <form class="form-inline" role="form" ng-submit="loadVisualizations()">
-                      <div class="form-group">
-                        <input class="form-control" placeholder="Query Id" ng-model="queryId">
-                      </div>
-                      <button type="submit" class="btn btn-primary" ng-disabled="!queryId">
-                        <span class="glyphicon glyphicon-refresh"></span> Load
-                      </button>
-                    </form>
+                <p class="btn-group">
+                    <button type="button" class="btn btn-default" ng-class="{active: isVisualization()}" ng-click="setType('visualization')">Visualization</button>
+                    <button type="button" class="btn btn-default" ng-class="{active: isTextBox()}" ng-click="setType('textbox')">Text Box</button>
                 </p>
 
-                <div ng-show="query">
-                  <div class="form-group">
-                      <label for="">Choose Visualation</label>
-                      <select ng-model="selectedVis" ng-options="vis as vis.name group by vis.type for vis in query.visualizations" class="form-control"></select>
-                  </div>
+                <div ng-show="isTextBox()">
+                    <div class="form-group">
+                        <textarea class="form-control" ng-model="text" rows="3"></textarea>
+                    </div>
+                    <div ng-show="text">
+                        <strong>Preview:</strong>
+                        <p ng-bind-html="text | markdown"></p>
+                    </div>
+                </div>
 
-                  <div class="form-group">
-                      <label for="">Widget Size</label>
-                      <select class="form-control" ng-model="widgetSize" ng-options="c.value as c.name for c in widgetSizes"></select>
-                  </div>
+                <div ng-show="isVisualization()">
+                    <p>
+                        <form class="form-inline" role="form" ng-submit="loadVisualizations()">
+                            <div class="form-group">
+                                <input class="form-control" placeholder="Query Id" ng-model="queryId">
+                            </div>
+                            <button type="submit" class="btn btn-primary" ng-disabled="!queryId">
+                                Load visualizations
+                            </button>
+                        </form>
+                    </p>
+
+                    <div ng-show="query">
+                      <div class="form-group">
+                          <label for="">Choose Visualization</label>
+                          <select ng-model="selectedVis" ng-options="vis as vis.name group by vis.type for vis in query.visualizations" class="form-control"></select>
+                      </div>
+                    </div>
+                </div>
+
+                <div class="form-group">
+                    <label for="">Widget Size</label>
+                    <select class="form-control" ng-model="widgetSize"
+                            ng-options="c.value as c.name for c in widgetSizes"></select>
                 </div>
             </div>
+
             <div class="modal-footer">
                 <button type="button" class="btn btn-default" ng-disabled="saveInProgress" data-dismiss="modal">Close</button>
-                <button type="button" class="btn btn-primary" ng-disabled="saveInProgress" ng-click="saveWidget()">Add to Dashboard</button>
+                <button type="button" class="btn btn-primary" ng-disabled="saveInProgress || !(selectedVis || isTextBox())" ng-click="saveWidget()">Add to Dashboard</button>
             </div>
         </div>
         <!-- /.modal-content -->
     </div>
     <!-- /.modal-dialog -->
-</div>
+</div>

rd_ui/app/views/query.html

@@ -0,0 +1,183 @@
+
+<div class="container">
+
+    <alert-unsaved-changes ng-if="canEdit" is-dirty="isDirty"></alert-unsaved-changes>
+
+    <div class="row">
+        <div class="col-lg-12">
+            <div class="row">
+                <div class="col-lg-10">
+                    <h2>
+                        <edit-in-place editable="isQueryOwner" done="saveName" ignore-blanks='true' value="query.name"></edit-in-place>
+                    </h2>
+                    <p>
+                        <em>
+                            <edit-in-place editable="isQueryOwner" done="saveDescription" editor="textarea" placeholder="No description" ignore-blanks='false' value="query.description"></edit-in-place>
+                        </em>
+                    </p>
+                </div>
+
+                <div class="col-lg-2">
+                    <div class="rd-hidden-xs pull-right">
+                        <query-source-link></query-source-link>
+                    </div>
+                </div>
+            </div>
+
+            <div class="visible-xs">
+                <p>
+                    <span class="text-muted">Last update </span>
+                    <strong>
+                        <rd-time-ago value="queryResult.query_result.retrieved_at"></rd-time-ago>
+                    </strong>
+                    &nbsp;
+                    <span class="text-muted">Created By </span>
+                    <strong ng-hide="isQueryOwner">{{query.user.name}}</strong>
+                    <strong ng-show="isQueryOwner">You</strong>
+                    &nbsp;
+                    <span class="text-muted">Runtime </span>
+                    <strong ng-show="!queryExecuting">{{queryResult.getRuntime() | durationHumanize}}</strong>
+                    <span ng-show="queryExecuting">Running&hellip;</span>
+                    &nbsp;
+                    <span class="text-muted">Rows </span>
+                    <strong>{{queryResult.getData().length}}</strong>
+                </p>
+                <p>
+                    <query-source-link></query-source-link>
+                </p>
+            </div>
+        </div>
+    </div>
+
+    <hr>
+
+    <div class="row">
+        <div class="col-lg-12">
+            <div ng-show="sourceMode">
+                <p>
+                    <button type="button" class="btn btn-primary btn-xs" ng-disabled="queryExecuting" ng-click="executeQuery()">
+                        <span class="glyphicon glyphicon-play"></span> Execute
+                    </button>
+                    <query-formatter></query-formatter>
+                    <span class="pull-right">
+                        <button class="btn btn-xs btn-default rd-hidden-xs" ng-click="duplicateQuery()">
+                            <span class="glyphicon glyphicon-share-alt"></span> Fork
+                        </button>
+
+                        <button class="btn btn-success btn-xs" ng-show="canEdit" ng-click="saveQuery()">
+                            <span class="glyphicon glyphicon-floppy-disk"> </span> Save<span ng-show="isDirty">&#42;</span>
+                        </button>
+                    </span>
+                </p>
+            </div>
+
+            <!-- code editor -->
+            <div ng-show="sourceMode">
+                <p>
+                    <query-editor query="query" lock="queryExecuting"></query-editor>
+                </p>
+                <hr>
+            </div>
+        </div>
+
+    </div>
+
+    <div class="row">
+        <div class="col-lg-3 rd-hidden-xs">
+            <p>
+                <span class="glyphicon glyphicon-time"></span>
+                <span class="text-muted">Last update </span>
+                <strong>
+                    <rd-time-ago value="queryResult.query_result.retrieved_at"></rd-time-ago>
+                </strong>
+            </p>
+            <p>
+                <span class="glyphicon glyphicon-user"></span>
+                <span class="text-muted">Created By </span>
+                <strong ng-hide="isQueryOwner">{{query.user.name}}</strong>
+                <strong ng-show="isQueryOwner">You</strong>
+            </p>
+            <p>
+                <span class="glyphicon glyphicon-play"></span>
+                <span class="text-muted">Runtime </span>
+                <strong ng-show="!queryExecuting">{{queryResult.getRuntime() | durationHumanize}}</strong>
+                <span ng-show="queryExecuting">Running&hellip;</span>
+            </p>
+            <p>
+                <span class="glyphicon glyphicon-align-justify"></span>
+                <span class="text-muted">Rows </span><strong>{{queryResult.getData().length}}</strong>
+            </p>
+            <p>
+                <span class="glyphicon glyphicon-refresh"></span>
+                <span class="text-muted">Refresh Interval</span>
+                <query-refresh-select></query-refresh-select>
+            </p>
+
+            <p>
+                <span class="glyphicon glyphicon-hdd"></span>
+                <span class="text-muted">Data Source</span>
+                <select ng-disabled="!isQueryOwner" ng-model="query.data_source_id" ng-change="updateDataSource()" ng-options="ds.id as ds.name for ds in dataSources"></select>
+            </p>
+
+            <hr>
+
+            <p>
+                <a class="btn btn-primary btn-sm" ng-disabled="queryExecuting || !queryResult.getData()" query-result-link target="_self">
+                    <span class="glyphicon glyphicon-cloud-download"></span>
+                    <span class="rd-hidden-xs">Download Dataset</span>
+                </a>
+            </p>
+        </div>
+
+        <div class="col-lg-9">
+            <!-- alerts -->
+            <div class="alert alert-info" ng-show="queryResult.getStatus() == 'processing'">
+                Executing query&hellip; <rd-timer timestamp="queryResult.getUpdatedAt()"></rd-timer>
+                <button type="button" class="btn btn-warning btn-xs pull-right" ng-disabled="cancelling" ng-click="cancelExecution()">Cancel</button>
+            </div>
+            <div class="alert alert-info" ng-show="queryResult.getStatus() == 'waiting'">
+                Query in queue&hellip; <rd-timer timestamp="queryResult.getUpdatedAt()"></rd-timer>
+                <button type="button" class="btn btn-warning btn-xs pull-right" ng-disabled="cancelling" ng-click="cancelExecution()">Cancel</button>
+            </div>
+            <div class="alert alert-danger" ng-show="queryResult.getError()">Error running query: <strong>{{queryResult.getError()}}</strong></div>
+
+            <!-- tabs and data -->
+            <div ng-show="showDataset">
+                <div class="row">
+                    <div class="col-lg-12">
+                        <ul class="nav nav-tabs">
+                            <rd-tab tab-id="table" name="Table"></rd-tab>
+                            <rd-tab tab-id="pivot" name="Pivot Table"></rd-tab>
+                            <rd-tab tab-id="{{vis.id}}" name="{{vis.name}}" ng-if="vis.type!='TABLE'" ng-repeat="vis in query.visualizations">
+                                <span class="remove" ng-click="deleteVisualization($event, vis)" ng-show="canEdit"> &times;</span>
+                            </rd-tab>
+                            <rd-tab tab-id="add" name="&plus; New" removeable="true" ng-show="canEdit"></rd-tab>
+                            <li ng-if="!sourceMode" class="rd-tab-btn"><button class="btn btn-sm btn-default" ng-click="executeQuery()" ng-disabled="queryExecuting" title="Refresh Dataset"><span class="glyphicon glyphicon-refresh"></span></button></li>
+                        </ul>
+                    </div>
+                </div>
+                <div class="row">
+                    <div class="col-lg-12">
+                        <div ng-show="selectedTab == 'table'" >
+                            <filters></filters>
+                            <grid-renderer query-result="queryResult" items-per-page="50"></grid-renderer>
+                        </div>
+
+                        <pivot-table-renderer ng-show="selectedTab == 'pivot'" query-result="queryResult"></pivot-table-renderer>
+
+                        <div ng-show="selectedTab == vis.id" ng-repeat="vis in query.visualizations">
+                            <visualization-renderer visualization="vis" query-result="queryResult"></visualization-renderer>
+
+                            <edit-visulatization-form visualization="vis" query="query" query-result="queryResult" ng-show="canEdit"></edit-visulatization-form>
+                        </div>
+
+                        <div ng-if="canEdit" ng-show="selectedTab == 'add'">
+                            <visualization-renderer visualization="newVisualization" query-result="queryResult"></visualization-renderer>
+                            <edit-visulatization-form visualization="newVisualization" query="query" query-result="queryResult" ng-show="canEdit" open-editor="true" on-new-success="setVisualizationTab"></edit-visulatization-form>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+    </div>
+</div>

rd_ui/app/views/queryfiddle.html

@@ -1,102 +0,0 @@
-<div class="container">
-    <div class="row">
-        <div class="panel panel-default">
-            <div class="panel-heading">
-                <h3 class="panel-title">
-                    <p>
-                     <edit-in-place editable="currentUser.canEdit(query)" ignore-blanks='true' value="query.name"></edit-in-place>
-                 </p>
-                </h3>
-                <p>
-                    <edit-in-place editable="currentUser.canEdit(query)" editor="textarea" placeholder="No description" ignore-blanks='false' value="query.description" class="text-muted"></edit-in-place>
-                </p>
-            </div>
-            <div class="panel-body">
-                <textarea ui-codemirror="editorOptions" ng-model="query.query"></textarea>
-
-                <div>
-                    <a class="btn btn-default" ng-disabled="queryExecuting || !queryResult.getData()" ng-href="{{dataUri}}" download="{{dataFilename}}" target="_self">
-                        <span class="glyphicon glyphicon-floppy-disk"></span> Download Data Set
-                    </a>
-                    <button type="button" class="btn btn-default center-x" ng-click="formatQuery()"><span class="glyphicon glyphicon-ok"></span> Format SQL</button>
-
-                    <div class="btn-group pull-right">
-                        <button type="button" class="btn btn-default" ng-click="duplicateQuery()">Duplicate</button>
-                        <button type="button" class="btn btn-default" ng-disabled="!currentUser.canEdit(query)" ng-click="saveQuery()">Save
-                            <span ng-show="dirty">&#42;</span>
-                        </button>
-                        <button type="button" class="btn btn-primary" ng-disabled="queryExecuting" ng-click="executeQuery()">Execute</button>
-                    </div>
-                </div>
-
-            </div>
-            <div class="panel-footer">
-                <span ng-show="queryResult.getRuntime()>=0">Query runtime: {{queryResult.getRuntime() | durationHumanize}} | </span>
-                <span ng-show="queryResult.query_result.retrieved_at">Last update time: <span am-time-ago="queryResult.query_result.retrieved_at"></span> | </span>
-                <span ng-show="queryResult.getStatus() == 'done'">Rows: {{queryResult.getData().length}} | </span>
-                Created by: {{query.user.name}}
-                <div class="pull-right">Refresh query: <select ng-model="query.ttl" ng-options="c.value as c.name for c in refreshOptions"></select><br></div>
-            </div>
-        </div>
-
-        <div class="alert alert-info" ng-show="queryResult.getStatus() == 'processing'">
-            Executing query... <rd-timer timestamp="queryResult.getUpdatedAt()"></rd-timer>
-            <button type="button" class="btn btn-warning btn-xs pull-right" ng-disabled="cancelling" ng-click="cancelExecution()">Cancel</button>
-        </div>
-        <div class="alert alert-info" ng-show="queryResult.getStatus() == 'waiting'">
-            Query in queue... <rd-timer timestamp="queryResult.getUpdatedAt()"></rd-timer>
-            <button type="button" class="btn btn-warning btn-xs pull-right" ng-disabled="cancelling" ng-click="cancelExecution()">Cancel</button>
-        </div>
-        <div class="alert alert-danger" ng-show="queryResult.getError()">Error running query: <strong>{{queryResult.getError()}}</strong></div>
-    </div>
-
-    <div class="row" ng-show="queryResult.getStatus() == 'done'">
-        <ul class="nav nav-tabs">
-            <rd-tab id="table" name="Table"></rd-tab>
-            <rd-tab id="pivot" name="Pivot Table"></rd-tab>
-            <!-- hide the table visualization -->
-            <rd-tab id="{{vis.id}}" name="{{vis.name}}" ng-hide="vis.type=='TABLE'" ng-repeat="vis in query.visualizations">
-                <span class="remove" ng-click="deleteVisualization($event, vis)" ng-show="currentUser.canEdit(query)"> &times;</span>
-            </rd-tab>
-            <rd-tab id="add" name="&plus;New" removeable="true" ng-show="currentUser.canEdit(query)"></rd-tab>
-        </ul>
-
-        <div class="col-lg-12" ng-show="selectedTab == 'table'">
-            <grid-renderer query-result="queryResult" items-per-page="50"></grid-renderer>
-        </div>
-
-        <div class="col-lg-12" ng-show="selectedTab == 'pivot'">
-            <pivot-table-renderer query-result="queryResult"></pivot-table-renderer>
-        </div>
-
-        <div class="col-lg-12" ng-show="selectedTab == vis.id" ng-repeat="vis in query.visualizations">
-            <div class="row" ng-show="currentUser.canEdit(query)">
-                <p>
-                    <div class="col-lg-12">
-                        <edit-visulatization-form vis="vis" query="query"></edit-visulatization-form>
-                    </div>
-                </p>
-            </div>
-            <div class="row">
-                <p>
-                    <div class="col-lg-12">
-                        <visualization-renderer visualization="vis" query-result="queryResult"></visualization-renderer>
-                    </div>
-                </p>
-            </div>
-        </div>
-
-        <div class="col-lg-12" ng-show="selectedTab == 'add'">
-            <div class="row">
-                 <p>
-                    <div class="col-lg-6">
-                        <edit-visulatization-form vis="newVisualization" query="query"></edit-visulatization-form>
-                    </div>
-                    <div class="col-lg-6">
-                        <visualization-renderer visualization="newVisualization" query-result="queryResult"></visualization-renderer>
-                    </div>
-                </p>
-            </div>
-        </div>
-    </div>
-</div>

rd_ui/app/views/visualizations/chart_editor.html

@@ -0,0 +1,100 @@
+<div class="form-horizontal">
+    <div class="panel panel-default">
+        <div class="panel-body">
+            <div class="form-group">
+                <label class="control-label col-sm-2">Stacking</label>
+
+                <div class="col-sm-10">
+                    <select required ng-model="stacking"
+                            ng-options="value as key for (key, value) in stackingOptions"
+                            class="form-control"></select>
+                </div>
+            </div>
+            <div class="form-group">
+                <label class="control-label col-sm-2">X Axis Type</label>
+
+                <div class="col-sm-10">
+                    <select required ng-model="xAxisType" ng-options="value as key for (key, value) in xAxisOptions"
+                            class="form-control"></select>
+                </div>
+            </div>
+
+            <div class="form-group">
+                <label class="control-label col-sm-2">Series Type</label>
+
+                <div class="col-sm-10">
+                    <select required ng-options="value as key for (key, value) in seriesTypes"
+                            ng-model="globalSeriesType" class="form-control"></select>
+                </div>
+            </div>
+        </div>
+    </div>
+
+    <div class="row">
+        <div class="col-lg-6">
+            <div class="list-group">
+                <div class="list-group-item active">
+                    Columns Mapping
+                </div>
+                <div class="list-group-item">
+                    <div class="form-group" ng-repeat="column in columns">
+                        <label class="control-label col-sm-4">{{column.name}}</label>
+
+                        <div class="col-sm-8">
+                            <select ng-options="value as key for (key, value) in columnTypes" class="form-control"
+                                    ng-model="columnTypeSelection[column.name]"></select>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+
+        <div class="col-lg-6" ng-if="series.length > 0">
+            <div class="list-group" ng-repeat="seriesName in series">
+                <div class="list-group-item active">
+                    {{seriesName}}
+                </div>
+                <div class="list-group-item">
+                    <div class="form-group">
+                        <label class="control-label col-sm-3">Type</label>
+
+                        <div class="col-sm-9">
+                            <select required ng-model="visualization.options.seriesOptions[seriesName].type"
+                                    ng-options="value as key for (key, value) in seriesTypes"
+                                    class="form-control"></select>
+                        </div>
+                    </div>
+                    <div class="form-group">
+                        <label class="control-label col-sm-3">zIndex</label>
+
+                        <div class="col-sm-9">
+                            <select required ng-model="visualization.options.seriesOptions[seriesName].zIndex"
+                                    ng-options="o as o for o in zIndexes"
+                                    class="form-control"></select>
+                        </div>
+                    </div>
+                    <div class="form-group">
+                        <label class="control-label col-sm-3">y Axis</label>
+
+                        <div class="col-sm-9">
+                            <select required ng-model="visualization.options.seriesOptions[seriesName].yAxis"
+                                    ng-options="o[0] as o[1] for o in yAxes"
+                                    class="form-control"></select>
+                        </div>
+                    </div>
+                    <div class="form-group">
+                        <label class="control-label col-sm-3">Name</label>
+
+                        <div class="col-sm-9">
+                            <input name="seriesName" type="text" class="form-control"
+                                   ng-model="visualization.options.seriesOptions[seriesName].name"
+                                   placeholder="{{seriesName}}">
+                        </div>
+                    </div>
+                </div>
+            </div>
+
+
+        </div>
+    </div>
+</div>

rd_ui/app/views/visualizations/cohort_editor.html

@@ -0,0 +1,15 @@
+<div class="form-group">
+    <label class="control-label">Time Label</label>
+    <input type="text" class="form-control" ng-model="cohortOptions.timeLabel">
+    <label class="control-label">People Label</label>
+    <input type="text" class="form-control" ng-model="cohortOptions.peopleLabel">
+
+    <label class="control-label">Bucket Column</label>
+    <select ng-model="bucket_column" ng-options="value as key for (key, value) in columns" class="form-control"></select>
+    <label class="control-label">Bucket Total Value Column</label>
+    <select ng-model="total_column" ng-options="value as key for (key, value) in columns" class="form-control"></select>
+    <label class="control-label">Day Number Column</label>
+    <select ng-model="value_column" ng-options="value as key for (key, value) in columns" class="form-control"></select>
+    <label class="control-label">Day Value Column</label>
+    <select ng-model="day_column" ng-options="value as key for (key, value) in columns" class="form-control"></select>
+</div>

rd_ui/app/views/visualizations/edit_visualization.html

@@ -0,0 +1,27 @@
+<div>
+  <span ng-click="openEditor=!openEditor" class="details-toggle" ng-class="{open: openEditor}">Edit</span>
+
+  <form ng-if="openEditor" role="form" name="visForm" ng-submit="submit()">
+    <div class="form-group">
+      <label class="control-label">Name</label>
+      <input name="name" type="text" class="form-control" ng-model="visualization.name" placeholder="{{visualization.type | capitalize}}">
+    </div>
+
+    <div class="form-group">
+      <label class="control-label">Visualization Type</label>
+      <select required ng-model="visualization.type" ng-options="value as key for (key, value) in visTypes" class="form-control" ng-change="typeChanged()"></select>
+    </div>
+
+    <visualization-options-editor></visualization-options-editor>
+
+    <div class="form-group" ng-if="editRawOptions">
+      <label class="control-label">Advanced</label>
+      <textarea json-text ng-model="visualization.options" class="form-control" rows="10"></textarea>
+    </div>
+
+    <div class="form-group">
+      <button type="submit" class="btn btn-primary">Save</button>
+    </div>
+
+  </form>
+</div>

rd_ui/app/views/visualizations/filters.html

@@ -0,0 +1,8 @@
+<div class="well well-sm" ng-show="filters">
+    <div ng-repeat="filter in filters">
+        {{filter.friendlyName}}:
+        <select ui-select2='select2Options' ng-model="filter.current" ng-multiple="{{filter.multiple}}">
+            <option ng-repeat="value in filter.values" value="{{value}}">{{value}}</option>
+        </select>
+    </div>
+</div>

rd_ui/bower.json

@@ -11,22 +11,27 @@
     "moment": "2.1.0",
     "angular-ui-bootstrap": "0.5.0",
     "angular-ui-codemirror": "0.0.5",
-    "highcharts": "3.0.1",
+    "highcharts": "3.0.10",
     "underscore": "1.5.1",
-    "angular-resource": "1.0.7",
+    "angular-resource": "1.2.15",
     "angular-growl": "0.3.1",
     "angular-route": "1.2.7",
-    "pivottable": "git@github.com:arikfr/pivottable.git#master",
-    "cornelius": "git@github.com:restorando/cornelius.git",
+    "pivottable": "~1.1.1",
+    "cornelius": "https://github.com/restorando/cornelius.git",
     "gridster": "0.2.0",
-    "mousetrap": "~1.4.6"
+    "mousetrap": "~1.4.6",
+    "angular-ui-select2": "~0.0.5",
+    "jquery-ui": "~1.10.4",
+    "underscore.string": "~2.3.3",
+    "marked": "~0.3.2",
+    "bucky": "~0.2.6",
+    "pace": "~0.5.1"
   },
   "devDependencies": {
     "angular-mocks": "~1.0.7",
     "angular-scenario": "~1.0.7"
   },
   "resolutions": {
-    "angular": "~1.2.7",
-    "jquery": "~1.9.1"
+    "angular": "1.2.7"
   }
 }

rd_ui/karma-e2e.conf.js

@@ -1,58 +0,0 @@
-// Karma E2E configuration
-
-// base path, that will be used to resolve files and exclude
-basePath = '';
-
-// list of files / patterns to load in the browser
-files = [
-  ANGULAR_SCENARIO,
-  ANGULAR_SCENARIO_ADAPTER,
-  'test/e2e/**/*.js'
-];
-
-// list of files to exclude
-exclude = [];
-
-// test results reporter to use
-// possible values: dots || progress || growl
-reporters = ['progress'];
-
-// web server port
-port = 8080;
-
-// cli runner port
-runnerPort = 9100;
-
-// enable / disable colors in the output (reporters and logs)
-colors = true;
-
-// level of logging
-// possible values: LOG_DISABLE || LOG_ERROR || LOG_WARN || LOG_INFO || LOG_DEBUG
-logLevel = LOG_INFO;
-
-// enable / disable watching file and executing tests whenever any file changes
-autoWatch = false;
-
-// Start these browsers, currently available:
-// - Chrome
-// - ChromeCanary
-// - Firefox
-// - Opera
-// - Safari (only Mac)
-// - PhantomJS
-// - IE (only Windows)
-browsers = ['Chrome'];
-
-// If browser does not capture in given timeout [ms], kill it
-captureTimeout = 5000;
-
-// Continuous Integration mode
-// if true, it capture browsers, run tests and exit
-singleRun = false;
-
-// Uncomment the following lines if you are using grunt's server to run the tests
-// proxies = {
-//   '/': 'http://localhost:9000/'
-// };
-// URL root prevent conflicts with the site root
-// urlRoot = '_karma_';

rd_ui/karma.conf.js

@@ -1,56 +0,0 @@
-// Karma configuration
-
-// base path, that will be used to resolve files and exclude
-basePath = '';
-
-// list of files / patterns to load in the browser
-files = [
-  JASMINE,
-  JASMINE_ADAPTER,
-  'app/bower_components/angular/angular.js',
-  'app/bower_components/angular-mocks/angular-mocks.js',
-  'app/scripts/*.js',
-  'app/scripts/**/*.js',
-  'test/mock/**/*.js',
-  'test/spec/**/*.js'
-];
-
-// list of files to exclude
-exclude = [];
-
-// test results reporter to use
-// possible values: dots || progress || growl
-reporters = ['progress'];
-
-// web server port
-port = 8080;
-
-// cli runner port
-runnerPort = 9100;
-
-// enable / disable colors in the output (reporters and logs)
-colors = true;
-
-// level of logging
-// possible values: LOG_DISABLE || LOG_ERROR || LOG_WARN || LOG_INFO || LOG_DEBUG
-logLevel = LOG_INFO;
-
-// enable / disable watching file and executing tests whenever any file changes
-autoWatch = false;
-
-// Start these browsers, currently available:
-// - Chrome
-// - ChromeCanary
-// - Firefox
-// - Opera
-// - Safari (only Mac)
-// - PhantomJS
-// - IE (only Windows)
-browsers = ['Chrome'];
-
-// If browser does not capture in given timeout [ms], kill it
-captureTimeout = 5000;
-
-// Continuous Integration mode
-// if true, it capture browsers, run tests and exit
-singleRun = false;

rd_ui/package.json

@@ -1,38 +1,39 @@
 {
-  "name": "rd-ui",
-  "version": "0.1.0",
-  "dependencies": {
-  },
+  "name": "rdui",
+  "version": "0.0.0",
+  "dependencies": {},
   "devDependencies": {
-    "grunt": "git+https://github.com/gruntjs/grunt.git#08a3af5",
-    "grunt-contrib-copy": "~0.4.1",
-    "grunt-contrib-concat": "~0.3.0",
-    "grunt-contrib-coffee": "~0.7.0",
-    "grunt-contrib-uglify": "~0.2.0",
-    "grunt-contrib-compass": "~0.5.0",
-    "grunt-contrib-jshint": "~0.6.0",
-    "grunt-contrib-cssmin": "~0.6.0",
-    "grunt-contrib-connect": "~0.3.0",
-    "grunt-contrib-clean": "~0.5.0",
-    "grunt-contrib-htmlmin": "~0.1.3",
-    "grunt-contrib-imagemin": "~0.2.0",
-    "grunt-contrib-watch": "~0.5.2",
-    "grunt-autoprefixer": "~0.2.0",
-    "grunt-usemin": "~0.1.11",
-    "grunt-svgmin": "~0.2.0",
-    "grunt-rev": "~0.1.0",
-    "grunt-open": "~0.2.0",
-    "grunt-concurrent": "~0.3.0",
-    "load-grunt-tasks": "~0.1.0",
-    "connect-livereload": "~0.2.0",
-    "grunt-google-cdn": "~0.2.0",
-    "grunt-ngmin": "~0.0.2",
-    "time-grunt": "~0.1.0",
-    "bower": "~1.2.7",
-    "grunt-cli": "~0.1.9"
+    "grunt": "^0.4.1",
+    "grunt-autoprefixer": "^0.7.3",
+    "grunt-concurrent": "^0.5.0",
+    "grunt-contrib-clean": "^0.5.0",
+    "grunt-contrib-concat": "^0.4.0",
+    "grunt-contrib-connect": "^0.7.1",
+    "grunt-contrib-copy": "^0.5.0",
+    "grunt-contrib-cssmin": "^0.9.0",
+    "grunt-contrib-htmlmin": "^0.3.0",
+    "grunt-contrib-imagemin": "^0.7.0",
+    "grunt-contrib-jshint": "^0.10.0",
+    "grunt-contrib-uglify": "^0.4.0",
+    "grunt-contrib-watch": "^0.6.1",
+    "grunt-filerev": "^0.2.1",
+    "grunt-google-cdn": "^0.4.0",
+    "grunt-newer": "^0.7.0",
+    "grunt-ngmin": "^0.0.3",
+    "grunt-svgmin": "^0.4.0",
+    "grunt-usemin": "^2.1.1",
+    "grunt-wiredep": "^1.7.0",
+    "jshint-stylish": "^0.2.0",
+    "load-grunt-tasks": "^0.4.0",
+    "time-grunt": "^0.3.1",
+    "karma-jasmine": "~0.1.5",
+    "grunt-karma": "~0.8.3",
+    "karma-phantomjs-launcher": "~0.1.4",
+    "karma": "~0.12.19",
+    "karma-ng-html2js-preprocessor": "~0.1.0"
   },
   "engines": {
-    "node": ">=0.8.0"
+    "node": ">=0.10.0"
   },
   "scripts": {
     "test": "grunt test"

rd_ui/test/.jshintrc

@@ -29,6 +29,7 @@
     "expect": false,
     "inject": false,
     "it": false,
+    "jasmine": false,
     "spyOn": false
   }
 }

rd_ui/test/karma.conf.js

@@ -0,0 +1,131 @@
+// Karma configuration
+// http://karma-runner.github.io/0.12/config/configuration-file.html
+// Generated on 2014-07-30 using
+// generator-karma 0.8.3
+
+module.exports = function(config) {
+  'use strict';
+
+  config.set({
+    // enable / disable watching file and executing tests whenever any file changes
+    autoWatch: true,
+
+    // base path, that will be used to resolve files and exclude
+    basePath: '../',
+
+    // testing framework to use (jasmine/mocha/qunit/...)
+    frameworks: ['jasmine'],
+
+    // list of files / patterns to load in the browser
+    files: [
+      'app/bower_components/jquery/jquery.js',
+      'app/bower_components/jquery-ui/ui/jquery-ui.js',
+
+      'app/bower_components/angular/angular.js',
+      'app/bower_components/angular-route/angular-route.js',
+      'app/bower_components/angular-mocks/angular-mocks.js',
+
+      'app/bower_components/bootstrap/js/collapse.js',
+      'app/bower_components/bootstrap/js/modal.js',
+      'app/bower_components/angular-resource/angular-resource.js',
+      'app/bower_components/underscore/underscore.js',
+      'app/bower_components/moment/moment.js',
+      'app/bower_components/angular-moment/angular-moment.js',
+      'app/bower_components/codemirror/lib/codemirror.js',
+      'app/bower_components/codemirror/addon/edit/matchbrackets.js',
+      'app/bower_components/codemirror/addon/edit/closebrackets.js',
+      'app/bower_components/codemirror/mode/sql/sql.js',
+      'app/bower_components/codemirror/mode/javascript/javascript.js',
+      'app/bower_components/angular-ui-codemirror/ui-codemirror.js',
+      'app/bower_components/highcharts/highcharts.js',
+      'app/bower_components/highcharts/modules/exporting.js',
+      'app/bower_components/gridster/dist/jquery.gridster.js',
+      'app/bower_components/angular-growl/build/angular-growl.js',
+      'app/bower_components/pivottable/dist/pivot.js',
+      'app/bower_components/cornelius/src/cornelius.js',
+      'app/bower_components/mousetrap/mousetrap.js',
+      'app/bower_components/mousetrap/plugins/global-bind/mousetrap-global-bind.js',
+      'app/bower_components/select2/select2.js',
+      'app/bower_components/angular-ui-select2/src/select2.js',
+      'app/bower_components/underscore.string/lib/underscore.string.js',
+      'app/bower_components/marked/lib/marked.js',
+      'app/scripts/ng_highchart.js',
+      'app/scripts/ng_smart_table.js',
+      'app/scripts/ui-bootstrap-tpls-0.5.0.min.js',
+      'app/bower_components/bucky/bucky.js',
+      'app/bower_components/pace/pace.js',
+
+      'app/scripts/app.js',
+      'app/scripts/services/services.js',
+      'app/scripts/services/resources.js',
+      'app/scripts/services/notifications.js',
+      'app/scripts/services/dashboards.js',
+      'app/scripts/controllers/controllers.js',
+      'app/scripts/controllers/dashboard.js',
+      'app/scripts/controllers/admin_controllers.js',
+      'app/scripts/controllers/query_view.js',
+      'app/scripts/controllers/query_source.js',
+      'app/scripts/visualizations/base.js',
+      'app/scripts/visualizations/chart.js',
+      'app/scripts/visualizations/cohort.js',
+      'app/scripts/visualizations/table.js',
+      'app/scripts/visualizations/pivot.js',
+      'app/scripts/directives/directives.js',
+      'app/scripts/directives/query_directives.js',
+      'app/scripts/directives/dashboard_directives.js',
+      'app/scripts/filters.js',
+
+      'app/views/**/*.html',
+
+      'test/mocks/*.js',
+      'test/unit/*.js'
+    ],
+
+    // generate js files from html templates
+    preprocessors: {
+      'app/views/**/*.html': 'ng-html2js'
+    },
+
+    // list of files / patterns to exclude
+    exclude: [],
+
+    // web server port
+    port: 8080,
+
+    // Start these browsers, currently available:
+    // - Chrome
+    // - ChromeCanary
+    // - Firefox
+    // - Opera
+    // - Safari (only Mac)
+    // - PhantomJS
+    // - IE (only Windows)
+    browsers: [
+      'PhantomJS'
+    ],
+
+    // Which plugins to enable
+    plugins: [
+      'karma-phantomjs-launcher',
+      'karma-jasmine',
+      'karma-ng-html2js-preprocessor'
+    ],
+
+    // Continuous Integration mode
+    // if true, it capture browsers, run tests and exit
+    singleRun: false,
+
+    colors: true,
+
+    // level of logging
+    // possible values: LOG_DISABLE || LOG_ERROR || LOG_WARN || LOG_INFO || LOG_DEBUG
+    logLevel: config.LOG_INFO,
+
+    // Uncomment the following lines if you are using grunt's server to run the tests
+    // proxies: {
+    //   '/': 'http://localhost:9000/'
+    // },
+    // URL root prevent conflicts with the site root
+    // urlRoot: '_karma_'
+  });
+};

rd_ui/test/mocks/redash_mocks.js

@@ -0,0 +1,108 @@
+featureFlags = [];
+currentUser = {
+  id: 1,
+  name: 'John Mock',
+  email: 'john@example.com',
+  groups: ['default'],
+  permissions: [],
+  canEdit: function(object) {
+    var user_id = object.user_id || (object.user && object.user.id);
+    return user_id && (user_id == currentUser.id);
+  },
+  hasPermission: function(permission) {
+    return this.permissions.indexOf(permission) != -1;
+  }
+};
+
+
+angular.module('redashMocks', [])
+  .value('MockData', {
+    query: {
+      "ttl": -1,
+      "query": "select name from users;",
+      "id": 1803,
+      "description": "",
+      "name": "my test query",
+      "created_at": "2014-01-07T16:11:31.859528+02:00",
+      "query_hash": "c89c235bc73e462e9702debc56adc309",
+
+      "user": {
+        "email": "amirn@everything.me",
+        "id": 48,
+        "name": "Amir Nissim"
+      },
+
+      "visualizations": [{
+        "description": "",
+        "options": {},
+        "type": "TABLE",
+        "id": 636,
+        "name": "Table"
+      }],
+
+      "api_key": "123456789",
+
+      "data_source_id": 1,
+
+      "latest_query_data_id": 106632,
+
+      "latest_query_data": {
+        "retrieved_at": "2014-07-29T10:49:10.951364+03:00",
+        "query_hash": "c89c235bc73e462e9702debc56adc309",
+        "query": "select name from users;",
+        "runtime": 0.0139260292053223,
+        "data": {
+          "rows": [{
+            "name": "Amir Nissim"
+          }, {
+            "name": "Arik Fraimovich"
+          }],
+          "columns": [{
+            "friendly_name": "name",
+            "type": null,
+            "name": "name"
+          }, {
+            "friendly_name": "mail::filter",
+            "type": null,
+            "name": "mail::filter"
+          }]
+        },
+        "id": 106632,
+        "data_source_id": 1
+      }
+
+    },
+
+    queryResult: {
+      "job": {},
+      "query_result": {
+        "retrieved_at": "2014-08-04T13:33:45.563486+03:00",
+        "query_hash": "9951c38c9cf00e6ee8aecce026b51c19",
+        "query": "select name as \"name::filter\" from users",
+        "runtime": 0.00896096229553223,
+        "data": {
+          "rows": [],
+          "columns": [{
+            "friendly_name": "name::filter",
+            "type": null,
+            "name": "name::filter"
+          }]
+        },
+        "id": 106673,
+        "data_source_id": 1
+      },
+      "status": "done",
+      "filters": [],
+      "filterFreeze": "test@example.com",
+      "updatedAt": "2014-08-05T13:13:40.833Z",
+      "columnNames": ["name::filter"],
+      "filteredData": [{
+        "name::filter": "test@example.com"
+      }],
+      "columns": [{
+        "friendly_name": "name::filter",
+        "type": null,
+        "name": "name::filter"
+      }]
+    }
+  });

rd_ui/test/unit/example_test.js

@@ -0,0 +1,5 @@
+describe('example test', function() {
+  it('should expect the obvious', function() {
+    expect(0).toBe(0);
+  });
+});

rd_ui/test/unit/test_query_view.js

@@ -0,0 +1,34 @@
+'use strict';
+
+describe('QueryViewCtrl', function() {
+  var scope;
+  var MockData;
+
+  beforeEach(module('redash', 'redashMocks'));
+
+  beforeEach(inject(function($injector, $controller, $rootScope, Query, _MockData_) {
+    MockData = _MockData_;
+    scope = $rootScope.$new();
+
+    var route = {
+      current: {
+        locals: {
+          query: new Query(MockData.query)
+        }
+      }
+    };
+
+    $controller('QueryViewCtrl', {$scope: scope, $route: route});
+  }));
+
+  it('should have a query', function() {
+    expect(scope.query).toBeDefined();
+  });
+
+  it('should update the executing state', function() {
+    expect(scope.queryExecuting).toBe(false);
+    scope.executeQuery();
+    expect(scope.queryExecuting).toBe(true);
+  });
+
+});

rd_ui/test/unit/test_visualization_renderer.js

@@ -0,0 +1,89 @@
+'use strict';
+
+describe('VisualizationRenderer', function() {
+  var element;
+  var scope;
+
+  var filters = [{
+    "name": "name::filter",
+    "friendlyName": "Name",
+    "values": ["test@example.com", "amirn@example.com"],
+    "multiple": false
+  }];
+
+  beforeEach(module('redash', 'redashMocks'));
+
+  // loading templates
+  beforeEach(module('app/views/grid_renderer.html',
+    'app/views/visualizations/filters.html'));
+
+  // serving templates
+  beforeEach(inject(function($httpBackend, $templateCache) {
+    $httpBackend.whenGET('/views/grid_renderer.html')
+      .respond($templateCache.get('app/views/grid_renderer.html'));
+
+    $httpBackend.whenGET('/views/visualizations/filters.html')
+      .respond($templateCache.get('app/views/visualizations/filters.html'));
+  }));
+
+  // directive setup
+  beforeEach(inject(function($rootScope, $compile, MockData, QueryResult) {
+    var qr = new QueryResult(MockData.queryResult)
+    qr.filters = filters;
+
+    $rootScope.queryResult = qr;
+
+    element = angular.element(
+      '<visualization-renderer query-result="queryResult">' +
+      '</visualization-renderer>');
+  }));
+
+
+  describe('scope', function() {
+    beforeEach(inject(function($rootScope, $compile) {
+      $compile(element)($rootScope);
+
+      // we will test the isolated scope of the directive
+      scope = element.isolateScope();
+      scope.$digest();
+    }));
+
+    it('should have filters', function() {
+      expect(scope.filters).toBeDefined();
+    });
+  });
+
+
+  /*describe('URL binding', function() {
+
+    beforeEach(inject(function($rootScope, $compile, $location) {
+      spyOn($location, 'search').andCallThrough();
+
+      // set initial search
+      var initialSearch = {};
+      initialSearch[filters[0].friendlyName] = filters[0].values[0];
+      $location.search('filters', initialSearch);
+
+      $compile(element)($rootScope);
+
+      // we will test the isolated scope of the directive
+      scope = element.isolateScope();
+      scope.$digest();
+    }));
+
+    it('should update scope from URL',
+      inject(function($location) {
+        expect($location.search).toHaveBeenCalled();
+        expect(scope.filters[0].current).toEqual(filters[0].values[0]);
+      }));
+
+    it('should update URL from scope',
+      inject(function($location) {
+        scope.filters[0].current = 'newValue';
+        scope.$digest();
+
+        var searchFilters = angular.fromJson($location.search().filters);
+        expect(searchFilters[filters[0].friendlyName]).toEqual('newValue');
+    }));
+  });*/
+});

redash/__init__.py

@@ -1,44 +1,35 @@
-import json
+import logging
 import urlparse
-from flask import Flask, make_response
-from flask.ext.restful import Api
-from flask_peewee.db import Database
-
 import redis
-from redash import settings, utils
+from statsd import StatsClient
 
-__version__ = '0.3.3'
+from redash import settings, events
 
-app = Flask(__name__,
-            template_folder=settings.STATIC_ASSETS_PATH,
-            static_folder=settings.STATIC_ASSETS_PATH,
-            static_path='/static')
-
-api = Api(app)
-
-# configure our database
-settings.DATABASE_CONFIG.update({'threadlocals': True})
-app.config['DATABASE'] = settings.DATABASE_CONFIG
-db = Database(app)
-
-from redash.authentication import setup_authentication
-auth = setup_authentication(app)
-
-@api.representation('application/json')
-def json_representation(data, code, headers=None):
-    resp = make_response(json.dumps(data, cls=utils.JSONEncoder), code)
-    resp.headers.extend(headers or {})
-    return resp
+__version__ = '0.4.0'
 
 
-redis_url = urlparse.urlparse(settings.REDIS_URL)
-if redis_url.path:
-    redis_db = redis_url.path[1]
-else:
-    redis_db = 0
-redis_connection = redis.StrictRedis(host=redis_url.hostname, port=redis_url.port, db=redis_db, password=redis_url.password)
+def setup_logging():
+    handler = logging.StreamHandler()
+    formatter = logging.Formatter('[%(asctime)s][PID:%(process)d][%(levelname)s][%(name)s] %(message)s')
+    handler.setFormatter(formatter)
+    logging.getLogger().addHandler(handler)
+    logging.getLogger().setLevel(settings.LOG_LEVEL)
 
-from redash import data
-data_manager = data.Manager(redis_connection, db)
+    events.setup_logging(settings.EVENTS_LOG_PATH, settings.EVENTS_CONSOLE_OUTPUT)
 
-from redash import controllers
+
+def create_redis_connection():
+    redis_url = urlparse.urlparse(settings.REDIS_URL)
+    if redis_url.path:
+        redis_db = redis_url.path[1]
+    else:
+        redis_db = 0
+
+    r = redis.StrictRedis(host=redis_url.hostname, port=redis_url.port, db=redis_db, password=redis_url.password)
+
+    return r
+
+
+setup_logging()
+redis_connection = create_redis_connection()
+statsd_client = StatsClient(host=settings.STATSD_HOST, port=settings.STATSD_PORT, prefix=settings.STATSD_PREFIX)

redash/authentication.py

@@ -1,12 +1,19 @@
 import functools
 import hashlib
 import hmac
-from flask import current_app, request, make_response, g, redirect, url_for
-from flask.ext.googleauth import GoogleAuth, login
 import time
+import logging
+
+from flask import request, make_response, redirect, url_for
+from flask.ext.login import LoginManager, login_user, current_user
+from flask.ext.googleauth import GoogleAuth, login
 from werkzeug.contrib.fixers import ProxyFix
+
 from redash import models, settings
 
+login_manager = LoginManager()
+logger = logging.getLogger('authentication')
+
 
 def sign(key, path, expires):
     if not key:
@@ -19,9 +26,6 @@ def sign(key, path, expires):
 
 
 class HMACAuthentication(object):
-    def __init__(self, auth):
-        self.auth = auth
-
     @staticmethod
     def api_key_authentication():
         signature = request.args.get('signature')
@@ -34,65 +38,67 @@ class HMACAuthentication(object):
             calculated_signature = sign(query.api_key, request.path, expires)
 
             if query.api_key and signature == calculated_signature:
+                login_user(models.ApiUser(query.api_key), remember=False)
                 return True
 
         return False
 
-    @staticmethod
-    def is_user_logged_in():
-        return g.user is not None
-
-    @staticmethod
-    def valid_user():
-        email = g.user['email']
-        if not settings.GOOGLE_APPS_DOMAIN:
-            return True
-
-        return email in settings.ALLOWED_EXTERNAL_USERS or email.endswith("@%s" % settings.GOOGLE_APPS_DOMAIN)
-
     def required(self, fn):
         @functools.wraps(fn)
         def decorated(*args, **kwargs):
-            if self.is_user_logged_in() and self.valid_user():
+            if current_user.is_authenticated():
                 return fn(*args, **kwargs)
 
             if self.api_key_authentication():
                 return fn(*args, **kwargs)
 
-            blueprint = current_app.extensions['googleauth'].blueprint
-            # The make_response call is a work around for flask-restful testing only for
-            # flask.wrappers.Resource instead of werkzeug.wrappers.Response
-            return make_response(redirect(url_for("%s.login" % blueprint.name, next=request.url)))
+            return make_response(redirect(url_for("login", next=request.url)))
 
         return decorated
 
 
-def create_user(_, user):
+def validate_email(email):
+    if not settings.GOOGLE_APPS_DOMAIN:
+        return True
+
+    return email in settings.ALLOWED_EXTERNAL_USERS or email.endswith("@%s" % settings.GOOGLE_APPS_DOMAIN)
+
+
+def create_and_login_user(app, user):
+    if not validate_email(user.email):
+        return
+
     try:
-        u = models.User.get(models.User.email == user.email)
-        if u.name != user.name:
-            current_app.logger.debug("Updating user name (%r -> %r)", u.name, user.name)
-            u.name = user.name
-            u.save()
+        user_object = models.User.get(models.User.email == user.email)
+        if user_object.name != user.name:
+            logger.debug("Updating user name (%r -> %r)", user_object.name, user.name)
+            user_object.name = user.name
+            user_object.save()
     except models.User.DoesNotExist:
-        current_app.logger.debug("Creating user object (%r)", user.name)
-        u = models.User(name=user.name, email=user.email)
-        u.save()
+        logger.debug("Creating user object (%r)", user.name)
+        user_object = models.User.create(name=user.name, email=user.email, groups = models.User.DEFAULT_GROUPS)
 
-    user['id'] = u.id
-    user['is_admin'] = u.is_admin
+    login_user(user_object, remember=True)
+
+login.connect(create_and_login_user)
 
 
-login.connect(create_user)
+@login_manager.user_loader
+def load_user(user_id):
+    return models.User.select().where(models.User.id == user_id).first()
 
 
 def setup_authentication(app):
-    openid_auth = GoogleAuth(app)
-    # If we don't have a list of external users, we can use Google's federated login, which limits
-    # the domain with which you can sign in.
-    if not settings.ALLOWED_EXTERNAL_USERS and settings.GOOGLE_APPS_DOMAIN:
-        openid_auth._OPENID_ENDPOINT = "https://www.google.com/a/%s/o8/ud?be=o8" % settings.GOOGLE_APPS_DOMAIN
+    if settings.GOOGLE_OPENID_ENABLED:
+        openid_auth = GoogleAuth(app, url_prefix="/google_auth")
+        # If we don't have a list of external users, we can use Google's federated login, which limits
+        # the domain with which you can sign in.
+        if not settings.ALLOWED_EXTERNAL_USERS and settings.GOOGLE_APPS_DOMAIN:
+            openid_auth._OPENID_ENDPOINT = "https://www.google.com/a/%s/o8/ud?be=o8" % settings.GOOGLE_APPS_DOMAIN
+
+    login_manager.init_app(app)
+    login_manager.anonymous_user = models.AnonymousUser
     app.wsgi_app = ProxyFix(app.wsgi_app)
     app.secret_key = settings.COOKIE_SECRET
 
-    return HMACAuthentication(openid_auth)
+    return HMACAuthentication()

redash/cache.py

@@ -0,0 +1,8 @@
+from flask import make_response
+from functools import update_wrapper
+
+ONE_YEAR = 60 * 60 * 24 * 365.25
+
+headers = {
+  'Cache-Control': 'max-age=%d' % ONE_YEAR
+}

redash/controllers.py

@@ -11,16 +11,22 @@ import numbers
 import cStringIO
 import datetime
 
-from flask import g, render_template, send_from_directory, make_response, request, jsonify
+from flask import render_template, send_from_directory, make_response, request, jsonify, redirect, \
+    session, url_for
 from flask.ext.restful import Resource, abort
+from flask_login import current_user, login_user, logout_user
 
 import sqlparse
-from redash import settings, utils
-from redash import data
+import events
+from permissions import require_permission
 
-from redash import app, auth, api, redis_connection, data_manager
-from redash import models
+from redash import redis_connection, statsd_client, models, settings, utils, __version__
+from redash.wsgi import app, auth, api
 
+import logging
+from tasks import QueryTask
+
+from cache import headers as cache_headers
 
 @app.route('/ping', methods=['GET'])
 def ping():
@@ -30,42 +36,96 @@ def ping():
 @app.route('/admin/<anything>')
 @app.route('/dashboard/<anything>')
 @app.route('/queries')
-@app.route('/queries/<anything>')
+@app.route('/queries/<query_id>')
+@app.route('/queries/<query_id>/<anything>')
 @app.route('/')
 @auth.required
-def index(anything=None):
-    email_md5 = hashlib.md5(g.user['email'].lower()).hexdigest()
+def index(**kwargs):
+    email_md5 = hashlib.md5(current_user.email.lower()).hexdigest()
     gravatar_url = "https://www.gravatar.com/avatar/%s?s=40" % email_md5
 
     user = {
         'gravatar_url': gravatar_url,
-        'is_admin': g.user['is_admin'],
-        'id': g.user['id'],
-        'name': g.user['name'],
-        'email': g.user['email']
+        'id': current_user.id,
+        'name': current_user.name,
+        'email': current_user.email,
+        'groups': current_user.groups,
+        'permissions': current_user.permissions
     }
 
-    return render_template("index.html", user=json.dumps(user), analytics=settings.ANALYTICS)
+    features = {
+        'clientSideMetrics': settings.CLIENT_SIDE_METRICS,
+        'flowerUrl': settings.CELERY_FLOWER_URL
+    }
 
+    return render_template("index.html", user=json.dumps(user), name=settings.NAME,
+                           features=json.dumps(features),
+                           analytics=settings.ANALYTICS)
+
+
+@app.route('/login', methods=['GET', 'POST'])
+def login():
+    if current_user.is_authenticated():
+        return redirect(request.args.get('next') or '/')
+
+    if not settings.PASSWORD_LOGIN_ENABLED:
+        blueprint = app.extensions['googleauth'].blueprint
+        return redirect(url_for("%s.login" % blueprint.name, next=request.args.get('next')))
+
+    if request.method == 'POST':
+        user = models.User.select().where(models.User.email == request.form['username']).first()
+        if user and user.verify_password(request.form['password']):
+            remember = ('remember' in request.form)
+            login_user(user, remember=remember)
+            return redirect(request.args.get('next') or '/')
+
+    return render_template("login.html",
+                           name=settings.NAME,
+                           analytics=settings.ANALYTICS,
+                           next=request.args.get('next'),
+                           username=request.form.get('username', ''),
+                           show_google_openid=settings.GOOGLE_OPENID_ENABLED)
+
+
+@app.route('/logout')
+def logout():
+    logout_user()
+    session.pop('openid', None)
+
+    return redirect('/login')
 
 @app.route('/status.json')
 @auth.required
+@require_permission('admin')
 def status_api():
     status = {}
     info = redis_connection.info()
     status['redis_used_memory'] = info['used_memory_human']
-
+    status['version'] = __version__
     status['queries_count'] = models.Query.select().count()
     status['query_results_count'] = models.QueryResult.select().count()
     status['dashboards_count'] = models.Dashboard.select().count()
     status['widgets_count'] = models.Widget.select().count()
 
-    status['workers'] = [redis_connection.hgetall(w)
-                         for w in redis_connection.smembers('workers')]
+    status['workers'] = []
 
-    manager_status = redis_connection.hgetall('manager:status')
+    manager_status = redis_connection.hgetall('redash:status')
     status['manager'] = manager_status
-    status['manager']['queue_size'] = redis_connection.zcard('jobs')
+    status['manager']['queue_size'] = redis_connection.llen('queries') + redis_connection.llen('scheduled_queries')
+    status['manager']['outdated_queries_count'] = models.Query.outdated_queries().count()
+
+    queues = {}
+    for ds in models.DataSource.select():
+        for queue in (ds.queue_name, ds.scheduled_queue_name):
+            queues.setdefault(queue, set())
+            queues[queue].add(ds.name)
+
+    status['manager']['queues'] = {}
+    for queue, sources in queues.iteritems():
+        status['manager']['queues'][queue] = {
+            'data_sources': ', '.join(sources),
+            'size': redis_connection.llen(queue)
+        }
 
     return jsonify(status)
 
@@ -88,10 +148,41 @@ class BaseResource(Resource):
 
     @property
     def current_user(self):
-        if not self._user:
-            self._user = models.User(id=g.user['id'], email=g.user['email'], name=g.user['name'],
-                                     is_admin=g.user['is_admin'])
-        return self._user
+        return current_user._get_current_object()
+
+    def dispatch_request(self, *args, **kwargs):
+        with statsd_client.timer('requests.{}.{}'.format(request.endpoint, request.method.lower())):
+            response = super(BaseResource, self).dispatch_request(*args, **kwargs)
+        return response
+
+
+class EventAPI(BaseResource):
+    def post(self):
+        events_list = request.get_json(force=True)
+        for event in events_list:
+            events.record_event(event)
+
+
+api.add_resource(EventAPI, '/api/events', endpoint='events')
+
+
+class MetricsAPI(BaseResource):
+    def post(self):
+        for stat_line in request.data.split():
+            stat, value = stat_line.split(':')
+            statsd_client._send_stat('client.{}'.format(stat), value, 1)
+
+        return "OK."
+
+api.add_resource(MetricsAPI, '/api/metrics/v1/send', endpoint='metrics')
+
+
+class DataSourceListAPI(BaseResource):
+    def get(self):
+        data_sources = [ds.to_dict() for ds in models.DataSource.all()]
+        return data_sources
+
+api.add_resource(DataSourceListAPI, '/api/data_sources', endpoint='data_sources')
 
 
 class DashboardListAPI(BaseResource):
@@ -101,6 +192,7 @@ class DashboardListAPI(BaseResource):
 
         return dashboards
 
+    @require_permission('create_dashboard')
     def post(self):
         dashboard_properties = request.get_json(force=True)
         dashboard = models.Dashboard(name=dashboard_properties['name'],
@@ -119,6 +211,7 @@ class DashboardAPI(BaseResource):
 
         return dashboard.to_dict(with_widgets=True)
 
+    @require_permission('edit_dashboard')
     def post(self, dashboard_slug):
         dashboard_properties = request.get_json(force=True)
         # TODO: either convert all requests to use slugs or ids
@@ -129,6 +222,7 @@ class DashboardAPI(BaseResource):
 
         return dashboard.to_dict(with_widgets=True)
 
+    @require_permission('edit_dashboard')
     def delete(self, dashboard_slug):
         dashboard = models.Dashboard.get_by_slug(dashboard_slug)
         dashboard.is_archived = True
@@ -139,6 +233,7 @@ api.add_resource(DashboardAPI, '/api/dashboards/<dashboard_slug>', endpoint='das
 
 
 class WidgetListAPI(BaseResource):
+    @require_permission('edit_dashboard')
     def post(self):
         widget_properties = request.get_json(force=True)
         widget_properties['options'] = json.dumps(widget_properties['options'])
@@ -170,6 +265,7 @@ class WidgetListAPI(BaseResource):
 
 
 class WidgetAPI(BaseResource):
+    @require_permission('edit_dashboard')
     def delete(self, widget_id):
         widget = models.Widget.get(models.Widget.id == widget_id)
         # TODO: reposition existing ones
@@ -186,25 +282,28 @@ api.add_resource(WidgetAPI, '/api/widgets/<int:widget_id>', endpoint='widget')
 
 
 class QueryListAPI(BaseResource):
+    @require_permission('create_query')
     def post(self):
         query_def = request.get_json(force=True)
-        # id, created_at, api_key
         for field in ['id', 'created_at', 'api_key', 'visualizations', 'latest_query_data']:
             query_def.pop(field, None)
 
         query_def['user'] = self.current_user
+        query_def['data_source'] = query_def.pop('data_source_id')
         query = models.Query(**query_def)
         query.save()
 
         query.create_default_visualizations()
 
-        return query.to_dict(with_result=False)
+        return query.to_dict()
 
+    @require_permission('view_query')
     def get(self):
-        return [q.to_dict(with_result=False, with_stats=True) for q in models.Query.all_queries()]
+        return [q.to_dict(with_stats=True) for q in models.Query.all_queries()]
 
 
 class QueryAPI(BaseResource):
+    @require_permission('edit_query')
     def post(self, query_id):
         query_def = request.get_json(force=True)
         for field in ['id', 'created_at', 'api_key', 'visualizations', 'latest_query_data', 'user']:
@@ -213,12 +312,16 @@ class QueryAPI(BaseResource):
         if 'latest_query_data_id' in query_def:
             query_def['latest_query_data'] = query_def.pop('latest_query_data_id')
 
+        if 'data_source_id' in query_def:
+            query_def['data_source'] = query_def.pop('data_source_id')
+
         models.Query.update_instance(query_id, **query_def)
-        
+
         query = models.Query.get_by_id(query_id)
 
-        return query.to_dict(with_result=False, with_visualizations=True)
+        return query.to_dict(with_visualizations=True)
 
+    @require_permission('view_query')
     def get(self, query_id):
         q = models.Query.get(models.Query.id == query_id)
         if q:
@@ -231,6 +334,7 @@ api.add_resource(QueryAPI, '/api/queries/<query_id>', endpoint='query')
 
 
 class VisualizationListAPI(BaseResource):
+    @require_permission('edit_query')
     def post(self):
         kwargs = request.get_json(force=True)
         kwargs['options'] = json.dumps(kwargs['options'])
@@ -243,11 +347,13 @@ class VisualizationListAPI(BaseResource):
 
 
 class VisualizationAPI(BaseResource):
+    @require_permission('edit_query')
     def post(self, visualization_id):
         kwargs = request.get_json(force=True)
         if 'options' in kwargs:
             kwargs['options'] = json.dumps(kwargs['options'])
         kwargs.pop('id', None)
+        kwargs.pop('query_id', None)
 
         update = models.Visualization.update(**kwargs).where(models.Visualization.id == visualization_id)
         update.execute()
@@ -256,6 +362,7 @@ class VisualizationAPI(BaseResource):
 
         return vis.to_dict(with_query=False)
 
+    @require_permission('edit_query')
     def delete(self, visualization_id):
         vis = models.Visualization.get(models.Visualization.id == visualization_id)
         vis.delete_instance()
@@ -265,38 +372,67 @@ api.add_resource(VisualizationAPI, '/api/visualizations/<visualization_id>', end
 
 
 class QueryResultListAPI(BaseResource):
+    @require_permission('execute_query')
     def post(self):
         params = request.json
 
+        if settings.FEATURE_TABLES_PERMISSIONS:
+            metadata = utils.SQLMetaData(params['query'])
+
+            if metadata.has_non_select_dml_statements or metadata.has_ddl_statements:
+                return {
+                    'job': {
+                        'error': 'Only SELECT statements are allowed'
+                    }
+                }
+
+            if len(metadata.used_tables - current_user.allowed_tables) > 0 and '*' not in current_user.allowed_tables:
+                logging.warning('Permission denied for user %s to table %s', self.current_user.name, metadata.used_tables)
+                return {
+                    'job': {
+                        'error': 'Access denied for table(s): %s' % (metadata.used_tables)
+                    }
+                }
+
+        models.ActivityLog(
+            user=self.current_user,
+            type=models.ActivityLog.QUERY_EXECUTION,
+            activity=params['query']
+        ).save()
+
         if params['ttl'] == 0:
             query_result = None
         else:
-            query_result = data_manager.get_query_result(params['query'], int(params['ttl']))
+            query_result = models.QueryResult.get_latest(params['data_source_id'], params['query'], int(params['ttl']))
 
         if query_result:
-            return {'query_result': query_result.to_dict(parse_data=True)}
+            return {'query_result': query_result.to_dict()}
         else:
-            job = data_manager.add_job(params['query'], data.Job.HIGH_PRIORITY)
+            data_source = models.DataSource.get_by_id(params['data_source_id'])
+            job = QueryTask.add_task(params['query'], data_source)
             return {'job': job.to_dict()}
 
 
 class QueryResultAPI(BaseResource):
+    @require_permission('view_query')
     def get(self, query_result_id):
-        query_result = data_manager.get_query_result_by_id(query_result_id)
+        query_result = models.QueryResult.get_by_id(query_result_id)
         if query_result:
-            return {'query_result': query_result.to_dict(parse_data=True)}
+            data = json.dumps({'query_result': query_result.to_dict()}, cls=utils.JSONEncoder)
+            return make_response(data, 200, cache_headers)
         else:
             abort(404)
 
 
 class CsvQueryResultsAPI(BaseResource):
+    @require_permission('view_query')
     def get(self, query_id, query_result_id=None):
         if not query_result_id:
             query = models.Query.get(models.Query.id == query_id)
             if query:
                 query_result_id = query._data['latest_query_data']
 
-        query_result = query_result_id and data_manager.get_query_result_by_id(query_result_id)
+        query_result = query_result_id and models.QueryResult.get_by_id(query_result_id)
         if query_result:
             s = cStringIO.StringIO()
 
@@ -325,17 +461,16 @@ api.add_resource(QueryResultAPI, '/api/query_results/<query_result_id>', endpoin
 class JobAPI(BaseResource):
     def get(self, job_id):
         # TODO: if finished, include the query result
-        job = data.Job.load(data_manager.redis_connection, job_id)
+        job = QueryTask(job_id=job_id)
         return {'job': job.to_dict()}
 
     def delete(self, job_id):
-        job = data.Job.load(data_manager.redis_connection, job_id)
+        job = QueryTask(job_id=job_id)
         job.cancel()
 
 api.add_resource(JobAPI, '/api/jobs/<job_id>', endpoint='job')
 
 @app.route('/<path:filename>')
-@auth.required
 def send_static(filename):
     return send_from_directory(settings.STATIC_ASSETS_PATH, filename)
 

redash/data/__init__.py

@@ -1,2 +0,0 @@
-from manager import Manager
-from worker import Job

redash/data/manager.py

@@ -1,200 +0,0 @@
-"""
-Data manager. Used to manage and coordinate execution of queries.
-"""
-from contextlib import contextmanager
-import collections
-import json
-import time
-import logging
-import psycopg2
-import qr
-import redis
-from redash import settings
-from redash.data import worker
-from redash.utils import gen_query_hash
-
-class QueryResult(collections.namedtuple('QueryData', 'id query data runtime retrieved_at query_hash')):
-    def to_dict(self, parse_data=False):
-        d = self._asdict()
-
-        if parse_data and d['data']:
-            d['data'] = json.loads(d['data'])
-
-        return d
-
-
-class Manager(object):
-    def __init__(self, redis_connection, db):
-        self.redis_connection = redis_connection
-        self.db = db
-        self.workers = []
-        self.queue = qr.PriorityQueue("jobs", **self.redis_connection.connection_pool.connection_kwargs)
-        self.max_retries = 5
-        self.status = {
-            'last_refresh_at': 0,
-            'started_at': time.time()
-        }
-
-        self._save_status()
-
-    # TODO: Use our Django Models
-    def get_query_result_by_id(self, query_result_id):
-        with self.db_transaction() as cursor:
-            sql = "SELECT id, query, data, runtime, retrieved_at, query_hash FROM query_results " \
-                  "WHERE id=%s LIMIT 1"
-            cursor.execute(sql, (query_result_id,))
-            query_result = cursor.fetchone()
-
-        if query_result:
-            query_result = QueryResult(*query_result)
-
-        return query_result
-
-    def get_query_result(self, query, ttl=0):
-        query_hash = gen_query_hash(query)
-
-        with self.db_transaction() as cursor:
-            sql = "SELECT id, query, data, runtime, retrieved_at, query_hash FROM query_results " \
-                  "WHERE query_hash=%s " \
-                  "AND retrieved_at < now() at time zone 'utc' - interval '%s second'" \
-                  "ORDER BY retrieved_at DESC LIMIT 1"
-            cursor.execute(sql, (query_hash, psycopg2.extensions.AsIs(ttl)))
-            query_result = cursor.fetchone()
-
-        if query_result:
-            query_result = QueryResult(*query_result)
-
-        return query_result
-
-    def add_job(self, query, priority):
-        query_hash = gen_query_hash(query)
-        logging.info("[Manager][%s] Inserting job with priority=%s", query_hash, priority)
-        try_count = 0
-        job = None
-
-        while try_count < self.max_retries:
-            try_count += 1
-
-            pipe = self.redis_connection.pipeline()
-            try:
-                pipe.watch('query_hash_job:%s' % query_hash)
-                job_id = pipe.get('query_hash_job:%s' % query_hash)
-                if job_id:
-                    logging.info("[Manager][%s] Found existing job: %s", query_hash, job_id)
-                    job = worker.Job.load(self.redis_connection, job_id)
-                else:
-                    job = worker.Job(self.redis_connection, query, priority)
-                    pipe.multi()
-                    job.save(pipe)
-                    logging.info("[Manager][%s] Created new job: %s", query_hash, job.id)
-                    self.queue.push(job.id, job.priority)
-                break
-
-            except redis.WatchError:
-                continue
-
-        if not job:
-            logging.error("[Manager][%s] Failed adding job for query.", query_hash)
-
-        return job
-
-    def refresh_queries(self):
-        sql = """SELECT queries.query, queries.ttl, retrieved_at
-        FROM (SELECT query, min(ttl) as ttl FROM queries  WHERE ttl > 0 GROUP by query) queries
-        JOIN (SELECT query, max(retrieved_at) as retrieved_at
-        FROM query_results
-        GROUP BY query) query_results on query_results.query=queries.query
-        WHERE queries.ttl > 0
-          AND query_results.retrieved_at + ttl * interval '1 second' < now() at time zone 'utc';"""
-
-        self.status['last_refresh_at'] = time.time()
-        self._save_status()
-
-        logging.info("Refreshing queries...")
-        queries = self.run_query(sql)
-        for query, ttl, retrieved_at in queries:
-            self.add_job(query, worker.Job.LOW_PRIORITY)
-
-        logging.info("Done refreshing queries... %d" % len(queries))
-
-    def store_query_result(self, query, data, run_time, retrieved_at):
-        query_result_id = None
-        query_hash = gen_query_hash(query)
-        sql = "INSERT INTO query_results (query_hash, query, data, runtime, retrieved_at) " \
-              "VALUES (%s, %s, %s, %s, %s) RETURNING id"
-        with self.db_transaction() as cursor:
-            cursor.execute(sql, (query_hash, query, data, run_time, retrieved_at))
-            if cursor.rowcount == 1:
-                query_result_id = cursor.fetchone()[0]
-                logging.info("[Manager][%s] Inserted query data; id=%s", query_hash, query_result_id)
-
-                sql = "UPDATE queries SET latest_query_data_id=%s WHERE query_hash=%s"
-                cursor.execute(sql, (query_result_id, query_hash))
-
-                logging.info("[Manager][%s] Updated %s queries.", query_hash, cursor.rowcount)
-            else:
-                logging.error("[Manager][%s] Failed inserting query data.", query_hash)
-        return query_result_id
-
-    def run_query(self, *args):
-        sql = args[0]
-        logging.debug("running query: %s %s", sql, args[1:])
-
-        with self.db_transaction() as cursor:
-            cursor.execute(sql, args[1:])
-            if cursor.description:
-                data = list(cursor)
-            else:
-                data = cursor.rowcount
-
-        return data
-
-    def start_workers(self, workers_count, connection_type, connection_string):
-        if self.workers:
-            return self.workers
-        
-        if connection_type == 'mysql':
-            from redash.data import query_runner_mysql
-            runner = query_runner_mysql.mysql(connection_string)
-        elif connection_type == 'graphite':
-            from redash.data import query_runner_graphite
-            connection_params = json.loads(connection_string)
-            if connection_params['auth']:
-                connection_params['auth'] = tuple(connection_params['auth'])
-            else:
-                connection_params['auth'] = None
-            runner = query_runner_graphite.graphite(connection_params)
-        else:
-            from redash.data import query_runner
-            runner = query_runner.redshift(connection_string)
-
-        redis_connection_params = self.redis_connection.connection_pool.connection_kwargs
-        self.workers = [worker.Worker(self, redis_connection_params, runner)
-                        for _ in range(workers_count)]
-        for w in self.workers:
-            w.start()
-
-        return self.workers
-
-    def stop_workers(self):
-        for w in self.workers:
-            w.continue_working = False
-            w.join()
-
-    @contextmanager
-    def db_transaction(self):
-        self.db.connect_db()
-
-        cursor = self.db.database.get_cursor()
-        try:
-            yield cursor
-        except:
-            self.db.database.rollback()
-            raise
-        else:
-            self.db.database.commit()
-        finally:
-            self.db.close_db(None)
-
-    def _save_status(self):
-        self.redis_connection.hmset('manager:status', self.status)

redash/data/query_runner.py

@@ -1,69 +1,30 @@
-"""
-QueryRunner is the function that the workers use, to execute queries. This is the Redshift
-(PostgreSQL in fact) version, but easily we can write another to support additional databases
-(MySQL and others).
-
-Because the worker just pass the query, this can be used with any data store that has some sort of
-query language (for example: HiveQL).
-"""
-import logging
 import json
-import sys
-import select
 
-import psycopg2
 
-from redash.utils import JSONEncoder
+def get_query_runner(connection_type, connection_string):
+    if connection_type == 'mysql':
+        from redash.data import query_runner_mysql
+        runner = query_runner_mysql.mysql(connection_string)
+    elif connection_type == 'graphite':
+        from redash.data import query_runner_graphite
+        connection_params = json.loads(connection_string)
+        if connection_params['auth']:
+            connection_params['auth'] = tuple(connection_params['auth'])
+        else:
+            connection_params['auth'] = None
+        runner = query_runner_graphite.graphite(connection_params)
+    elif connection_type == 'bigquery':
+        from redash.data import query_runner_bigquery
+        connection_params = json.loads(connection_string)
+        runner = query_runner_bigquery.bigquery(connection_params)
+    elif connection_type == 'script':
+        from redash.data import query_runner_script
+        runner = query_runner_script.script(connection_string)
+    elif connection_type == 'url':
+        from redash.data import query_runner_url
+        runner = query_runner_url.url(connection_string)
+    else:
+        from redash.data import query_runner_pg
+        runner = query_runner_pg.pg(connection_string)
 
-def redshift(connection_string):
-    def column_friendly_name(column_name):
-        return column_name
- 
-    def wait(conn):
-        while 1:
-            state = conn.poll()
-            if state == psycopg2.extensions.POLL_OK:
-                break
-            elif state == psycopg2.extensions.POLL_WRITE:
-                select.select([], [conn.fileno()], [])
-            elif state == psycopg2.extensions.POLL_READ:
-                select.select([conn.fileno()], [], [])
-            else:
-                raise psycopg2.OperationalError("poll() returned %s" % state)
- 
-    def query_runner(query):
-        connection = psycopg2.connect(connection_string, async=True)
-        wait(connection)
- 
-        cursor = connection.cursor()
- 
-        try:
-            cursor.execute(query)
-            wait(connection)
- 
-            column_names = [col.name for col in cursor.description]
- 
-            rows = [dict(zip(column_names, row)) for row in cursor]
-            columns = [{'name': col.name,
-                        'friendly_name': column_friendly_name(col.name),
-                        'type': None} for col in cursor.description]
- 
-            data = {'columns': columns, 'rows': rows}
-            json_data = json.dumps(data, cls=JSONEncoder)
-            error = None
-            cursor.close()
-        except psycopg2.DatabaseError as e:
-            json_data = None
-            error = e.message
-        except KeyboardInterrupt:
-            connection.cancel()
-            error = "Query cancelled by user."
-            json_data = None
-        except Exception as e:
-            raise sys.exc_info()[1], None, sys.exc_info()[2]
-        finally:
-            connection.close()
- 
-        return json_data, error
- 
-    return query_runner
+    return runner

redash/data/query_runner_bigquery.py

@@ -0,0 +1,138 @@
+import datetime
+import httplib2
+import json
+import logging
+import sys
+import time
+
+try:
+    import apiclient.errors
+    from apiclient.discovery import build
+    from apiclient.errors import HttpError
+    from oauth2client.client import SignedJwtAssertionCredentials
+except ImportError:
+    print "Missing dependencies. Please install google-api-python-client and oauth2client."
+    print "You can use pip:   pip install google-api-python-client oauth2client"
+
+from redash.utils import JSONEncoder
+
+types_map = {
+    'INTEGER': 'integer',
+    'FLOAT': 'float',
+    'BOOLEAN': 'boolean',
+    'STRING': 'string',
+    'TIMESTAMP': 'datetime',
+}
+
+def transform_row(row, fields):
+    column_index = 0
+    row_data = {}
+
+    for cell in row["f"]:
+        field = fields[column_index]
+        cell_value = cell['v']
+
+        if cell_value is None:
+            pass
+        # Otherwise just cast the value
+        elif field['type'] == 'INTEGER':
+            cell_value = int(cell_value)
+        elif field['type'] == 'FLOAT':
+            cell_value = float(cell_value)
+        elif field['type'] == 'BOOLEAN':
+            cell_value = cell_value.lower() == "true"
+        elif field['type'] == 'TIMESTAMP':
+            cell_value = datetime.datetime.fromtimestamp(float(cell_value))
+
+        row_data[field["name"]] = cell_value
+        column_index += 1
+
+    return row_data
+
+def bigquery(connection_string):
+    def load_key(filename):
+        f = file(filename, "rb")
+        try:
+            return f.read()
+        finally:
+            f.close()
+
+    def get_bigquery_service():
+        scope = [
+            "https://www.googleapis.com/auth/bigquery",
+        ]
+
+        credentials = SignedJwtAssertionCredentials(connection_string["serviceAccount"],
+                                                    load_key(connection_string["privateKey"]), scope=scope)
+        http = httplib2.Http()
+        http = credentials.authorize(http)
+
+        return build("bigquery", "v2", http=http)
+
+    def get_query_results(jobs, project_id, job_id, start_index):
+        query_reply = jobs.getQueryResults(projectId=project_id, jobId=job_id, startIndex=start_index).execute()
+        logging.debug('query_reply %s', query_reply)
+        if not query_reply['jobComplete']:
+            time.sleep(10)
+            return get_query_results(jobs, project_id, job_id, start_index)
+
+        return query_reply
+
+    def query_runner(query):
+        bigquery_service = get_bigquery_service()
+
+        jobs = bigquery_service.jobs()
+        job_data = {
+            "configuration": {
+                "query": {
+                    "query": query,
+                }
+            }
+        }
+
+        logging.debug("bigquery got query: %s", query)
+
+        project_id = connection_string["projectId"]
+
+        try:
+            insert_response = jobs.insert(projectId=project_id, body=job_data).execute()
+            current_row = 0
+            query_reply = get_query_results(jobs, project_id=project_id,
+                                            job_id=insert_response['jobReference']['jobId'], start_index=current_row)
+
+            logging.debug("bigquery replied: %s", query_reply)
+
+            rows = []
+
+            while ("rows" in query_reply) and current_row < query_reply['totalRows']:
+                for row in query_reply["rows"]:
+                    rows.append(transform_row(row, query_reply["schema"]["fields"]))
+
+                current_row += len(query_reply['rows'])
+                query_reply = jobs.getQueryResults(projectId=project_id, jobId=query_reply['jobReference']['jobId'],
+                                                   startIndex=current_row).execute()
+
+            columns = [{'name': f["name"],
+                        'friendly_name': f["name"],
+                        'type': types_map.get(f['type'], "string")} for f in query_reply["schema"]["fields"]]
+
+            data = {
+                "columns": columns,
+                "rows": rows
+            }
+            error = None
+
+            json_data = json.dumps(data, cls=JSONEncoder)
+        except apiclient.errors.HttpError, e:
+            json_data = None
+            error = e.content
+        except KeyboardInterrupt:
+            error = "Query cancelled by user."
+            json_data = None
+        except Exception:
+            raise sys.exc_info()[1], None, sys.exc_info()[2]
+
+        return json_data, error
+
+
+    return query_runner

redash/data/query_runner_mysql.py

@@ -28,18 +28,24 @@ def mysql(connection_string):
             
             data = cursor.fetchall()
             
-            num_fields = len(cursor.description)
-            column_names = [i[0] for i in cursor.description]
+            cursor_desc = cursor.description
+            if (cursor_desc != None):
+                num_fields = len(cursor_desc)
+                column_names = [i[0] for i in cursor.description]
             
-            rows = [dict(zip(column_names, row)) for row in data]
+                rows = [dict(zip(column_names, row)) for row in data]
 
-            columns = [{'name': col_name,
-                        'friendly_name': col_name,
-                        'type': None} for col_name in column_names]
+                columns = [{'name': col_name,
+                            'friendly_name': col_name,
+                            'type': None} for col_name in column_names]
             
-            data = {'columns': columns, 'rows': rows}
-            json_data = json.dumps(data, cls=JSONEncoder)
-            error = None
+                data = {'columns': columns, 'rows': rows}
+                json_data = json.dumps(data, cls=JSONEncoder)
+                error = None
+            else:
+                json_data = None
+                error = "No data was returned."
+                
             cursor.close()
         except MySQLdb.Error, e:
             json_data = None

redash/data/query_runner_pg.py

@@ -0,0 +1,110 @@
+"""
+QueryRunner is the function that the workers use, to execute queries. This is the PostgreSQL
+version, but easily we can write another to support additional databases (MySQL and others).
+
+Because the worker just pass the query, this can be used with any data store that has some sort of
+query language (for example: HiveQL).
+"""
+import json
+import sys
+import select
+import logging
+import psycopg2
+
+from redash.utils import JSONEncoder
+
+types_map = {
+    20: 'integer',
+    21: 'integer',
+    23: 'integer',
+    700: 'float',
+    1700: 'float',
+    701: 'float',
+    16: 'boolean',
+    1082: 'date',
+    1114: 'datetime',
+    1184: 'datetime',
+    1014: 'string',
+    1015: 'string',
+    1008: 'string',
+    1009: 'string',
+    2951: 'string'
+}
+
+
+def pg(connection_string):
+    def column_friendly_name(column_name):
+        return column_name
+
+    def wait(conn):
+        while 1:
+            try:
+                state = conn.poll()
+                if state == psycopg2.extensions.POLL_OK:
+                    break
+                elif state == psycopg2.extensions.POLL_WRITE:
+                    select.select([], [conn.fileno()], [])
+                elif state == psycopg2.extensions.POLL_READ:
+                    select.select([conn.fileno()], [], [])
+                else:
+                    raise psycopg2.OperationalError("poll() returned %s" % state)
+            except select.error:
+                raise psycopg2.OperationalError("select.error received")
+
+    def query_runner(query):
+        connection = psycopg2.connect(connection_string, async=True)
+        wait(connection)
+
+        cursor = connection.cursor()
+
+        try:
+            cursor.execute(query)
+            wait(connection)
+
+            # While set would be more efficient here, it sorts the data which is not what we want, but due to the small
+            # size of the data we can assume it's ok.
+            column_names = []
+            columns = []
+            duplicates_counter = 1
+
+            for column in cursor.description:
+                # TODO: this deduplication needs to be generalized and reused in all query runners.
+                column_name = column.name
+                if column_name in column_names:
+                    column_name = column_name + str(duplicates_counter)
+                    duplicates_counter += 1
+
+                column_names.append(column_name)
+
+                columns.append({
+                    'name': column_name,
+                    'friendly_name': column_friendly_name(column_name),
+                    'type': types_map.get(column.type_code, None)
+                })
+
+            rows = [dict(zip(column_names, row)) for row in cursor]
+
+            data = {'columns': columns, 'rows': rows}
+            json_data = json.dumps(data, cls=JSONEncoder)
+            error = None
+            cursor.close()
+        except (select.error, OSError) as e:
+            logging.exception(e)
+            error = "Query interrupted. Please retry."
+            json_data = None
+        except psycopg2.DatabaseError as e:
+            logging.exception(e)
+            json_data = None
+            error = e.message
+        except KeyboardInterrupt:
+            connection.cancel()
+            error = "Query cancelled by user."
+            json_data = None
+        except Exception as e:
+            raise sys.exc_info()[1], None, sys.exc_info()[2]
+        finally:
+            connection.close()
+
+        return json_data, error
+
+    return query_runner

redash/data/query_runner_script.py

@@ -0,0 +1,48 @@
+import json
+import logging
+import sys
+import os
+import subprocess
+
+# We use subprocess.check_output because we are lazy.
+# If someone will really want to run this on Python < 2.7 they can easily update the code to run
+# Popen, check the retcodes and other things and read the standard output to a variable.
+if not "check_output" in subprocess.__dict__:
+    print "ERROR: This runner uses subprocess.check_output function which exists in Python 2.7"
+
+def script(connection_string):
+
+    def query_runner(query):
+        try:
+            json_data = None
+            error = None
+
+            # Poor man's protection against running scripts from output the scripts directory
+            if connection_string.find("../") > -1:
+                return None, "Scripts can only be run from the configured scripts directory"
+
+            query = query.strip()
+
+            script = os.path.join(connection_string, query)
+            if not os.path.exists(script):
+                return None, "Script '%s' not found in script directory" % query
+
+            output = subprocess.check_output(script, shell=False)
+            if output != None:
+                output = output.strip()
+                if output != "":
+                    return output, None
+
+            error = "Error reading output"
+        except subprocess.CalledProcessError as e:
+            return None, str(e)
+        except KeyboardInterrupt:
+            error = "Query cancelled by user."
+            json_data = None
+        except Exception as e:
+            raise sys.exc_info()[1], None, sys.exc_info()[2]
+
+        return json_data, error
+
+    query_runner.annotate_query = False
+    return query_runner

redash/data/query_runner_url.py

@@ -0,0 +1,45 @@
+import json
+import logging
+import sys
+import os
+import urllib2
+
+def url(connection_string):
+
+    def query_runner(query):
+        base_url = connection_string
+
+        try:
+            json_data = None
+            error = None
+
+            query = query.strip()
+
+            if base_url is not None and base_url != "":
+                if query.find("://") > -1:
+                    return None, "Accepting only relative URLs to '%s'" % base_url
+
+            if base_url is None:
+                base_url = ""
+
+            url = base_url + query
+
+            json_data = urllib2.urlopen(url).read().strip()
+
+            if not json_data:
+                error = "Error reading data from '%s'" % url
+
+            return json_data, error
+
+        except urllib2.URLError as e:
+            return None, str(e)
+        except KeyboardInterrupt:
+            error = "Query cancelled by user."
+            json_data = None
+        except Exception as e:
+            raise sys.exc_info()[1], None, sys.exc_info()[2]
+
+        return json_data, error
+
+    query_runner.annotate_query = False
+    return query_runner

redash/data/worker.py

@@ -1,258 +0,0 @@
-"""
-Worker implementation to execute incoming queries.
-"""
-import json
-import logging
-import os
-import threading
-import uuid
-import datetime
-import time
-import signal
-import setproctitle
-import redis
-from redash.utils import gen_query_hash
-
-
-class Job(object):
-    HIGH_PRIORITY = 1
-    LOW_PRIORITY = 2
-
-    WAITING = 1
-    PROCESSING = 2
-    DONE = 3
-    FAILED = 4
-
-    def __init__(self, redis_connection, query, priority,
-                 job_id=None,
-                 wait_time=None, query_time=None,
-                 updated_at=None, status=None, error=None, query_result_id=None,
-                 process_id=0):
-        self.redis_connection = redis_connection
-        self.query = query
-        self.priority = priority
-        self.query_hash = gen_query_hash(self.query)
-        self.query_result_id = query_result_id
-        if process_id == 'None':
-            self.process_id = None
-        else:
-            self.process_id = int(process_id)
-
-        if job_id is None:
-            self.id = str(uuid.uuid1())
-            self.new_job = True
-            self.wait_time = 0
-            self.query_time = 0
-            self.error = None
-            self.updated_at = time.time() # job_dict.get('updated_at', time.time())
-            self.status = self.WAITING # int(job_dict.get('status', self.WAITING))
-        else:
-            self.id = job_id
-            self.new_job = False
-            self.error = error
-            self.wait_time = wait_time
-            self.query_time = query_time
-            self.updated_at = updated_at
-            self.status = status
-
-    def to_dict(self):
-        return {
-            'query': self.query,
-            'priority': self.priority,
-            'id': self.id,
-            'wait_time': self.wait_time,
-            'query_time': self.query_time,
-            'updated_at': self.updated_at,
-            'status': self.status,
-            'error': self.error,
-            'query_result_id': self.query_result_id,
-            'process_id': self.process_id
-        }
-
-    @staticmethod
-    def _redis_key(job_id):
-        return 'job:%s' % job_id
-
-    def cancel(self):
-        # TODO: Race condition:
-        # it's possible that it will be picked up by worker while processing the cancel order
-        if self.is_finished():
-            return
-
-        if self.status == self.PROCESSING:
-            os.kill(self.process_id, signal.SIGINT)
-        else:
-            self.done(None, "Interrupted/Cancelled while running.")
-
-    def save(self, pipe=None):
-        if not pipe:
-            pipe = self.redis_connection.pipeline()
-
-        if self.new_job:
-            pipe.set('query_hash_job:%s' % self.query_hash, self.id)
-
-        if self.is_finished():
-            pipe.delete('query_hash_job:%s' % self.query_hash)
-
-        pipe.sadd('jobs_set', self.id)
-        pipe.hmset(self._redis_key(self.id), self.to_dict())
-        pipe.publish(self._redis_key(self.id), json.dumps(self.to_dict()))
-        pipe.execute()
-
-    def processing(self, process_id):
-        self.status = self.PROCESSING
-        self.process_id = process_id
-        self.wait_time = time.time() - self.updated_at
-        self.updated_at = time.time()
-        self.save()
-
-    def is_finished(self):
-        return self.status in (self.FAILED, self.DONE)
-
-    def done(self, query_result_id, error):
-        if error:
-            self.status = self.FAILED
-        else:
-            self.status = self.DONE
-
-        self.query_result_id = query_result_id
-        self.error = error
-        self.query_time = time.time() - self.updated_at
-        self.updated_at = time.time()
-        self.save()
-
-    def __str__(self):
-        return "<Job:%s,priority:%d,status:%d>" % (self.id, self.priority, self.status)
-
-    @classmethod
-    def _load(cls, redis_connection, job_id):
-        return redis_connection.hgetall(cls._redis_key(job_id))
-
-    @classmethod
-    def load(cls, redis_connection, job_id):
-        job_dict = cls._load(redis_connection, job_id)
-        job = None
-        if job_dict:
-            job = Job(redis_connection, job_id=job_dict['id'], query=job_dict['query'].decode('utf-8'),
-                      priority=int(job_dict['priority']), updated_at=float(job_dict['updated_at']),
-                      status=int(job_dict['status']), wait_time=float(job_dict['wait_time']),
-                      query_time=float(job_dict['query_time']), error=job_dict['error'],
-                      query_result_id=job_dict['query_result_id'],
-                      process_id=job_dict['process_id'])
-
-        return job
-
-
-class Worker(threading.Thread):
-    def __init__(self, manager, redis_connection_params, query_runner, sleep_time=0.1):
-        self.manager = manager
-
-        self.redis_connection_params = {k: v for k, v in redis_connection_params.iteritems()
-                                        if k in ('host', 'db', 'password', 'port')}
-        self.continue_working = True
-        self.query_runner = query_runner
-        self.sleep_time = sleep_time
-        self.child_pid = None
-        self.worker_id = uuid.uuid1()
-        self.status = {
-            'jobs_count': 0,
-            'cancelled_jobs_count': 0,
-            'done_jobs_count': 0,
-            'updated_at': time.time(),
-            'started_at': time.time()
-        }
-        self._save_status()
-        self.manager.redis_connection.sadd('workers', self._key)
-
-        super(Worker, self).__init__(name="Worker-%s" % self.worker_id)
-
-    def set_title(self, title=None):
-        base_title = "redash worker:%s" % self.worker_id
-        if title:
-            full_title = "%s - %s" % (base_title, title)
-        else:
-            full_title = base_title
-
-        setproctitle.setproctitle(full_title)
-
-    def run(self):
-        logging.info("[%s] started.", self.name)
-        while self.continue_working:
-            job_id = self.manager.queue.pop()
-            if job_id:
-                self._update_status('jobs_count')
-                logging.info("[%s] Processing %s", self.name, job_id)
-                self._fork_and_process(job_id)
-                if self.child_pid == 0:
-                    return
-            else:
-                time.sleep(self.sleep_time)
-
-    def _update_status(self, counter):
-        self.status['updated_at'] = time.time()
-        self.status[counter] += 1
-        self._save_status()
-
-    @property
-    def _key(self):
-        return 'worker:%s' % self.worker_id
-
-    def _save_status(self):
-        self.manager.redis_connection.hmset(self._key, self.status)
-
-    def _fork_and_process(self, job_id):
-        self.child_pid = os.fork()
-        if self.child_pid == 0:
-            self.set_title("processing %s" % job_id)
-            self._process(job_id)
-        else:
-            logging.info("[%s] Waiting for pid: %d", self.name, self.child_pid)
-            _, status = os.waitpid(self.child_pid, 0)
-            self._update_status('done_jobs_count')
-            if status > 0:
-                job = Job.load(self.manager.redis_connection, job_id)
-                if not job.is_finished():
-                    self._update_status('cancelled_jobs_count')
-                    logging.info("[%s] process interrupted and job %s hasn't finished; registering interruption in job",
-                                 self.name, job_id)
-                    job.done(None, "Interrupted/Cancelled while running.")
-
-            logging.info("[%s] Finished Processing %s (pid: %d status: %d)",
-                         self.name, job_id, self.child_pid, status)
-
-    def _process(self, job_id):
-        redis_connection = redis.StrictRedis(**self.redis_connection_params)
-        job = Job.load(redis_connection, job_id)
-        if job.is_finished():
-            logging.warning("[%s][%s] tried to process finished job.", self.name, job)
-            return
-
-        pid = os.getpid()
-        job.processing(pid)
-
-        logging.info("[%s][%s] running query...", self.name, job.id)
-        start_time = time.time()
-        self.set_title("running query %s" % job_id)
-
-        if getattr(self.query_runner, 'annotate_query', True):
-            annotated_query = "/* Pid: %s, Job Id: %s, Query hash: %s, Priority: %s */ %s" % \
-                              (pid, job.id, job.query_hash, job.priority, job.query)
-        else:
-            annotated_query = job.query
-
-        # TODO: here's the part that needs to be forked, not all of the worker process...
-        data, error = self.query_runner(annotated_query)
-        run_time = time.time() - start_time
-        logging.info("[%s][%s] query finished... data length=%s, error=%s",
-                     self.name, job.id, data and len(data), error)
-
-        # TODO: it is possible that storing the data will fail, and we will need to retry
-        # while we already marked the job as done
-        query_result_id = None
-        if not error:
-            self.set_title("storing results %s" % job_id)
-            query_result_id = self.manager.store_query_result(job.query, data, run_time,
-                                                              datetime.datetime.utcnow())
-
-        self.set_title("marking job as done %s" % job_id)
-        job.done(query_result_id, error)

redash/events.py

@@ -0,0 +1,23 @@
+import logging
+import json
+
+logger = logging.getLogger("redash.events")
+logger.propagate = False
+
+
+def setup_logging(log_path, console_output=False):
+    if log_path:
+        fh = logging.FileHandler(log_path)
+        formatter = logging.Formatter('%(message)s')
+        fh.setFormatter(formatter)
+        logger.addHandler(fh)
+
+    if console_output:
+        handler = logging.StreamHandler()
+        formatter = logging.Formatter('[%(name)s] %(message)s')
+        handler.setFormatter(formatter)
+        logger.addHandler(handler)
+
+
+def record_event(event):
+    logger.info(json.dumps(event))

redash/import_export.py

@@ -0,0 +1,158 @@
+import contextlib
+import json
+from redash import models
+from flask.ext.script import Manager
+
+
+class Importer(object):
+    def __init__(self, object_mapping=None, data_source=None):
+        if object_mapping is None:
+            object_mapping = {}
+        self.object_mapping = object_mapping
+        self.data_source = data_source
+
+    def import_query_result(self, query_result):
+        query_result = self._get_or_create(models.QueryResult, query_result['id'],
+                                           data_source=self.data_source,
+                                           data=json.dumps(query_result['data']),
+                                           query_hash=query_result['query_hash'],
+                                           retrieved_at=query_result['retrieved_at'],
+                                           query=query_result['query'],
+                                           runtime=query_result['runtime'])
+
+        return query_result
+
+
+    def import_query(self, user, query):
+        query_result = self.import_query_result(query['latest_query_data'])
+
+        new_query = self._get_or_create(models.Query, query['id'], name=query['name'],
+                                        user=user,
+                                        ttl=-1,
+                                        query=query['query'],
+                                        query_hash=query['query_hash'],
+                                        description=query['description'],
+                                        latest_query_data=query_result,
+                                        data_source=self.data_source)
+
+        return new_query
+
+
+    def import_visualization(self, user, visualization):
+        query = self.import_query(user, visualization['query'])
+
+        new_visualization = self._get_or_create(models.Visualization, visualization['id'],
+                                                name=visualization['name'],
+                                                description=visualization['description'],
+                                                type=visualization['type'],
+                                                options=json.dumps(visualization['options']),
+                                                query=query)
+        return new_visualization
+
+    def import_widget(self, dashboard, widget):
+        visualization = self.import_visualization(dashboard.user, widget['visualization'])
+
+        new_widget = self._get_or_create(models.Widget, widget['id'],
+                                         dashboard=dashboard,
+                                         width=widget['width'],
+                                         options=json.dumps(widget['options']),
+                                         visualization=visualization)
+
+        return new_widget
+
+    def import_dashboard(self, user, dashboard):
+        """
+        Imports dashboard along with widgets, visualizations and queries from another re:dash.
+
+        user - the user to associate all objects with.
+        dashboard - dashboard to import (can be result of loading a json output).
+        """
+
+        new_dashboard = self._get_or_create(models.Dashboard, dashboard['id'],
+                                            name=dashboard['name'],
+                                            slug=dashboard['slug'],
+                                            layout='[]',
+                                            user=user)
+
+        layout = []
+
+        for widgets in dashboard['widgets']:
+            row = []
+            for widget in widgets:
+                widget_id = self.import_widget(new_dashboard, widget).id
+                row.append(widget_id)
+
+            layout.append(row)
+
+        new_dashboard.layout = json.dumps(layout)
+        new_dashboard.save()
+
+        return new_dashboard
+
+    def _get_or_create(self, object_type, external_id, **properties):
+        internal_id = self._get_mapping(object_type, external_id)
+        if internal_id:
+            update = object_type.update(**properties).where(object_type.id == internal_id)
+            update.execute()
+            obj = object_type.get_by_id(internal_id)
+        else:
+            obj = object_type.create(**properties)
+            self._update_mapping(object_type, external_id, obj.id)
+
+        return obj
+
+    def _get_mapping(self, object_type, external_id):
+        self.object_mapping.setdefault(object_type.__name__, {})
+        return self.object_mapping[object_type.__name__].get(str(external_id), None)
+
+    def _update_mapping(self, object_type, external_id, internal_id):
+        self.object_mapping.setdefault(object_type.__name__, {})
+        self.object_mapping[object_type.__name__][str(external_id)] = internal_id
+
+import_manager = Manager(help="import utilities")
+export_manager = Manager(help="export utilities")
+
+
+@contextlib.contextmanager
+def importer_with_mapping_file(mapping_filename):
+    with open(mapping_filename) as f:
+        mapping = json.loads(f.read())
+
+    importer = Importer(object_mapping=mapping, data_source=get_data_source())
+    yield importer
+
+    with open(mapping_filename, 'w') as f:
+        f.write(json.dumps(importer.object_mapping, indent=2))
+
+
+def get_data_source():
+    try:
+        data_source = models.DataSource.get(models.DataSource.name=="Import")
+    except models.DataSource.DoesNotExist:
+        data_source = models.DataSource.create(name="Import", type="import", options='{}')
+
+    return data_source
+
+@import_manager.command
+def query(mapping_filename, query_filename, user_id):
+    user = models.User.get_by_id(user_id)
+    with open(query_filename) as f:
+        query = json.loads(f.read())
+
+    with importer_with_mapping_file(mapping_filename) as importer:
+        imported_query = importer.import_query(user, query)
+
+        print "New query id: {}".format(imported_query.id)
+
+
+@import_manager.command
+def dashboard(mapping_filename, dashboard_filename, user_id):
+    user = models.User.get_by_id(user_id)
+    with open(dashboard_filename) as f:
+        dashboard = json.loads(f.read())
+
+    with importer_with_mapping_file(mapping_filename) as importer:
+        importer.import_dashboard(user, dashboard)
+
+
+

redash/models.py

@@ -1,23 +1,114 @@
 import json
 import hashlib
+import logging
+import os
+import threading
 import time
 import datetime
-from flask.ext.peewee.utils import slugify
+import itertools
+
 import peewee
-from redash import db, utils
+from passlib.apps import custom_app_context as pwd_context
+from playhouse.postgres_ext import ArrayField
+from flask.ext.login import UserMixin, AnonymousUserMixin
+
+from redash import utils, settings
 
 
-class BaseModel(db.Model):
+class Database(object):
+    def __init__(self):
+        self.database_config = dict(settings.DATABASE_CONFIG)
+        self.database_name = self.database_config.pop('name')
+        self.database = peewee.PostgresqlDatabase(self.database_name, **self.database_config)
+        self.app = None
+        self.pid = os.getpid()
+
+    def init_app(self, app):
+        self.app = app
+        self.register_handlers()
+
+    def connect_db(self):
+        self._check_pid()
+        self.database.connect()
+
+    def close_db(self, exc):
+        self._check_pid()
+        if not self.database.is_closed():
+            self.database.close()
+
+    def _check_pid(self):
+        current_pid = os.getpid()
+        if self.pid != current_pid:
+            logging.info("New pid detected (%d!=%d); resetting database lock.", self.pid, current_pid)
+            self.pid = os.getpid()
+            self.database._conn_lock = threading.Lock()
+
+    def register_handlers(self):
+        self.app.before_request(self.connect_db)
+        self.app.teardown_request(self.close_db)
+
+
+db = Database()
+
+
+class BaseModel(peewee.Model):
+    class Meta:
+        database = db.database
+
     @classmethod
     def get_by_id(cls, model_id):
         return cls.get(cls.id == model_id)
 
 
-class User(BaseModel):
+class AnonymousUser(AnonymousUserMixin):
+    @property
+    def permissions(self):
+        return []
+
+
+class ApiUser(UserMixin):
+    def __init__(self, api_key):
+        self.id = api_key
+
+    @property
+    def permissions(self):
+        return ['view_query']
+
+
+class Group(BaseModel):
+    DEFAULT_PERMISSIONS = ['create_dashboard', 'create_query', 'edit_dashboard', 'edit_query',
+                           'view_query', 'view_source', 'execute_query']
+
+    id = peewee.PrimaryKeyField()
+    name = peewee.CharField(max_length=100)
+    permissions = ArrayField(peewee.CharField, default=DEFAULT_PERMISSIONS)
+    tables = ArrayField(peewee.CharField)
+    created_at = peewee.DateTimeField(default=datetime.datetime.now)
+
+    class Meta:
+        db_table = 'groups'
+
+    def to_dict(self):
+        return {
+            'id': self.id,
+            'name': self.name,
+            'permissions': self.permissions,
+            'tables': self.tables,
+            'created_at': self.created_at
+        }
+
+    def __unicode__(self):
+        return unicode(self.id)
+
+
+class User(BaseModel, UserMixin):
+    DEFAULT_GROUPS = ['default']
+
     id = peewee.PrimaryKeyField()
     name = peewee.CharField(max_length=320)
     email = peewee.CharField(max_length=320, index=True, unique=True)
-    is_admin = peewee.BooleanField(default=False)
+    password_hash = peewee.CharField(max_length=128, null=True)
+    groups = ArrayField(peewee.CharField, default=DEFAULT_GROUPS)
 
     class Meta:
         db_table = 'users'
@@ -26,16 +117,94 @@ class User(BaseModel):
         return {
             'id': self.id,
             'name': self.name,
-            'email': self.email,
-            'is_admin': self.is_admin
+            'email': self.email
         }
 
+    def __init__(self, *args, **kwargs):
+        super(User, self).__init__(*args, **kwargs)
+        self._allowed_tables = None
+
+    @property
+    def permissions(self):
+        # TODO: this should be cached.
+        return list(itertools.chain(*[g.permissions for g in
+                                      Group.select().where(Group.name << self.groups)]))
+
+    @property
+    def allowed_tables(self):
+        # TODO: cache this as weel
+        if self._allowed_tables is None:
+            self._allowed_tables = set([t.lower() for t in itertools.chain(*[g.tables for g in
+                                        Group.select().where(Group.name << self.groups)])])
+
+        return self._allowed_tables
+
+    @classmethod
+    def get_by_email(cls, email):
+        return cls.get(cls.email == email)
+
     def __unicode__(self):
         return '%r, %r' % (self.name, self.email)
 
+    def hash_password(self, password):
+        self.password_hash = pwd_context.encrypt(password)
+
+    def verify_password(self, password):
+        return self.password_hash and pwd_context.verify(password, self.password_hash)
+
+
+class ActivityLog(BaseModel):
+    QUERY_EXECUTION = 1
 
-class QueryResult(db.Model):
     id = peewee.PrimaryKeyField()
+    user = peewee.ForeignKeyField(User)
+    type = peewee.IntegerField()
+    activity = peewee.TextField()
+    created_at = peewee.DateTimeField(default=datetime.datetime.now)
+
+    class Meta:
+        db_table = 'activity_log'
+
+    def to_dict(self):
+        return {
+            'id': self.id,
+            'user': self.user.to_dict(),
+            'type': self.type,
+            'activity': self.activity,
+            'created_at': self.created_at
+        }
+
+    def __unicode__(self):
+        return unicode(self.id)
+
+
+class DataSource(BaseModel):
+    id = peewee.PrimaryKeyField()
+    name = peewee.CharField()
+    type = peewee.CharField()
+    options = peewee.TextField()
+    queue_name = peewee.CharField(default="queries")
+    scheduled_queue_name = peewee.CharField(default="queries")
+    created_at = peewee.DateTimeField(default=datetime.datetime.now)
+
+    class Meta:
+        db_table = 'data_sources'
+
+    def to_dict(self):
+        return {
+            'id': self.id,
+            'name': self.name,
+            'type': self.type
+        }
+
+    @classmethod
+    def all(cls):
+        return cls.select().order_by(cls.id.asc())
+
+
+class QueryResult(BaseModel):
+    id = peewee.PrimaryKeyField()
+    data_source = peewee.ForeignKeyField(DataSource)
     query_hash = peewee.CharField(max_length=32, index=True)
     query = peewee.TextField()
     data = peewee.TextField()
@@ -51,16 +220,51 @@ class QueryResult(db.Model):
             'query_hash': self.query_hash,
             'query': self.query,
             'data': json.loads(self.data),
+            'data_source_id': self._data.get('data_source', None),
             'runtime': self.runtime,
             'retrieved_at': self.retrieved_at
         }
 
+    @classmethod
+    def get_latest(cls, data_source, query, ttl=0):
+        query_hash = utils.gen_query_hash(query)
+
+        if ttl == -1:
+            query = cls.select().where(cls.query_hash == query_hash,
+                                       cls.data_source == data_source).order_by(cls.retrieved_at.desc())
+        else:
+            query = cls.select().where(cls.query_hash == query_hash, cls.data_source == data_source,
+                                       peewee.SQL("retrieved_at + interval '%s second' >= now() at time zone 'utc'",
+                                                  ttl)).order_by(cls.retrieved_at.desc())
+
+        return query.first()
+
+    @classmethod
+    def store_result(cls, data_source_id, query_hash, query, data, run_time, retrieved_at):
+        query_result = cls.create(query_hash=query_hash,
+                                  query=query,
+                                  runtime=run_time,
+                                  data_source=data_source_id,
+                                  retrieved_at=retrieved_at,
+                                  data=data)
+
+        logging.info("Inserted query (%s) data; id=%s", query_hash, query_result.id)
+
+        updated_count = Query.update(latest_query_data=query_result).\
+            where(Query.query_hash==query_hash, Query.data_source==data_source_id).\
+            execute()
+
+        logging.info("Updated %s queries with result (%s).", updated_count, query_hash)
+
+        return query_result
+
     def __unicode__(self):
         return u"%d | %s | %s" % (self.id, self.query_hash, self.retrieved_at)
 
 
 class Query(BaseModel):
     id = peewee.PrimaryKeyField()
+    data_source = peewee.ForeignKeyField(DataSource)
     latest_query_data = peewee.ForeignKeyField(QueryResult, null=True)
     name = peewee.CharField(max_length=255)
     description = peewee.CharField(max_length=4096, null=True)
@@ -81,7 +285,7 @@ class Query(BaseModel):
                                             type="TABLE", options="{}")
         table_visualization.save()
 
-    def to_dict(self, with_result=True, with_stats=False, with_visualizations=False, with_user=True):
+    def to_dict(self, with_stats=False, with_visualizations=False, with_user=True):
         d = {
             'id': self.id,
             'latest_query_data_id': self._data.get('latest_query_data', None),
@@ -92,6 +296,7 @@ class Query(BaseModel):
             'ttl': self.ttl,
             'api_key': self.api_key,
             'created_at': self.created_at,
+            'data_source_id': self._data.get('data_source', None)
         }
 
         if with_user:
@@ -110,9 +315,6 @@ class Query(BaseModel):
             d['visualizations'] = [vis.to_dict(with_query=False)
                                    for vis in self.visualizations]
 
-        if with_result and self.latest_query_data:
-            d['latest_query_data'] = self.latest_query_data.to_dict()
-
         return d
 
     @classmethod
@@ -129,6 +331,23 @@ class Query(BaseModel):
 
         return q
 
+    @classmethod
+    def outdated_queries(cls):
+        # TODO: this will only find scheduled queries that were executed before. I think this is
+        # a reasonable assumption, but worth revisiting.
+        outdated_queries_ids = cls.select(
+            peewee.Func('first_value', cls.id).over(partition_by=[cls.query_hash, cls.data_source])) \
+            .join(QueryResult) \
+            .where(cls.ttl > 0,
+                   (QueryResult.retrieved_at +
+                    (cls.ttl * peewee.SQL("interval '1 second'"))) <
+                   peewee.SQL("(now() at time zone 'utc')"))
+
+        queries = cls.select(cls, DataSource).join(DataSource) \
+            .where(cls.id << outdated_queries_ids)
+
+        return queries
+
     @classmethod
     def update_instance(cls, query_id, **kwargs):
         if 'query' in kwargs:
@@ -158,6 +377,7 @@ class Dashboard(BaseModel):
     user_email = peewee.CharField(max_length=360, null=True)
     user = peewee.ForeignKeyField(User)
     layout = peewee.TextField()
+    dashboard_filters_enabled = peewee.BooleanField(default=False)
     is_archived = peewee.BooleanField(default=False, index=True)
     created_at = peewee.DateTimeField(default=datetime.datetime.now)
 
@@ -168,15 +388,29 @@ class Dashboard(BaseModel):
         layout = json.loads(self.layout)
 
         if with_widgets:
-            widgets = Widget.select(Widget, Visualization, Query, QueryResult, User)\
+            widgets = Widget.select(Widget, Visualization, Query, User)\
                 .where(Widget.dashboard == self.id)\
-                .join(Visualization)\
-                .join(Query)\
-                .join(User)\
-                .switch(Query)\
-                .join(QueryResult)
+                .join(Visualization, join_type=peewee.JOIN_LEFT_OUTER)\
+                .join(Query, join_type=peewee.JOIN_LEFT_OUTER)\
+                .join(User, join_type=peewee.JOIN_LEFT_OUTER)
             widgets = {w.id: w.to_dict() for w in widgets}
-            widgets_layout = map(lambda row: map(lambda widget_id: widgets.get(widget_id, None), row), layout)
+
+            # The following is a workaround for cases when the widget object gets deleted without the dashboard layout
+            # updated. This happens for users with old databases that didn't have a foreign key relationship between
+            # visualizations and widgets.
+            # It's temporary until better solution is implemented (we probably should move the position information
+            # to the widget).
+            widgets_layout = []
+            for row in layout:
+                new_row = []
+                for widget_id in row:
+                    widget = widgets.get(widget_id, None)
+                    if widget:
+                        new_row.append(widget)
+
+                widgets_layout.append(new_row)
+
+            # widgets_layout = map(lambda row: map(lambda widget_id: widgets.get(widget_id, None), row), layout)
         else:
             widgets_layout = None
 
@@ -186,6 +420,7 @@ class Dashboard(BaseModel):
             'name': self.name,
             'user_id': self._data['user'],
             'layout': layout,
+            'dashboard_filters_enabled': self.dashboard_filters_enabled,
             'widgets': widgets_layout
         }
 
@@ -195,11 +430,11 @@ class Dashboard(BaseModel):
 
     def save(self, *args, **kwargs):
         if not self.slug:
-            self.slug = slugify(self.name)
+            self.slug = utils.slugify(self.name)
 
             tries = 1
             while self.select().where(Dashboard.slug == self.slug).first() is not None:
-                self.slug = slugify(self.name) + "_{0}".format(tries)
+                self.slug = utils.slugify(self.name) + "_{0}".format(tries)
                 tries += 1
 
         super(Dashboard, self).save(*args, **kwargs)
@@ -237,10 +472,10 @@ class Visualization(BaseModel):
         return u"%s %s" % (self.id, self.type)
 
 
-class Widget(db.Model):
+class Widget(BaseModel):
     id = peewee.PrimaryKeyField()
-    visualization = peewee.ForeignKeyField(Visualization, related_name='widgets')
-
+    visualization = peewee.ForeignKeyField(Visualization, related_name='widgets', null=True)
+    text = peewee.TextField(null=True)
     width = peewee.IntegerField()
     options = peewee.TextField()
     dashboard = peewee.ForeignKeyField(Dashboard, related_name='widgets', index=True)
@@ -254,18 +489,44 @@ class Widget(db.Model):
         db_table = 'widgets'
 
     def to_dict(self):
-        return {
+        d = {
             'id': self.id,
             'width': self.width,
             'options': json.loads(self.options),
-            'visualization': self.visualization.to_dict(),
-            'dashboard_id': self._data['dashboard']
+            'dashboard_id': self._data['dashboard'],
+            'text': self.text
         }
 
+        if self.visualization and self.visualization.id:
+            d['visualization'] = self.visualization.to_dict()
+
+        return d
+
     def __unicode__(self):
         return u"%s" % self.id
 
-all_models = (User, QueryResult, Query, Dashboard, Visualization, Widget)
+
+class Event(BaseModel):
+    user = peewee.ForeignKeyField(User, related_name="events")
+    action = peewee.CharField()
+    object_type = peewee.CharField()
+    object_id = peewee.CharField(null=True)
+    additional_properties = peewee.TextField(null=True)
+    created_at = peewee.DateTimeField(default=datetime.datetime.now)
+
+    class Meta:
+        db_table = 'events'
+
+    def __unicode__(self):
+        return u"%s,%s,%s,%s" % (self._data['user'], self.action, self.object_type, self.object_id)
+
+
+all_models = (DataSource, User, QueryResult, Query, Dashboard, Visualization, Widget, ActivityLog, Group, Event)
+
+
+def init_db():
+    Group.insert(name='admin', permissions=['admin'], tables=['*']).execute()
+    Group.insert(name='default', permissions=Group.DEFAULT_PERMISSIONS, tables=['*']).execute()
 
 
 def create_db(create_tables, drop_tables):
@@ -277,7 +538,7 @@ def create_db(create_tables, drop_tables):
             db.database.execute_sql('DROP TABLE %s CASCADE' % model._meta.db_table)
             #model.drop_table()
 
-        if create_tables:
+        if create_tables and not model.table_exists():
             model.create_table()
 
-    db.close_db(None)
+    db.close_db(None)

redash/permissions.py

@@ -0,0 +1,27 @@
+import functools
+from flask.ext.login import current_user
+from flask.ext.restful import abort
+
+
+class require_permissions(object):
+    def __init__(self, permissions):
+        self.permissions = permissions
+
+    def __call__(self, fn):
+        @functools.wraps(fn)
+        def decorated(*args, **kwargs):
+            has_permissions = reduce(lambda a, b: a and b,
+                                      map(lambda permission: permission in current_user.permissions,
+                                          self.permissions),
+                                      True)
+
+            if has_permissions:
+                return fn(*args, **kwargs)
+            else:
+                abort(403)
+
+        return decorated
+
+
+def require_permission(permission):
+    return require_permissions((permission,))

redash/settings.py

@@ -1,12 +1,11 @@
+import json
 import os
 import urlparse
 
 
 def parse_db_url(url):
     url_parts = urlparse.urlparse(url)
-    connection = {
-        'engine': 'peewee.PostgresqlDatabase',
-    }
+    connection = {'threadlocals': True}
 
     if url_parts.hostname and not url_parts.path:
         connection['name'] = url_parts.hostname
@@ -32,27 +31,46 @@ def array_from_string(str):
 
     return array
 
-REDIS_URL = os.environ.get('REDASH_REDIS_URL', "redis://localhost:6379")
 
-# "pg", "graphite" or "mysql"
+def parse_boolean(str):
+    return json.loads(str.lower())
+
+
+NAME = os.environ.get('REDASH_NAME', 're:dash')
+
+REDIS_URL = os.environ.get('REDASH_REDIS_URL', "redis://localhost:6379/0")
+
+STATSD_HOST = os.environ.get('REDASH_STATSD_HOST', "127.0.0.1")
+STATSD_PORT = int(os.environ.get('REDASH_STATSD_PORT', "8125"))
+STATSD_PREFIX = os.environ.get('REDASH_STATSD_PREFIX', "redash")
+
+# The following is kept for backward compatability, and shouldn't be used any more.
 CONNECTION_ADAPTER = os.environ.get("REDASH_CONNECTION_ADAPTER", "pg")
-# Connection string for the database that is used to run queries against. Examples:
-# -- mysql:    CONNECTION_STRING = "Server=;User=;Pwd=;Database="
-# -- pg:       CONNECTION_STRING = "user= password= host= port=5439 dbname="
-# -- graphite: CONNECTION_STRING = {"url": "https://graphite.yourcompany.com", "auth": ["user", "password"], "verify": true}
 CONNECTION_STRING = os.environ.get("REDASH_CONNECTION_STRING", "user= password= host= port=5439 dbname=")
 
 # Connection settings for re:dash's own database (where we store the queries, results, etc)
 DATABASE_CONFIG = parse_db_url(os.environ.get("REDASH_DATABASE_URL", "postgresql://postgres"))
 
+# Celery related settings
+CELERY_BROKER = os.environ.get("REDASH_CELERY_BROKER", REDIS_URL)
+CELERY_BACKEND = os.environ.get("REDASH_CELERY_BACKEND", REDIS_URL)
+CELERY_FLOWER_URL = os.environ.get("REDASH_CELERY_FLOWER_URL", "/flower")
+
 # Google Apps domain to allow access from; any user with email in this Google Apps will be allowed
 # access
 GOOGLE_APPS_DOMAIN = os.environ.get("REDASH_GOOGLE_APPS_DOMAIN", "")
-# Email addresses of admin users (comma separated)
-ADMINS = array_from_string(os.environ.get("REDASH_ADMINS", ''))
+GOOGLE_OPENID_ENABLED = parse_boolean(os.environ.get("REDASH_GOOGLE_OPENID_ENABLED", "true"))
+PASSWORD_LOGIN_ENABLED = parse_boolean(os.environ.get("REDASH_PASSWORD_LOGIN_ENABLED", "false"))
 ALLOWED_EXTERNAL_USERS = array_from_string(os.environ.get("REDASH_ALLOWED_EXTERNAL_USERS", ''))
-STATIC_ASSETS_PATH = fix_assets_path(os.environ.get("REDASH_STATIC_ASSETS_PATH", "../rd_ui/dist/"))
+STATIC_ASSETS_PATH = fix_assets_path(os.environ.get("REDASH_STATIC_ASSETS_PATH", "../rd_ui/app/"))
 WORKERS_COUNT = int(os.environ.get("REDASH_WORKERS_COUNT", "2"))
+JOB_EXPIRY_TIME = int(os.environ.get("REDASH_JOB_EXPIRY_TIME", 3600*24))
 COOKIE_SECRET = os.environ.get("REDASH_COOKIE_SECRET", "c292a0a3aa32397cdb050e233733900f")
 LOG_LEVEL = os.environ.get("REDASH_LOG_LEVEL", "INFO")
-ANALYTICS = os.environ.get("REDASH_ANALYTICS", "")
+EVENTS_LOG_PATH = os.environ.get("REDASH_EVENTS_LOG_PATH", "")
+EVENTS_CONSOLE_OUTPUT = parse_boolean(os.environ.get("REDASH_EVENTS_CONSOLE_OUTPUT", "false"))
+CLIENT_SIDE_METRICS = parse_boolean(os.environ.get("REDASH_CLIENT_SIDE_METRICS", "false"))
+ANALYTICS = os.environ.get("REDASH_ANALYTICS", "")
+
+# Features:
+FEATURE_TABLES_PERMISSIONS = parse_boolean(os.environ.get("REDASH_FEATURE_TABLES_PERMISSIONS", "false"))

redash/tasks.py

@@ -0,0 +1,197 @@
+import time
+import datetime
+import logging
+import redis
+from celery import Task
+from celery.result import AsyncResult
+from celery.utils.log import get_task_logger
+from redash import redis_connection, models, statsd_client
+from redash.utils import gen_query_hash
+from redash.worker import celery
+from redash.data.query_runner import get_query_runner
+
+logger = get_task_logger(__name__)
+
+
+class BaseTask(Task):
+    abstract = True
+
+    def after_return(self, *args, **kwargs):
+        models.db.close_db(None)
+
+    def __call__(self, *args, **kwargs):
+        models.db.connect_db()
+        return super(BaseTask, self).__call__(*args, **kwargs)
+
+
+class QueryTask(object):
+    MAX_RETRIES = 5
+
+    # TODO: this is mapping to the old Job class statuses. Need to update the client side and remove this
+    STATUSES = {
+        'PENDING': 1,
+        'STARTED': 2,
+        'SUCCESS': 3,
+        'FAILURE': 4,
+        'REVOKED': 4
+    }
+
+    def __init__(self, job_id=None, async_result=None):
+        if async_result:
+            self._async_result = async_result
+        else:
+            self._async_result = AsyncResult(job_id, app=celery)
+
+    @property
+    def id(self):
+        return self._async_result.id
+
+    @classmethod
+    def add_task(cls, query, data_source, scheduled=False):
+        query_hash = gen_query_hash(query)
+        logging.info("[Manager][%s] Inserting job", query_hash)
+        try_count = 0
+        job = None
+
+        while try_count < cls.MAX_RETRIES:
+            try_count += 1
+
+            pipe = redis_connection.pipeline()
+            try:
+                pipe.watch(cls._job_lock_id(query_hash, data_source.id))
+                job_id = pipe.get(cls._job_lock_id(query_hash, data_source.id))
+                if job_id:
+                    logging.info("[Manager][%s] Found existing job: %s", query_hash, job_id)
+
+                    job = cls(job_id=job_id)
+                else:
+                    pipe.multi()
+
+                    if scheduled:
+                        queue_name = data_source.scheduled_queue_name
+                    else:
+                        queue_name = data_source.queue_name
+
+                    result = execute_query.apply_async(args=(query, data_source.id), queue=queue_name)
+                    job = cls(async_result=result)
+                    logging.info("[Manager][%s] Created new job: %s", query_hash, job.id)
+                    pipe.set(cls._job_lock_id(query_hash, data_source.id), job.id)
+                    pipe.execute()
+                break
+
+            except redis.WatchError:
+                continue
+
+        if not job:
+            logging.error("[Manager][%s] Failed adding job for query.", query_hash)
+
+        return job
+
+    def to_dict(self):
+        if self._async_result.status == 'STARTED':
+            updated_at = self._async_result.result.get('start_time', 0)
+        else:
+            updated_at = 0
+
+        if self._async_result.failed() and isinstance(self._async_result.result, Exception):
+            error = self._async_result.result.message
+        elif self._async_result.status == 'REVOKED':
+            error = 'Query execution cancelled.'
+        else:
+            error = ''
+
+        if self._async_result.successful():
+            query_result_id = self._async_result.result
+        else:
+            query_result_id = None
+
+        return {
+            'id': self._async_result.id,
+            'updated_at': updated_at,
+            'status': self.STATUSES[self._async_result.status],
+            'error': error,
+            'query_result_id': query_result_id,
+        }
+
+    def cancel(self):
+        return self._async_result.revoke(terminate=True)
+
+    @staticmethod
+    def _job_lock_id(query_hash, data_source_id):
+        return "query_hash_job:%s:%s" % (data_source_id, query_hash)
+
+
+@celery.task(base=BaseTask)
+def refresh_queries():
+    # self.status['last_refresh_at'] = time.time()
+    # self._save_status()
+
+    logger.info("Refreshing queries...")
+
+    outdated_queries_count = 0
+    for query in models.Query.outdated_queries():
+        # TODO: this should go into lower priority
+        QueryTask.add_task(query.query, query.data_source, scheduled=True)
+        outdated_queries_count += 1
+
+    statsd_client.gauge('manager.outdated_queries', outdated_queries_count)
+    # TODO: decide if we still need this
+    # statsd_client.gauge('manager.queue_size', self.redis_connection.zcard('jobs'))
+
+    logger.info("Done refreshing queries. Found %d outdated queries." % outdated_queries_count)
+
+    status = redis_connection.hgetall('redash:status')
+    now = time.time()
+
+    redis_connection.hmset('redash:status', {
+        'outdated_queries_count': outdated_queries_count,
+        'last_refresh_at': now
+    })
+
+    statsd_client.gauge('manager.seconds_since_refresh', now - float(status.get('last_refresh_at', now)))
+
+
+@celery.task(bind=True, base=BaseTask, track_started=True)
+def execute_query(self, query, data_source_id):
+    # TODO: maybe this should be a class?
+    start_time = time.time()
+
+    logger.info("Loading data source (%d)...", data_source_id)
+
+    # TODO: we should probably cache data sources in Redis
+    data_source = models.DataSource.get_by_id(data_source_id)
+
+    self.update_state(state='STARTED', meta={'start_time': start_time, 'custom_message': ''})
+
+    logger.info("Executing query:\n%s", query)
+
+    query_hash = gen_query_hash(query)
+    query_runner = get_query_runner(data_source.type, data_source.options)
+
+    if getattr(query_runner, 'annotate_query', True):
+        # TODO: anotate with queu ename
+        annotated_query = "/* Task Id: %s, Query hash: %s */ %s" % \
+                          (self.request.id, query_hash, query)
+    else:
+        annotated_query = query
+
+    with statsd_client.timer('query_runner.{}.{}.run_time'.format(data_source.type, data_source.name)):
+        data, error = query_runner(annotated_query)
+
+    run_time = time.time() - start_time
+    logger.info("Query finished... data length=%s, error=%s", data and len(data), error)
+
+    self.update_state(state='STARTED', meta={'start_time': start_time, 'error': error, 'custom_message': ''})
+
+    # Delete query_hash
+    redis_connection.delete(QueryTask._job_lock_id(query_hash, data_source.id))
+
+    # TODO: it is possible that storing the data will fail, and we will need to retry
+    # while we already marked the job as done
+    if not error:
+        query_result = models.QueryResult.store_result(data_source.id, query_hash, query, data, run_time, datetime.datetime.utcnow())
+    else:
+        raise Exception(error)
+
+    return query_result.id
+