Merge pull request #181 from EverythingMe/feature_imrpove_updater

Switch to multiprocessing instead of threading

.coveragerc

@@ -0,0 +1,5 @@
+[report]
+omit =
+    */settings.py
+    */python?.?/*
+    */site-packages/nose/*

.env.example

@@ -0,0 +1,9 @@
+REDASH_CONNECTION_ADAPTER=pg
+REDASH_CONNECTION_STRING="dbname=data"
+REDASH_STATIC_ASSETS_PATH=../rd_ui/app/
+REDASH_GOOGLE_APPS_DOMAIN=
+REDASH_ADMINS=
+REDASH_WORKERS_COUNT=2
+REDASH_COOKIE_SECRET=
+REDASH_DATABASE_URL='postgresql://rd'
+REDASH_LOG_LEVEL = "INFO"

.gitignore

@@ -1,10 +1,13 @@
+.coveralls.yml
 .idea
 *.pyc
-rd_service/settings.py
+.coverage
 rd_ui/dist
 .DS_Store
 
 # Vagrant related
 .vagrant
 Berksfile.lock
-rd_service/dump.rdb
+redash/dump.rdb
+.env
+.ruby-version

Berksfile

@@ -1,3 +0,0 @@
-cookbook 'apt'
-cookbook 'postgresql'
-cookbook 'redash', git: 'git@github.com:EverythingMe/chef-redash.git'

Makefile

@@ -1,6 +1,6 @@
 NAME=redash
-VERSION=0.2
-FULL_VERSION=$(VERSION).$(CIRCLE_BUILD_NUM)
+VERSION=`python ./manage.py version`
+FULL_VERSION=$(VERSION)+b$(CIRCLE_BUILD_NUM)
 FILENAME=$(CIRCLE_ARTIFACTS)/$(NAME).$(FULL_VERSION).tar.gz
 
 deps:
@@ -14,3 +14,6 @@ pack:
 
 upload:
 	python bin/upload_version.py $(FULL_VERSION) $(FILENAME)
+
+test:
+	nosetests --with-coverage --cover-package=redash tests/*.py

Procfile

@@ -0,0 +1 @@
+web: honcho start -f Procfile.heroku -p $PORT

Procfile.dev

@@ -0,0 +1,2 @@
+web: ./manage.py runserver -p $PORT
+worker: ./manage.py runworkers

Procfile.heroku

@@ -0,0 +1,2 @@
+web: ./manage.py runserver -p $PORT --host 0.0.0.0 -d -r
+worker: ./manage.py runworkers

README.md

@@ -1,4 +1,5 @@
 # [_re:dash_](https://github.com/everythingme/redash)
+![Build Status](https://circleci.com/gh/EverythingMe/redash.png?circle-token=8a695aa5ec2cbfa89b48c275aea298318016f040 "Build Status")
 
 **_re:dash_** is our take on freeing the data within our company in a way that will better fit our culture and usage patterns.
 
@@ -28,8 +29,8 @@ Due to Heroku dev plan limits, it has a small database of flights (see schema [h
 
 ## Technology
 
+* Python
 * [AngularJS](http://angularjs.org/)
-* [Tornado](http://tornadoweb.org)
 * [PostgreSQL](http://www.postgresql.org/) / [AWS Redshift](http://aws.amazon.com/redshift/)
 * [Redis](http://redis.io)
 
@@ -45,62 +46,21 @@ It's very likely that in the future we will switch to [D3.js](http://d3js.org/)
 
 ## Getting Started
 
-1. Clone the repo:
-```bash
-git clone git@github.com:EverythingMe/redash.git
-```
-2. Create settings file from the example one (& update relevant settings):
-```bash
-cp rd_service/settings_example.py rd_service/settings.py
-```
-It's highly recommended that the user you use to connect to the data database (the one you query) is read-only.
-3. Create the operational databases from rd_service/data/tables.sql
-3. Install `npm` packages (mainly: Bower & Grunt):
-```bash
-cd rd_ui
-npm install
-```
-4. Install `bower` packages:
-```bash
-bower install
-```
-5. Build the UI:
-```bash
-grunt build
-```
-6. Install PIP packages:
-```bash
-pip install -r ../rd_service/requirements.txt
-```
-6. Start the API server:
-```bash
-cd ../rd_service
-python server.py
-```
-7. Start the workers:
-```bash
-python cli.py worker
-```
-8. Open `http://localhost:8888/` and query away.
+* [Setting up re:dash on Heroku in 5 minutes](https://github.com/EverythingMe/redash/wiki/Setting-up-re:dash-on-Heroku-in-5-minutes)
+* [Setting re:dash on your own server (Ubuntu)](https://github.com/EverythingMe/redash/wiki/Setting-re:dash-on-your-own-server-(Ubuntu))
+
+**Need help setting re:dash or one of the dependencies up?** Ping @arikfr on the IRC #redash channel or send a message to the [mailing list](https://groups.google.com/forum/#!forum/redash-users), and he will gladly help.
 
 ## Roadmap
 
-We plan to release new minor version every 2-3 weeks. Of course, if we get additional help from contributors it will help speed things up.
-
 Below you can see the "big" features of the next 3 releases (for full list, click on the link):
 
-### [v0.2](https://github.com/EverythingMe/redash/issues?milestone=1&state=open)
-
-- Ability to generate multiple visualizations for a single query (dataset) in a more flexible way than today. Also easier extensbility points to add additional visualizations.
-- Dashboard filters: ability to filter/slice the data you see in a single dashboard using filters (date or selectors).
-- UI Improvements (better notifications & flows, improved queries page) 
-- Comments on queries.
-
 ### [v0.3](https://github.com/EverythingMe/redash/issues?milestone=2&state=open)
 
-- Support for API access using API keys, instead of Google Login.
+- Dashboard filters: ability to filter/slice the data you see in a single dashboard using filters (date or selectors).
 - Multiple databases support (including other database type than PostgreSQL).
 - Scheduled reports by email.
+- Comments on queries.
 
 ### [v0.4](https://github.com/EverythingMe/redash/issues?milestone=3&state=open)
 

Vagrantfile

@@ -1,60 +0,0 @@
-# -*- mode: ruby -*-
-# vi: set ft=ruby :
-
-# Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
-VAGRANTFILE_API_VERSION = '2'
-
-POSTGRES_PASSWORD     = 'securepass'
-
-# Currently, chef postgress cookbook works with cleartext paswords,
-# unless the password begins with 'md5'
-# See https://github.com/hw-cookbooks/postgresql/issues/95
-require "digest/md5"
-postgres_password_md5 = 'md5'+Digest::MD5.hexdigest(POSTGRES_PASSWORD+'postgres')
-
-# After starting the vagrant machine, the application is accessible via the URL
-# http://localhost:9999
-HOST_PORT_TO_FORWARD_TO_REDASH = 9999
-
-# Deploy direcly the code in parent dir; Don't download a release tarball 
-live_testing_deployment = true
-
-Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
-  config.vm.box     = 'ubuntu-precise-cloudimg-amd64'
-  config.vm.box_url = 'http://cloud-images.ubuntu.com/vagrant/precise/current/precise-server-cloudimg-amd64-vagrant-disk1.box'
-  
-  if config.respond_to? :cache
-    config.cache.auto_detect  = true
-  end
-  
-  config.berkshelf.enabled    = true
-  config.omnibus.chef_version = :latest
-
-  config.vm.network 'forwarded_port', guest: 8888, host: HOST_PORT_TO_FORWARD_TO_REDASH
-  
-  if live_testing_deployment
-    config.vm.synced_folder "..", "/opt/redash"
-  end
-  
-  config.vm.provision :chef_solo do |chef|
-     # run apt-get update before anything else (specifically postgresql)..
-     chef.add_recipe 'apt'
-     chef.add_recipe 'redash::redis_for_redash'
-     chef.add_recipe 'postgresql::client'
-     chef.add_recipe 'postgresql::server'
-     chef.add_recipe 'redash::redash_pg_schema'
-     chef.add_recipe 'redash::redash'
-     # chef.log_level  = :debug
-     chef.json = {
-       'apt'        => { 'compiletime' => true },
-       'postgresql' => { 'password'    => {'postgres' => postgres_password_md5 } },
-       'redash'     => { 'db'          => {'host'     => 'localhost',
-                                           'user'     => 'postgres',
-                                           'password' => POSTGRES_PASSWORD },
-                         'allow'           => {'google_app_domain' => 'gmail.com',
-                                               'admins'            => ['joe@egmail.com','jack@gmail.com']},
-                         'install_tarball' => !live_testing_deployment,
-                         'user'            => live_testing_deployment ? 'vagrant' : 'redash'}
-     }
-   end
-end

bin/latest_release.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+import sys
+import requests
+
+if __name__ == '__main__':
+    response = requests.get('https://api.github.com/repos/EverythingMe/redash/releases')
+
+    if response.status_code != 200:
+        exit("Failed getting releases (status code: %s)." % response.status_code)
+
+    sorted_releases = sorted(response.json(), key=lambda release: release['id'], reverse=True)
+
+    latest_release = sorted_releases[0]
+    asset_url = latest_release['assets'][0]['url']
+
+    if '--url-only' in sys.argv:
+        print asset_url
+    else:
+        print "Latest release: %s" % latest_release['tag_name']
+        print latest_release['body']
+
+        print "\nTarball URL: %s" % asset_url
+        print 'wget: wget --header="Accept: application/octet-stream" %s' % asset_url
+
+

bin/run

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+# Ideally I would use stdin with source, but in older bash versions this
+# wasn't supported properly.
+TEMP_ENV_FILE=`mktemp /tmp/redash_env.XXXXXX`
+sed 's/^REDASH/export REDASH/' .env > $TEMP_ENV_FILE
+source $TEMP_ENV_FILE
+rm $TEMP_ENV_FILE
+
+exec "$@"

bin/test_multithreading.py

@@ -0,0 +1,63 @@
+"""
+Script to test concurrency (multithreading/multiprocess) issues with the workers. Use with caution.
+"""
+import json
+import atfork
+atfork.monkeypatch_os_fork_functions()
+import atfork.stdlib_fixer
+atfork.stdlib_fixer.fix_logging_module()
+
+import time
+from redash.data import worker
+from redash import models, data_manager, redis_connection
+
+if __name__ == '__main__':
+    models.create_db(True, False)
+
+    print "Creating data source..."
+    data_source = models.DataSource.create(name="Concurrency", type="pg", options="dbname=postgres")
+
+    print "Clear jobs/hashes:"
+    redis_connection.delete("jobs")
+    query_hashes = redis_connection.keys("query_hash_*")
+    if query_hashes:
+        redis_connection.delete(*query_hashes)
+
+    starting_query_results_count = models.QueryResult.select().count()
+    jobs_count = 5000
+    workers_count = 10
+
+    print "Creating jobs..."
+    for i in xrange(jobs_count):
+        query = "SELECT {}".format(i)
+        print "Inserting: {}".format(query)
+        data_manager.add_job(query=query, priority=worker.Job.LOW_PRIORITY,
+                             data_source=data_source)
+
+    print "Starting workers..."
+    workers = data_manager.start_workers(workers_count)
+
+    print "Waiting for jobs to be done..."
+    keep_waiting = True
+    while keep_waiting:
+        results_count = models.QueryResult.select().count() - starting_query_results_count
+        print "QueryResults: {}".format(results_count)
+        time.sleep(5)
+        if results_count == jobs_count:
+            print "Yay done..."
+            keep_waiting = False
+
+    data_manager.stop_workers()
+
+    qr_count = 0
+    for qr in models.QueryResult.select():
+        number = int(qr.query.split()[1])
+        data_number = json.loads(qr.data)['rows'][0].values()[0]
+
+        if number != data_number:
+            print "Oops? {} != {} ({})".format(number, data_number, qr.id)
+        qr_count += 1
+
+    print "Verified {} query results.".format(qr_count)
+
+    print "Done."

bin/upload_version.py

@@ -3,29 +3,44 @@ import os
 import sys
 import json
 import requests
+import subprocess
+
+
+def capture_output(command):
+    proc = subprocess.Popen(command, stdout=subprocess.PIPE)
+    return proc.stdout.read()
+
 
 if __name__ == '__main__':
-  version = sys.argv[1]
-  filepath = sys.argv[2]
-  filename = filepath.split('/')[-1]
-  github_token = os.environ['GITHUB_TOKEN']
-  auth = (github_token, 'x-oauth-basic')
-  commit_sha = os.environ['CIRCLE_SHA1']
+    version = sys.argv[1]
+    filepath = sys.argv[2]
+    filename = filepath.split('/')[-1]
+    github_token = os.environ['GITHUB_TOKEN']
+    auth = (github_token, 'x-oauth-basic')
+    commit_sha = os.environ['CIRCLE_SHA1']
 
-  params = json.dumps({
-    'tag_name': 'v{0}'.format(version),
-    'name': 're:dash v{0}'.format(version),
-    'target_commitish': commit_sha
-  })
+    commit_body = capture_output(["git", "log", "--format=%b", "-n", "1", commit_sha])
+    file_md5_checksum = capture_output(["md5sum", filepath]).split()[0]
+    file_sha256_checksum = capture_output(["sha256sum", filepath]).split()[0]
+    version_body = "%s\n\nMD5: %s\nSHA256: %s" % (commit_body, file_md5_checksum, file_sha256_checksum)
 
-  response = requests.post('https://api.github.com/repos/everythingme/redash/releases',
-                            data=params,
-                            auth=auth)
+    params = json.dumps({
+        'tag_name': 'v{0}'.format(version),
+        'name': 're:dash v{0}'.format(version),
+        'body': version_body,
+        'target_commitish': commit_sha,
+        'prerelease': True
+    })
 
-  upload_url = response.json()['upload_url']
-  upload_url = upload_url.replace('{?name}', '')
+    response = requests.post('https://api.github.com/repos/everythingme/redash/releases',
+                             data=params,
+                             auth=auth)
 
-  with open(filepath) as file_content:
-    headers = {'Content-Type': 'application/gzip'}
-    response = requests.post(upload_url, file_content, params={'name': filename}, auth=auth, headers=headers, verify=False)
+    upload_url = response.json()['upload_url']
+    upload_url = upload_url.replace('{?name}', '')
+
+    with open(filepath) as file_content:
+        headers = {'Content-Type': 'application/gzip'}
+        response = requests.post(upload_url, file_content, params={'name': filename}, auth=auth,
+                                 headers=headers, verify=False)
 

circle.yml

@@ -8,8 +8,14 @@ machine:
 dependencies:
   pre:
     - make deps
-    - pip install requests
+    - pip install -r dev_requirements.txt
+    - pip install -r requirements.txt
+  cache_directories:
+    - rd_ui/node_modules/
+    - rd_ui/app/bower_components/
 test:
+  override:
+    - make test
   post:
     - make pack 
 deployment:

dev_requirements.txt

@@ -0,0 +1,3 @@
+nose==1.3.0
+coverage==3.7.1
+mock==1.0.1

manage.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python
+"""
+CLI to manage redash.
+"""
+import signal
+import logging
+import time
+from redash import settings, app, db, models, data_manager, __version__
+from redash.import_export import import_manager
+from flask.ext.script import Manager, prompt_pass
+
+manager = Manager(app)
+database_manager = Manager(help="Manages the database (create/drop tables).")
+users_manager = Manager(help="Users management commands.")
+data_sources_manager = Manager(help="Data sources management commands.")
+
+@manager.command
+def version():
+    """Displays re:dash version."""
+    print __version__
+
+
+@manager.command
+def runworkers():
+    """Starts the re:dash query executors/workers."""
+
+    def stop_handler(signum, frame):
+        logging.warning("Exiting; waiting for workers")
+        data_manager.stop_workers()
+        exit()
+
+    signal.signal(signal.SIGTERM, stop_handler)
+    signal.signal(signal.SIGINT, stop_handler)
+
+    old_workers = data_manager.redis_connection.smembers('workers')
+    data_manager.redis_connection.delete('workers')
+
+    logging.info("Cleaning old workers: %s", old_workers)
+
+    data_manager.start_workers(settings.WORKERS_COUNT)
+    logging.info("Workers started.")
+
+    while True:
+        try:
+            data_manager.refresh_queries()
+            data_manager.report_status()
+        except Exception as e:
+            logging.error("Something went wrong with refreshing queries...")
+            logging.exception(e)
+        time.sleep(60)
+
+
+@manager.shell
+def make_shell_context():
+    return dict(app=app, db=db, models=models)
+
+@manager.command
+def check_settings():
+    from types import ModuleType
+
+    for name in dir(settings):
+        item = getattr(settings, name)
+        if not callable(item) and not name.startswith("__") and not isinstance(item, ModuleType):
+            print "{} = {}".format(name, item)
+
+@database_manager.command
+def create_tables():
+    """Creates the database tables."""
+    from redash.models import create_db
+
+    create_db(True, False)
+
+@database_manager.command
+def drop_tables():
+    """Drop the database tables."""
+    from redash.models import create_db
+
+    create_db(False, True)
+
+
+@users_manager.option('email', help="User's email")
+@users_manager.option('name', help="User's full name")
+@users_manager.option('--admin', dest='is_admin', action="store_true", default=False, help="set user as admin")
+@users_manager.option('--google', dest='google_auth', action="store_true", default=False, help="user uses Google Auth to login")
+@users_manager.option('--password', dest='password', default=None, help="Password for users who don't use Google Auth (leave blank for prompt).")
+@users_manager.option('--permissions', dest='permissions', default=models.User.DEFAULT_PERMISSIONS, help="Comma seperated list of permissions (leave blank for default).")
+def create(email, name, permissions, is_admin=False, google_auth=False, password=None):
+    print "Creating user (%s, %s)..." % (email, name)
+    print "Admin: %r" % is_admin
+    print "Login with Google Auth: %r\n" % google_auth
+    if isinstance(permissions, basestring):
+        permissions = permissions.split(',')
+        permissions.remove('') # in case it was empty string
+
+    if is_admin:
+        permissions += ['admin']
+
+    user = models.User(email=email, name=name, permissions=permissions)
+    if not google_auth:
+        password = password or prompt_pass("Password")
+        user.hash_password(password)
+
+    try:
+        user.save()
+    except Exception, e:
+        print "Failed creating user: %s" % e.message
+
+
+@users_manager.option('email', help="email address of user to delete")
+def delete(email):
+    deleted_count = models.User.delete().where(models.User.email == email).execute()
+    print "Deleted %d users." % deleted_count
+
+@data_sources_manager.command
+def import_from_settings(name=None):
+    """Import data source from settings (env variables)."""
+    name = name or "Default"
+    data_source = models.DataSource.create(name=name,
+                                           type=settings.CONNECTION_ADAPTER,
+                                           options=settings.CONNECTION_STRING)
+
+    print "Imported data source from settings (id={}).".format(data_source.id)
+
+
+@data_sources_manager.command
+def list():
+    """List currently configured data sources"""
+    for ds in models.DataSource.select():
+        print "Name: {}\nType: {}\nOptions: {}".format(ds.name, ds.type, ds.options)
+
+@data_sources_manager.command
+def new(name, type, options):
+    """Create new data source"""
+    # TODO: validate it's a valid type and in the future, validate the options.
+    print "Creating {} data source ({}) with options:\n{}".format(type, name, options)
+    data_source = models.DataSource.create(name=name,
+                                           type=type,
+                                           options=options)
+    print "Id: {}".format(data_source.id)
+
+
+manager.add_command("database", database_manager)
+manager.add_command("users", users_manager)
+manager.add_command("import", import_manager)
+manager.add_command("ds", data_sources_manager)
+
+if __name__ == '__main__':
+    manager.run()

migrations/add_created_at_field.py

@@ -0,0 +1,13 @@
+from playhouse.migrate import Migrator
+from redash import db
+from redash import models
+
+
+if __name__ == '__main__':
+    db.connect_db()
+    migrator = Migrator(db.database)
+    with db.database.transaction():
+        migrator.add_column(models.Dashboard, models.Dashboard.created_at, 'created_at')
+        migrator.add_column(models.Widget, models.Widget.created_at, 'created_at')
+
+    db.close_db(None)

migrations/add_password_to_users.py

@@ -0,0 +1,12 @@
+from playhouse.migrate import Migrator
+from redash import db
+from redash import models
+
+
+if __name__ == '__main__':
+    db.connect_db()
+    migrator = Migrator(db.database)
+    with db.database.transaction():
+        migrator.add_column(models.User, models.User.password_hash, 'password_hash')
+
+    db.close_db(None)

migrations/add_permissions_to_user.py

@@ -0,0 +1,13 @@
+from playhouse.migrate import Migrator
+from redash import db
+from redash import models
+
+
+if __name__ == '__main__':
+    db.connect_db()
+    migrator = Migrator(db.database)
+    with db.database.transaction():
+        migrator.add_column(models.User, models.User.permissions, 'permissions')
+        models.User.update(permissions=['admin'] + models.User.DEFAULT_PERMISSIONS).where(models.User.is_admin == True).execute()
+
+    db.close_db(None)

migrations/add_view_query_permission.py

@@ -0,0 +1,13 @@
+import peewee
+from redash import db
+from redash import models
+
+
+if __name__ == '__main__':
+    db.connect_db()
+
+    previous_default_permissions = models.User.DEFAULT_PERMISSIONS[:]
+    previous_default_permissions.remove('view_query')
+    models.User.update(permissions=peewee.fn.array_append(models.User.permissions, 'view_query')).where(peewee.SQL("'view_source' = any(permissions)")).execute()
+
+    db.close_db(None)

migrations/change_queries_description_to_nullable.py

@@ -0,0 +1,12 @@
+from playhouse.migrate import Migrator
+from redash import db
+from redash import models
+
+
+if __name__ == '__main__':
+    db.connect_db()
+    migrator = Migrator(db.database)
+    with db.database.transaction():
+        migrator.set_nullable(models.Query, models.Query.description, True)
+
+    db.close_db(None)

migrations/change_query_id_on_widgets_to_null.py

@@ -0,0 +1,13 @@
+from playhouse.migrate import Migrator
+from redash import db
+from redash import models
+
+
+if __name__ == '__main__':
+    db.connect_db()
+    migrator = Migrator(db.database)
+    with db.database.transaction():
+        migrator.set_nullable(models.Widget, models.Widget.query_id, True)
+        migrator.set_nullable(models.Widget, models.Widget.type, True)
+
+    db.close_db(None)

migrations/create_activity_log.py

@@ -0,0 +1,11 @@
+from redash import db
+from redash import models
+
+if __name__ == '__main__':
+    db.connect_db()
+
+    if not models.ActivityLog.table_exists():
+        print "Creating activity_log table..."
+        models.ActivityLog.create_table()
+
+    db.close_db(None)

migrations/create_data_sources.py

@@ -0,0 +1,48 @@
+import logging
+import peewee
+from playhouse.migrate import Migrator
+from redash import db
+from redash import models
+from redash import settings
+
+if __name__ == '__main__':
+    db.connect_db()
+
+    if not models.DataSource.table_exists():
+        print "Creating data_sources table..."
+        models.DataSource.create_table()
+
+        default_data_source = models.DataSource.create(name="Default",
+                                                       type=settings.CONNECTION_ADAPTER,
+                                                       options=settings.CONNECTION_STRING)
+    else:
+        default_data_source = models.DataSource.select().first()
+
+    migrator = Migrator(db.database)
+    models.Query.data_source.null = True
+    models.QueryResult.data_source.null = True
+    try:
+        with db.database.transaction():
+            migrator.add_column(models.Query, models.Query.data_source, "data_source_id")
+    except peewee.ProgrammingError:
+        print "Failed to create data_source_id column -- assuming it already exists"
+
+    try:
+        with db.database.transaction():
+            migrator.add_column(models.QueryResult, models.QueryResult.data_source, "data_source_id")
+    except peewee.ProgrammingError:
+        print "Failed to create data_source_id column -- assuming it already exists"
+
+    print "Updating data source to existing one..."
+    models.Query.update(data_source=default_data_source.id).execute()
+    models.QueryResult.update(data_source=default_data_source.id).execute()
+
+    with db.database.transaction():
+        print "Setting data source to non nullable..."
+        migrator.set_nullable(models.Query, models.Query.data_source, False)
+
+    with db.database.transaction():
+        print "Setting data source to non nullable..."
+        migrator.set_nullable(models.QueryResult, models.QueryResult.data_source, False)
+
+    db.close_db(None)

migrations/create_users.py

@@ -0,0 +1,56 @@
+import json
+import itertools
+import peewee
+from playhouse.migrate import Migrator
+from redash import db, settings
+from redash import models
+
+if __name__ == '__main__':
+    db.connect_db()
+
+    if not models.User.table_exists():
+        print "Creating user table..."
+        models.User.create_table()
+
+    migrator = Migrator(db.database)
+    with db.database.transaction():
+        print "Creating user field on dashboard and queries..."
+        try:
+            migrator.rename_column(models.Query, '"user"', "user_email")
+            migrator.rename_column(models.Dashboard, '"user"', "user_email")
+        except peewee.ProgrammingError:
+            print "Failed to rename user column -- assuming it already exists"
+
+    with db.database.transaction():
+        models.Query.user.null = True
+        models.Dashboard.user.null = True
+
+        try:
+            migrator.add_column(models.Query, models.Query.user, "user_id")
+            migrator.add_column(models.Dashboard, models.Dashboard.user, "user_id")
+        except peewee.ProgrammingError:
+            print "Failed to create user_id column -- assuming it already exists"
+
+    print "Creating user for all queries and dashboards..."
+    for obj in itertools.chain(models.Query.select(), models.Dashboard.select()):
+        # Some old databases might have queries with empty string as user email:
+        email = obj.user_email or settings.ADMINS[0]
+        email = email.split(',')[0]
+
+        print ".. {} , {}, {}".format(type(obj), obj.id, email)
+
+        try:
+            user = models.User.get(models.User.email == email)
+        except models.User.DoesNotExist:
+            is_admin = email in settings.ADMINS
+            user = models.User.create(email=email, name=email, is_admin=is_admin)
+
+        obj.user = user
+        obj.save()
+
+    print "Set user_id to non null..."
+    with db.database.transaction():
+        migrator.set_nullable(models.Query, models.Query.user, False)
+        migrator.set_nullable(models.Dashboard, models.Dashboard.user, False)
+        migrator.set_nullable(models.Query, models.Query.user_email, True)
+        migrator.set_nullable(models.Dashboard, models.Dashboard.user_email, True)

migrations/create_visualizations.py

@@ -0,0 +1,70 @@
+import json
+from playhouse.migrate import Migrator
+from redash import db
+from redash import models
+
+if __name__ == '__main__':
+    default_options = {"series": {"type": "column"}}
+
+    db.connect_db()
+
+    if not models.Visualization.table_exists():
+        print "Creating visualization table..."
+        models.Visualization.create_table()
+
+    with db.database.transaction():
+        migrator = Migrator(db.database)
+        print "Adding visualization_id to widgets:"
+        field = models.Widget.visualization
+        field.null = True
+        migrator.add_column(models.Widget, models.Widget.visualization, 'visualization_id')
+
+    print 'Creating TABLE visualizations for all queries...'
+    for query in models.Query.select():
+        vis = models.Visualization(query=query, name="Table",
+                                   description=query.description or "",
+                                   type="TABLE", options="{}")
+        vis.save()
+
+    print 'Creating COHORT visualizations for all queries named like %cohort%...'
+    for query in models.Query.select().where(models.Query.name ** "%cohort%"):
+        vis = models.Visualization(query=query, name="Cohort",
+                                   description=query.description or "",
+                                   type="COHORT", options="{}")
+        vis.save()
+
+    print 'Create visualization for all widgets (unless exists already):'
+    for widget in models.Widget.select():
+        print 'Processing widget id: %d:' % widget.id
+        vis_type = widget.type.upper()
+        if vis_type == 'GRID':
+            vis_type = 'TABLE'
+
+        query = models.Query.get_by_id(widget.query_id)
+        vis = query.visualizations.where(models.Visualization.type == vis_type).first()
+        if vis:
+            print '... visualization type (%s) found.' % vis_type
+            widget.visualization = vis
+            widget.save()
+        else:
+            vis_name = vis_type.title()
+
+            options = json.loads(widget.options)
+            vis_options = {"series": options} if options else default_options
+            vis_options = json.dumps(vis_options)
+
+            vis = models.Visualization(query=query, name=vis_name,
+                                       description=query.description or "",
+                                       type=vis_type, options=vis_options)
+
+            print '... Created visualization for type: %s' % vis_type
+            vis.save()
+            widget.visualization = vis
+            widget.save()
+
+    with db.database.transaction():
+        migrator = Migrator(db.database)
+        print "Setting visualization_id as not null..."
+        migrator.set_nullable(models.Widget, models.Widget.visualization, False)
+
+    db.close_db(None)

rd_service/__init__.py

@@ -1 +0,0 @@
-

rd_service/cli.py

@@ -1,59 +0,0 @@
-"""
-CLI to start the workers.
-
-TODO: move API server startup here.
-"""
-import atfork
-atfork.monkeypatch_os_fork_functions()
-import atfork.stdlib_fixer
-atfork.stdlib_fixer.fix_logging_module()
-
-import argparse
-import logging
-import urlparse
-import redis
-import time
-import settings
-import data
-
-
-def start_workers(data_manager):
-    try:
-        old_workers = data_manager.redis_connection.smembers('workers')
-        data_manager.redis_connection.delete('workers')
-
-        logging.info("Cleaning old workers: %s", old_workers)
-
-        data_manager.start_workers(settings.WORKERS_COUNT, settings.CONNECTION_STRING)
-        logging.info("Workers started.")
-
-        while True:
-            try:
-                data_manager.refresh_queries()
-            except Exception as e:
-                logging.error("Something went wrong with refreshing queries...")
-                logging.exception(e)
-            time.sleep(60)
-    except KeyboardInterrupt:
-        logging.warning("Exiting; waiting for threads")
-        data_manager.stop_workers()
-
-
-if __name__ == '__main__':
-    channel = logging.StreamHandler()
-    logging.getLogger().addHandler(channel)
-    logging.getLogger().setLevel(settings.LOG_LEVEL)
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument("command")
-    args = parser.parse_args()
-
-    url = urlparse.urlparse(settings.REDIS_URL)
-    redis_connection = redis.StrictRedis(host=url.hostname, port=url.port, db=0, password=url.password)
-    data_manager = data.Manager(redis_connection, settings.INTERNAL_DB_CONNECTION_STRING, settings.MAX_CONNECTIONS)
-
-    if args.command == "worker":
-        start_workers(data_manager)
-    else:
-        print "Unknown command"
-

rd_service/data/manager.py

@@ -1,192 +0,0 @@
-"""
-Data manager. Used to manage and coordinate execution of queries.
-"""
-import collections
-from contextlib import contextmanager
-import json
-import logging
-import psycopg2
-import qr
-import redis
-import time
-import worker
-import settings
-from utils import gen_query_hash
-
-
-class QueryResult(collections.namedtuple('QueryData', 'id query data runtime retrieved_at query_hash')):
-    def to_dict(self, parse_data=False):
-        d = self._asdict()
-
-        if parse_data and d['data']:
-            d['data'] = json.loads(d['data'])
-
-        return d
-
-
-class Manager(object):
-    def __init__(self, redis_connection, db_connection_string, db_max_connections):
-        self.redis_connection = redis_connection
-        self.workers = []
-        self.db_connection_string = db_connection_string
-        self.queue = qr.PriorityQueue("jobs", **self.redis_connection.connection_pool.connection_kwargs)
-        self.max_retries = 5
-        self.status = {
-            'last_refresh_at': 0,
-            'started_at': time.time()
-        }
-
-        self._save_status()
-
-    # TODO: Use our Django Models
-    def get_query_result_by_id(self, query_result_id):
-        with self.db_transaction() as cursor:
-            sql = "SELECT id, query, data, runtime, retrieved_at, query_hash FROM query_results " \
-                  "WHERE id=%s LIMIT 1"
-            cursor.execute(sql, (query_result_id,))
-            query_result = cursor.fetchone()
-
-        if query_result:
-            query_result = QueryResult(*query_result)
-
-        return query_result
-
-    def get_query_result(self, query, ttl=0):
-        query_hash = gen_query_hash(query)
-
-        with self.db_transaction() as cursor:
-            sql = "SELECT id, query, data, runtime, retrieved_at, query_hash FROM query_results " \
-                  "WHERE query_hash=%s " \
-                  "AND retrieved_at < now() at time zone 'utc' - interval '%s second'" \
-                  "ORDER BY retrieved_at DESC LIMIT 1"
-            cursor.execute(sql, (query_hash, psycopg2.extensions.AsIs(ttl)))
-            query_result = cursor.fetchone()
-
-        if query_result:
-            query_result = QueryResult(*query_result)
-
-        return query_result
-
-    def add_job(self, query, priority):
-        query_hash = gen_query_hash(query)
-        logging.info("[Manager][%s] Inserting job with priority=%s", query_hash, priority)
-        try_count = 0
-        job = None
-
-        while try_count < self.max_retries:
-            try_count += 1
-
-            pipe = self.redis_connection.pipeline()
-            try:
-                pipe.watch('query_hash_job:%s' % query_hash)
-                job_id = pipe.get('query_hash_job:%s' % query_hash)
-                if job_id:
-                    logging.info("[Manager][%s] Found existing job: %s", query_hash, job_id)
-                    job = worker.Job.load(self.redis_connection, job_id)
-                else:
-                    job = worker.Job(self.redis_connection, query, priority)
-                    pipe.multi()
-                    job.save(pipe)
-                    logging.info("[Manager][%s] Created new job: %s", query_hash, job.id)
-                    self.queue.push(job.id, job.priority)
-                break
-
-            except redis.WatchError:
-                continue
-
-        if not job:
-            logging.error("[Manager][%s] Failed adding job for query.", query_hash)
-
-        return job
-
-    def refresh_queries(self):
-        sql = """SELECT queries.query, queries.ttl, retrieved_at
-        FROM (SELECT query, min(ttl) as ttl FROM queries  WHERE ttl > 0 GROUP by query) queries
-        JOIN (SELECT query, max(retrieved_at) as retrieved_at
-        FROM query_results
-        GROUP BY query) query_results on query_results.query=queries.query
-        WHERE queries.ttl > 0
-          AND query_results.retrieved_at + ttl * interval '1 second' < now() at time zone 'utc';"""
-
-        self.status['last_refresh_at'] = time.time()
-        self._save_status()
-
-        logging.info("Refreshing queries...")
-        queries = self.run_query(sql)
-        for query, ttl, retrieved_at in queries:
-            self.add_job(query, worker.Job.LOW_PRIORITY)
-
-        logging.info("Done refreshing queries... %d" % len(queries))
-
-    def store_query_result(self, query, data, run_time, retrieved_at):
-        query_result_id = None
-        query_hash = gen_query_hash(query)
-        sql = "INSERT INTO query_results (query_hash, query, data, runtime, retrieved_at) " \
-              "VALUES (%s, %s, %s, %s, %s) RETURNING id"
-        with self.db_transaction() as cursor:
-            cursor.execute(sql, (query_hash, query, data, run_time, retrieved_at))
-            if cursor.rowcount == 1:
-                query_result_id = cursor.fetchone()[0]
-                logging.info("[Manager][%s] Inserted query data; id=%s", query_hash, query_result_id)
-
-                sql = "UPDATE queries SET latest_query_data_id=%s WHERE query_hash=%s"
-                cursor.execute(sql, (query_result_id, query_hash))
-
-                logging.info("[Manager][%s] Updated %s queries.", query_hash, cursor.rowcount)
-            else:
-                logging.error("[Manager][%s] Failed inserting query data.", query_hash)
-        return query_result_id
-
-    def run_query(self, *args):
-        sql = args[0]
-        logging.debug("running query: %s %s", sql, args[1:])
-
-        with self.db_transaction() as cursor:
-            cursor.execute(sql, args[1:])
-            if cursor.description:
-                data = list(cursor)
-            else:
-                data = cursor.rowcount
-
-        return data
-
-    def start_workers(self, workers_count, connection_string):
-        if self.workers:
-            return self.workers
-        
-        if settings.CONNECTION_ADAPTER == "mysql":
-            import query_runner_mysql            
-            runner = query_runner_mysql.mysql(connection_string)
-        else:
-            import query_runner            
-            runner = query_runner.redshift(connection_string)
-
-        redis_connection_params = self.redis_connection.connection_pool.connection_kwargs
-        self.workers = [worker.Worker(self, redis_connection_params, runner)
-                        for _ in range(workers_count)]
-        for w in self.workers:
-            w.start()
-
-        return self.workers
-
-    def stop_workers(self):
-        for w in self.workers:
-            w.continue_working = False
-            w.join()
-
-    @contextmanager
-    def db_transaction(self):
-        connection = psycopg2.connect(self.db_connection_string)
-        cursor = connection.cursor()
-        try:
-            yield cursor
-        except:
-            connection.rollback()
-            raise
-        else:
-            connection.commit()
-        finally:
-            connection.close()
-
-    def _save_status(self):
-        self.redis_connection.hmset('manager:status', self.status)

rd_service/data/models.py

@@ -1,210 +0,0 @@
-"""
-Django ORM based models to describe the data model of re:dash.
-"""
-import hashlib
-import json
-import time
-from django.db import models
-from django.template.defaultfilters import slugify
-import utils
-
-
-class QueryResult(models.Model):
-    id = models.AutoField(primary_key=True)
-    query_hash = models.CharField(max_length=32)
-    query = models.TextField()
-    data = models.TextField()
-    runtime = models.FloatField()
-    retrieved_at = models.DateTimeField()
-
-    class Meta:
-        app_label = 'redash'
-        db_table = 'query_results'
-
-    def to_dict(self):
-        return {
-            'id': self.id,
-            'query_hash': self.query_hash,
-            'query': self.query,
-            'data': json.loads(self.data),
-            'runtime': self.runtime,
-            'retrieved_at': self.retrieved_at
-        }
-
-    def __unicode__(self):
-        return u"%d | %s | %s" % (self.id, self.query_hash, self.retrieved_at)
-
-
-class Query(models.Model):
-    id = models.AutoField(primary_key=True)
-    latest_query_data = models.ForeignKey(QueryResult)
-    name = models.CharField(max_length=255)
-    description = models.CharField(max_length=4096)
-    query = models.TextField()
-    query_hash = models.CharField(max_length=32)
-    api_key = models.CharField(max_length=40)
-    ttl = models.IntegerField()
-    user = models.CharField(max_length=360)
-    created_at = models.DateTimeField(auto_now_add=True)
-
-    class Meta:
-        app_label = 'redash'
-        db_table = 'queries'
-
-    def to_dict(self, with_result=True, with_stats=False,
-                with_visualizations=False):
-        d = {
-            'id': self.id,
-            'latest_query_data_id': self.latest_query_data_id,
-            'name': self.name,
-            'description': self.description,
-            'query': self.query,
-            'query_hash': self.query_hash,
-            'ttl': self.ttl,
-            'user': self.user,
-            'api_key': self.api_key,
-            'created_at': self.created_at,
-        }
-
-        if with_stats:
-            d['avg_runtime'] = self.avg_runtime
-            d['min_runtime'] = self.min_runtime
-            d['max_runtime'] = self.max_runtime
-            d['last_retrieved_at'] = self.last_retrieved_at
-            d['times_retrieved'] = self.times_retrieved
-
-        if with_result and self.latest_query_data_id:
-            d['latest_query_data'] = self.latest_query_data.to_dict()
-
-        if with_visualizations:
-            d['visualizations'] = [vis.to_dict(with_query=False)
-                                   for vis in self.visualizations.all()]
-
-        return d
-
-    @classmethod
-    def all_queries(cls):
-        query = """SELECT queries.*, query_stats.*
-FROM queries
-LEFT OUTER JOIN
-  (SELECT qu.query_hash,
-          count(0) AS "times_retrieved",
-          avg(runtime) AS "avg_runtime",
-          min(runtime) AS "min_runtime",
-          max(runtime) AS "max_runtime",
-          max(retrieved_at) AS "last_retrieved_at"
-   FROM queries qu
-   JOIN query_results qr ON qu.query_hash=qr.query_hash
-   GROUP BY qu.query_hash) query_stats ON query_stats.query_hash = queries.query_hash
-        """
-        return cls.objects.raw(query)
-
-    def save(self, *args, **kwargs):
-        self.query_hash = utils.gen_query_hash(self.query)
-        self._set_api_key()
-        super(Query, self).save(*args, **kwargs)
-
-    def _set_api_key(self):
-        if not self.api_key:
-            self.api_key = hashlib.sha1(
-                u''.join([str(time.time()), self.query, self.user, self.name])).hexdigest()
-
-    def __unicode__(self):
-        return unicode(self.id)
-
-
-class Dashboard(models.Model):
-    id = models.AutoField(primary_key=True)
-    slug = models.CharField(max_length=140)
-    name = models.CharField(max_length=100)
-    user = models.CharField(max_length=360)
-    layout = models.TextField()
-    is_archived = models.BooleanField(default=False)
-
-    class Meta:
-        app_label = 'redash'
-        db_table = 'dashboards'
-
-    def to_dict(self, with_widgets=False):
-        layout = json.loads(self.layout)
-
-        if with_widgets:
-            widgets = {w.id: w.to_dict() for w in self.widgets.all()}
-            widgets_layout = map(lambda row: map(lambda widget_id: widgets.get(widget_id, None), row), layout)
-        else:
-            widgets_layout = None
-
-        return {
-            'id': self.id,
-            'slug': self.slug,
-            'name': self.name,
-            'user': self.user,
-            'layout': layout,
-            'widgets': widgets_layout
-        }
-
-    def save(self, *args, **kwargs):
-        # TODO: make sure slug is unique
-        if not self.slug:
-            self.slug = slugify(self.name)
-        super(Dashboard, self).save(*args, **kwargs)
-
-    def __unicode__(self):
-        return u"%s=%s" % (self.id, self.name)
-
-
-class Visualization(models.Model):
-    id = models.AutoField(primary_key=True)
-    type = models.CharField(max_length=100)
-    query = models.ForeignKey(Query, related_name='visualizations')
-    name = models.CharField(max_length=255)
-    description = models.CharField(max_length=4096)
-    options = models.TextField()
-
-    class Meta:
-        app_label = 'redash'
-        db_table = 'visualizations'
-
-    def to_dict(self, with_query=True):
-        d = {
-            'id': self.id,
-            'type': self.type,
-            'name': self.name,
-            'description': self.description,
-            'options': json.loads(self.options),
-        }
-
-        if with_query:
-            d['query'] = self.query.to_dict()
-
-        return d
-
-    def __unicode__(self):
-        return u"%s=>%s" % (self.id, self.query_id)
-
-
-class Widget(models.Model):
-    id = models.AutoField(primary_key=True)
-    type = models.CharField(max_length=100)
-    query = models.ForeignKey(Query, related_name='widgets')
-    visualization = models.ForeignKey(Visualization, related_name='widgets')
-    width = models.IntegerField()
-    options = models.TextField()
-    dashboard = models.ForeignKey(Dashboard, related_name='widgets')
-
-    class Meta:
-        app_label = 'redash'
-        db_table = 'widgets'
-
-    def to_dict(self):
-        return {
-            'id': self.id,
-            'type': self.type,
-            'width': self.width,
-            'options': json.loads(self.options),
-            'visualization': self.visualization.to_dict(),
-            'dashboard_id': self.dashboard_id
-        }
-
-    def __unicode__(self):
-        return u"%s=>%s" % (self.id, self.dashboard_id)

rd_service/data/tables.sql

@@ -1,56 +0,0 @@
-BEGIN;
-CREATE TABLE "query_results" (
-    "id" serial NOT NULL PRIMARY KEY,
-    "query_hash" varchar(32) NOT NULL,
-    "query" text NOT NULL,
-    "data" text NOT NULL,
-    "runtime" double precision NOT NULL,
-    "retrieved_at" timestamp with time zone NOT NULL
-)
-;
-CREATE TABLE "queries" (
-    "id" serial NOT NULL PRIMARY KEY,
-    "latest_query_data_id" integer REFERENCES "query_results" ("id") DEFERRABLE INITIALLY DEFERRED,
-    "name" varchar(255) NOT NULL,
-    "description" varchar(4096),
-    "query" text NOT NULL,
-    "query_hash" varchar(32) NOT NULL,
-    "api_key" varchar(40),
-    "ttl" integer NOT NULL,
-    "user" varchar(360) NOT NULL,
-    "created_at" timestamp with time zone NOT NULL
-)
-;
-CREATE TABLE "dashboards" (
-    "id" serial NOT NULL PRIMARY KEY,
-    "slug" varchar(140) NOT NULL,
-    "name" varchar(100) NOT NULL,
-    "user" varchar(360) NOT NULL,
-    "layout" text NOT NULL,
-    "is_archived" boolean NOT NULL
-)
-;
-CREATE TABLE "visualizations" (
-    "id" serial NOT NULL PRIMARY KEY,
-    "type" varchar(100) NOT NULL,
-    "query_id" integer NOT NULL REFERENCES "queries" ("id") DEFERRABLE INITIALLY DEFERRED,
-    "name" varchar(255) NOT NULL,
-    "description" varchar(4096),
-    "options" text NOT NULL
-)
-;
-CREATE TABLE "widgets" (
-    "id" serial NOT NULL PRIMARY KEY,
-    "type" varchar(100) NOT NULL,
-    "width" integer NOT NULL,
-    "options" text NOT NULL,
-    "query_id" integer,
-    "visualization_id" integer NOT NULL REFERENCES "visualizations" ("id") DEFERRABLE INITIALLY DEFERRED,
-    "dashboard_id" integer NOT NULL REFERENCES "dashboards" ("id") DEFERRABLE INITIALLY DEFERRED
-)
-;
-CREATE INDEX "queries_latest_query_data_id" ON "queries" ("latest_query_data_id");
-CREATE INDEX "widgets_query_id" ON "widgets" ("query_id");
-CREATE INDEX "widgets_dashboard_id" ON "widgets" ("dashboard_id");
-
-COMMIT;

rd_service/data/worker.py

@@ -1,255 +0,0 @@
-"""
-Worker implementation to execute incoming queries.
-"""
-import json
-import logging
-import os
-import threading
-import uuid
-import datetime
-import time
-import signal
-import setproctitle
-import redis
-from utils import gen_query_hash
-
-
-class Job(object):
-    HIGH_PRIORITY = 1
-    LOW_PRIORITY = 2
-
-    WAITING = 1
-    PROCESSING = 2
-    DONE = 3
-    FAILED = 4
-
-    def __init__(self, redis_connection, query, priority,
-                 job_id=None,
-                 wait_time=None, query_time=None,
-                 updated_at=None, status=None, error=None, query_result_id=None,
-                 process_id=0):
-        self.redis_connection = redis_connection
-        self.query = query
-        self.priority = priority
-        self.query_hash = gen_query_hash(self.query)
-        self.query_result_id = query_result_id
-        if process_id == 'None':
-            self.process_id = None
-        else:
-            self.process_id = int(process_id)
-
-        if job_id is None:
-            self.id = str(uuid.uuid1())
-            self.new_job = True
-            self.wait_time = 0
-            self.query_time = 0
-            self.error = None
-            self.updated_at = time.time() # job_dict.get('updated_at', time.time())
-            self.status = self.WAITING # int(job_dict.get('status', self.WAITING))
-        else:
-            self.id = job_id
-            self.new_job = False
-            self.error = error
-            self.wait_time = wait_time
-            self.query_time = query_time
-            self.updated_at = updated_at
-            self.status = status
-
-    def to_dict(self):
-        return {
-            'query': self.query,
-            'priority': self.priority,
-            'id': self.id,
-            'wait_time': self.wait_time,
-            'query_time': self.query_time,
-            'updated_at': self.updated_at,
-            'status': self.status,
-            'error': self.error,
-            'query_result_id': self.query_result_id,
-            'process_id': self.process_id
-        }
-
-    @staticmethod
-    def _redis_key(job_id):
-        return 'job:%s' % job_id
-
-    def cancel(self):
-        # TODO: Race condition:
-        # it's possible that it will be picked up by worker while processing the cancel order
-        if self.is_finished():
-            return
-
-        if self.status == self.PROCESSING:
-            os.kill(self.process_id, signal.SIGINT)
-        else:
-            self.done(None, "Interrupted/Cancelled while running.")
-
-    def save(self, pipe=None):
-        if not pipe:
-            pipe = self.redis_connection.pipeline()
-
-        if self.new_job:
-            pipe.set('query_hash_job:%s' % self.query_hash, self.id)
-
-        if self.is_finished():
-            pipe.delete('query_hash_job:%s' % self.query_hash)
-
-        pipe.sadd('jobs_set', self.id)
-        pipe.hmset(self._redis_key(self.id), self.to_dict())
-        pipe.publish(self._redis_key(self.id), json.dumps(self.to_dict()))
-        pipe.execute()
-
-    def processing(self, process_id):
-        self.status = self.PROCESSING
-        self.process_id = process_id
-        self.wait_time = time.time() - self.updated_at
-        self.updated_at = time.time()
-        self.save()
-
-    def is_finished(self):
-        return self.status in (self.FAILED, self.DONE)
-
-    def done(self, query_result_id, error):
-        if error:
-            self.status = self.FAILED
-        else:
-            self.status = self.DONE
-
-        self.query_result_id = query_result_id
-        self.error = error
-        self.query_time = time.time() - self.updated_at
-        self.updated_at = time.time()
-        self.save()
-
-    def __str__(self):
-        return "<Job:%s,priority:%d,status:%d>" % (self.id, self.priority, self.status)
-
-    @classmethod
-    def _load(cls, redis_connection, job_id):
-        return redis_connection.hgetall(cls._redis_key(job_id))
-
-    @classmethod
-    def load(cls, redis_connection, job_id):
-        job_dict = cls._load(redis_connection, job_id)
-        job = None
-        if job_dict:
-            job = Job(redis_connection, job_id=job_dict['id'], query=job_dict['query'].decode('utf-8'),
-                      priority=int(job_dict['priority']), updated_at=float(job_dict['updated_at']),
-                      status=int(job_dict['status']), wait_time=float(job_dict['wait_time']),
-                      query_time=float(job_dict['query_time']), error=job_dict['error'],
-                      query_result_id=job_dict['query_result_id'],
-                      process_id=job_dict['process_id'])
-
-        return job
-
-
-class Worker(threading.Thread):
-    def __init__(self, manager, redis_connection_params, query_runner, sleep_time=0.1):
-        self.manager = manager
-
-        self.redis_connection_params = {k: v for k, v in redis_connection_params.iteritems()
-                                        if k in ('host', 'db', 'password', 'port')}
-        self.continue_working = True
-        self.query_runner = query_runner
-        self.sleep_time = sleep_time
-        self.child_pid = None
-        self.worker_id = uuid.uuid1()
-        self.status = {
-            'jobs_count': 0,
-            'cancelled_jobs_count': 0,
-            'done_jobs_count': 0,
-            'updated_at': time.time(),
-            'started_at': time.time()
-        }
-        self._save_status()
-        self.manager.redis_connection.sadd('workers', self._key)
-
-        super(Worker, self).__init__(name="Worker-%s" % self.worker_id)
-
-    def set_title(self, title=None):
-        base_title = "redash worker:%s" % self.worker_id
-        if title:
-            full_title = "%s - %s" % (base_title, title)
-        else:
-            full_title = base_title
-
-        setproctitle.setproctitle(full_title)
-
-    def run(self):
-        logging.info("[%s] started.", self.name)
-        while self.continue_working:
-            job_id = self.manager.queue.pop()
-            if job_id:
-                self._update_status('jobs_count')
-                logging.info("[%s] Processing %s", self.name, job_id)
-                self._fork_and_process(job_id)
-                if self.child_pid == 0:
-                    return
-            else:
-                time.sleep(self.sleep_time)
-
-    def _update_status(self, counter):
-        self.status['updated_at'] = time.time()
-        self.status[counter] += 1
-        self._save_status()
-
-    @property
-    def _key(self):
-        return 'worker:%s' % self.worker_id
-
-    def _save_status(self):
-        self.manager.redis_connection.hmset(self._key, self.status)
-
-    def _fork_and_process(self, job_id):
-        self.child_pid = os.fork()
-        if self.child_pid == 0:
-            self.set_title("processing %s" % job_id)
-            self._process(job_id)
-        else:
-            logging.info("[%s] Waiting for pid: %d", self.name, self.child_pid)
-            _, status = os.waitpid(self.child_pid, 0)
-            self._update_status('done_jobs_count')
-            if status > 0:
-                job = Job.load(self.manager.redis_connection, job_id)
-                if not job.is_finished():
-                    self._update_status('cancelled_jobs_count')
-                    logging.info("[%s] process interrupted and job %s hasn't finished; registering interruption in job",
-                                 self.name, job_id)
-                    job.done(None, "Interrupted/Cancelled while running.")
-
-            logging.info("[%s] Finished Processing %s (pid: %d status: %d)",
-                         self.name, job_id, self.child_pid, status)
-
-    def _process(self, job_id):
-        redis_connection = redis.StrictRedis(**self.redis_connection_params)
-        job = Job.load(redis_connection, job_id)
-        if job.is_finished():
-            logging.warning("[%s][%s] tried to process finished job.", self.name, job)
-            return
-
-        pid = os.getpid()
-        job.processing(pid)
-
-        logging.info("[%s][%s] running query...", self.name, job.id)
-        start_time = time.time()
-        self.set_title("running query %s" % job_id)
-
-        annotated_query = "/* Pid: %s, Job Id: %s, Query hash: %s, Priority: %s */ %s" % \
-                          (pid, job.id, job.query_hash, job.priority, job.query)
-
-        # TODO: here's the part that needs to be forked, not all of the worker process...
-        data, error = self.query_runner(annotated_query)
-        run_time = time.time() - start_time
-        logging.info("[%s][%s] query finished... data length=%s, error=%s",
-                     self.name, job.id, data and len(data), error)
-
-        # TODO: it is possible that storing the data will fail, and we will need to retry
-        # while we already marked the job as done
-        query_result_id = None
-        if not error:
-            self.set_title("storing results %s" % job_id)
-            query_result_id = self.manager.store_query_result(job.query, data, run_time,
-                                                              datetime.datetime.utcnow())
-
-        self.set_title("marking job as done %s" % job_id)
-        job.done(query_result_id, error)

rd_service/migrate.py

@@ -1,50 +0,0 @@
-import json
-import settings
-from data.models import *
-
-# first run:
-
-# CREATE TABLE "visualizations" (
-#     "id" serial NOT NULL PRIMARY KEY,
-#     "type" varchar(100) NOT NULL,
-#     "query_id" integer NOT NULL REFERENCES "queries" ("id") DEFERRABLE INITIALLY DEFERRED,
-#     "name" varchar(255) NOT NULL,
-#     "description" varchar(4096),
-#     "options" text NOT NULL
-# )
-# ;
-
-# ALTER TABLE widgets ADD COLUMN "visualization_id" integer REFERENCES "visualizations" ("id") DEFERRABLE INITIALLY DEFERRED;
-
-if __name__ == '__main__':
-    print 'migrating Widgets -> Visualizations ...'
-
-    for query in Query.objects.filter(name__icontains="cohort"):
-        vis = Visualization(query=query, name=query.name,
-                            description=query.description,
-                            type="COHORT", options="{}")
-        vis.save()
-
-
-    for widget in Widget.objects.all():
-        print 'processing widget %d' % widget.id
-        query = widget.query
-        vis_type = widget.type.upper()
-
-        vis = query.visualizations.filter(type=vis_type)
-        if vis:
-            print 'found'
-            widget.visualization = vis[0]
-            widget.save()
-
-        else:
-            options = json.loads(widget.options)
-            vis_options = {"series": options} if options else {}
-            vis_options = json.dumps(vis_options)
-
-            vis = Visualization(query=query, name=query.name,
-                                            description=query.description,
-                                            type=vis_type, options=vis_options)
-            vis.save()
-            widget.visualization = vis
-            widget.save()

rd_service/requirements.txt

@@ -1,10 +0,0 @@
-psycopg2==2.5.1
-redis==2.7.5
-tornado==3.0.2
-sqlparse==0.1.8
-Django==1.5.4
-django-db-pool==0.0.10
-qr==0.6.0
-python-dateutil==2.1
-setproctitle==1.1.8
-atfork==0.1.2

rd_service/server.py

@@ -1,381 +0,0 @@
-"""
-Tornado based API implementation for re:dash.
-
-Also at the moment the Tornado server is used to serve the static assets (and the Angular.js app),
-but this is only due to configuration issues and temporary.
-
-Usage:
-    python server.py [--port=8888] [--debug] [--static=..]
-
-    port - port to listen to
-    debug - enable debug mode (extensive logging, restart on code change)
-    static - static assets path
-
-If static option isn't specified it will be taken from settings.py.
-"""
-import csv
-import hashlib
-import json
-import numbers
-import os
-import urlparse
-import logging
-import cStringIO
-import datetime
-import dateutil.parser
-import redis
-import sqlparse
-import tornado.ioloop
-import tornado.web
-import tornado.auth
-import tornado.options
-import settings
-import time
-from data import utils
-import data
-
-
-class BaseHandler(tornado.web.RequestHandler):
-    def initialize(self):
-        self.data_manager = self.application.settings.get('data_manager', None)
-        self.redis_connection = self.application.settings['redis_connection']
-
-    def get_current_user(self):
-        user = self.get_secure_cookie("user")
-        return user
-
-    def write_json(self, response, encode=True):
-        if encode:
-            response = json.dumps(response, cls=utils.JSONEncoder)
-        self.set_header("Content-Type", "application/json; charset=UTF-8")
-        self.write(response)
-
-
-class BaseAuthenticatedHandler(BaseHandler):
-    @tornado.web.authenticated
-    def prepare(self):
-        pass
-
-
-class PingHandler(tornado.web.RequestHandler):
-    def get(self):
-        self.write("PONG")
-
-
-class GoogleLoginHandler(tornado.web.RequestHandler,
-                         tornado.auth.GoogleMixin):
-    @tornado.web.asynchronous
-    @tornado.gen.coroutine
-    def get(self):
-        if self.get_argument("openid.mode", None):
-            user = yield self.get_authenticated_user()
-
-            if user['email'] in settings.ALLOWED_USERS or user['email'].endswith("@%s" % settings.GOOGLE_APPS_DOMAIN):
-                logging.info("Authenticated: %s", user['email'])
-                self.set_secure_cookie("user", user['email'])
-                self.redirect("/")
-            else:
-                logging.error("Failed logging in with: %s", user)
-                self.authenticate_redirect()
-        else:
-            self.authenticate_redirect()
-
-
-class MainHandler(BaseAuthenticatedHandler):
-    def get(self, *args):
-        email_md5 = hashlib.md5(self.current_user.lower()).hexdigest()
-        gravatar_url = "https://www.gravatar.com/avatar/%s?s=40" % email_md5
-
-        user = {
-            'gravatar_url': gravatar_url,
-            'is_admin': self.current_user in settings.ADMINS,
-            'name': self.current_user
-        }
-
-        self.render("index.html", user=json.dumps(user), analytics=settings.ANALYTICS)
-
-
-class QueryFormatHandler(BaseAuthenticatedHandler):
-    def post(self):
-        arguments = json.loads(self.request.body)
-        query = arguments.get("query", "")
-
-        self.write(sqlparse.format(query, reindent=True, keyword_case='upper'))
-
-
-class StatusHandler(BaseAuthenticatedHandler):
-    def get(self):
-        status = {}
-        info = self.redis_connection.info()
-        status['redis_used_memory'] = info['used_memory_human']
-
-        status['queries_count'] = data.models.Query.objects.count()
-        status['query_results_count'] = data.models.QueryResult.objects.count()
-        status['dashboards_count'] = data.models.Dashboard.objects.count()
-        status['widgets_count'] = data.models.Widget.objects.count()
-
-        status['workers'] = [self.redis_connection.hgetall(w)
-                             for w in self.redis_connection.smembers('workers')]
-
-        manager_status = self.redis_connection.hgetall('manager:status')
-        status['manager'] = manager_status
-        status['manager']['queue_size'] = self.redis_connection.zcard('jobs')
-
-        self.write_json(status)
-
-
-class WidgetsHandler(BaseAuthenticatedHandler):
-    def post(self, widget_id=None):
-        widget_properties = json.loads(self.request.body)
-        widget_properties['options'] = json.dumps(widget_properties['options'])
-        widget = data.models.Widget(**widget_properties)
-        widget.save()
-
-        layout = json.loads(widget.dashboard.layout)
-        new_row = True
-
-        if len(layout) == 0 or widget.width == 2:
-            layout.append([widget.id])
-        elif len(layout[-1]) == 1:
-            neighbour_widget = data.models.Widget.objects.get(pk=layout[-1][0])
-            if neighbour_widget.width == 1:
-                layout[-1].append(widget.id)
-                new_row = False
-            else:
-                layout.append([widget.id])
-        else:
-            layout.append([widget.id])
-
-        widget.dashboard.layout = json.dumps(layout)
-        widget.dashboard.save()
-
-        self.write_json({'widget': widget.to_dict(), 'layout': layout, 'new_row': new_row})
-
-    def delete(self, widget_id):
-        widget_id = int(widget_id)
-        widget = data.models.Widget.objects.get(pk=widget_id)
-        # TODO: reposition existing ones
-        layout = json.loads(widget.dashboard.layout)
-        layout = map(lambda row: filter(lambda w: w != widget_id, row), layout)
-        layout = filter(lambda row: len(row) > 0, layout)
-        widget.dashboard.layout = json.dumps(layout)
-        widget.dashboard.save()
-
-        widget.delete()
-
-
-class DashboardHandler(BaseAuthenticatedHandler):
-    def get(self, dashboard_slug=None):
-        if dashboard_slug:
-            dashboard = data.models.Dashboard.objects.prefetch_related('widgets__visualization__query__latest_query_data').get(slug=dashboard_slug)
-            self.write_json(dashboard.to_dict(with_widgets=True))
-        else:
-            dashboards = [d.to_dict() for d in
-                          data.models.Dashboard.objects.filter(is_archived=False)]
-            self.write_json(dashboards)
-
-    def post(self, dashboard_id):
-        if dashboard_id:
-            dashboard_properties = json.loads(self.request.body)
-            dashboard = data.models.Dashboard.objects.get(pk=dashboard_id)
-            dashboard.layout = dashboard_properties['layout']
-            dashboard.name = dashboard_properties['name']
-            dashboard.save()
-
-            self.write_json(dashboard.to_dict(with_widgets=True))
-        else:
-            dashboard_properties = json.loads(self.request.body)
-            dashboard = data.models.Dashboard(name=dashboard_properties['name'],
-                                                    user=self.current_user,
-                                                    layout='[]')
-            dashboard.save()
-            self.write_json(dashboard.to_dict())
-
-    def delete(self, dashboard_slug):
-        dashboard = data.models.Dashboard.objects.get(slug=dashboard_slug)
-        dashboard.is_archived = True
-        dashboard.save()
-
-
-class QueriesHandler(BaseAuthenticatedHandler):
-    def post(self, id=None):
-        query_def = json.loads(self.request.body)
-        if 'created_at' in query_def:
-            query_def['created_at'] = dateutil.parser.parse(query_def['created_at'])
-
-        query_def.pop('latest_query_data', None)
-        query_def.pop('visualizations', None)
-
-        if id:
-            query = data.models.Query(**query_def)
-            fields = query_def.keys()
-            fields.remove('id')
-            query.save(update_fields=fields)
-        else:
-            query_def['user'] = self.current_user
-            query = data.models.Query(**query_def)
-            query.save()
-
-        self.write_json(query.to_dict(with_result=False))
-
-    def get(self, id=None):
-        if id:
-            q = data.models.Query.objects.get(pk=id)
-            if q:
-                self.write_json(q.to_dict(with_visualizations=True))
-            else:
-                self.send_error(404)
-        else:
-            self.write_json([q.to_dict(with_result=False, with_stats=True) for q in data.models.Query.all_queries()])
-
-
-class QueryResultsHandler(BaseAuthenticatedHandler):
-    def get(self, query_result_id):
-        query_result = self.data_manager.get_query_result_by_id(query_result_id)
-        if query_result:
-            self.write_json({'query_result': query_result.to_dict(parse_data=True)})
-        else:
-            self.send_error(404)
-
-    def post(self, _):
-        params = json.loads(self.request.body)
-
-        if params['ttl'] == 0:
-            query_result = None
-        else:
-            query_result = self.data_manager.get_query_result(params['query'], int(params['ttl']))
-
-        if query_result:
-            self.write_json({'query_result': query_result.to_dict(parse_data=True)})
-        else:
-            job = self.data_manager.add_job(params['query'], data.Job.HIGH_PRIORITY)
-            self.write({'job': job.to_dict()})
-
-
-class VisualizationHandler(BaseAuthenticatedHandler):
-    def get(self, id):
-        pass
-
-    def post(self, id=None):
-        kwargs = json.loads(self.request.body)
-        kwargs['options'] = json.dumps(kwargs['options'])
-
-        if id:
-            vis = data.models.Visualization(**kwargs)
-            fields = kwargs.keys()
-            fields.remove('id')
-            vis.save(update_fields=fields)
-        else:
-            vis = data.models.Visualization(**kwargs)
-            vis.save()
-
-        self.write_json(vis.to_dict(with_query=False))
-
-    def delete(self, id):
-        vis = data.models.Visualization.objects.get(pk=id)
-        vis.delete()
-
-
-class CsvQueryResultsHandler(BaseAuthenticatedHandler):
-    def get_current_user(self):
-        user = super(CsvQueryResultsHandler, self).get_current_user()
-        if not user:
-            api_key = self.get_argument("api_key", None)
-            query = data.models.Query.objects.get(pk=self.path_args[0])
-
-            if query.api_key and query.api_key == api_key:
-                user = "API-Key=%s" % api_key
-
-        return user
-
-    def get(self, query_id, result_id=None):
-        if not result_id:
-            query = data.models.Query.objects.get(pk=query_id)
-            if query:
-                result_id = query.latest_query_data_id
-
-        query_result = result_id and self.data_manager.get_query_result_by_id(result_id)
-        if query_result:
-            self.set_header("Content-Type", "text/csv; charset=UTF-8")
-            s = cStringIO.StringIO()
-
-            query_data = json.loads(query_result.data)
-            writer = csv.DictWriter(s, fieldnames=[col['name'] for col in query_data['columns']])
-            writer.writer = utils.UnicodeWriter(s)
-            writer.writeheader()
-            for row in query_data['rows']:
-                for k, v in row.iteritems():
-                    if isinstance(v, numbers.Number) and (v > 1000 * 1000 * 1000 * 100):
-                        row[k] = datetime.datetime.fromtimestamp(v/1000.0)
-
-                writer.writerow(row)
-
-            self.write(s.getvalue())
-        else:
-            self.send_error(404)
-
-
-class JobsHandler(BaseAuthenticatedHandler):
-    def get(self, job_id=None):
-        if job_id:
-            # TODO: if finished, include the query result
-            job = data.Job.load(self.data_manager.redis_connection, job_id)
-            self.write({'job': job.to_dict()})
-        else:
-            raise NotImplemented
-
-    def delete(self, job_id):
-        job = data.Job.load(self.data_manager.redis_connection, job_id)
-        job.cancel()
-
-
-def get_application(static_path, is_debug, redis_connection, data_manager):
-    return tornado.web.Application([(r"/", MainHandler),
-                                    (r"/ping", PingHandler),
-                                    (r"/api/queries/([0-9]*)/results(?:/([0-9]*))?.csv", CsvQueryResultsHandler),
-                                    (r"/api/queries/format", QueryFormatHandler),
-                                    (r"/api/queries(?:/([0-9]*))?", QueriesHandler),
-                                    (r"/api/query_results(?:/([0-9]*))?", QueryResultsHandler),
-                                    (r"/api/jobs/(.*)", JobsHandler),
-                                    (r"/api/visualizations(?:/([0-9]*))?", VisualizationHandler),
-                                    (r"/api/widgets(?:/([0-9]*))?", WidgetsHandler),
-                                    (r"/api/dashboards(?:/(.*))?", DashboardHandler),
-                                    (r"/admin/(.*)", MainHandler),
-                                    (r"/dashboard/(.*)", MainHandler),
-                                    (r"/queries(.*)", MainHandler),
-                                    (r"/login", GoogleLoginHandler),
-                                    (r"/status.json", StatusHandler),
-                                    (r"/(.*)", tornado.web.StaticFileHandler,
-                                     {"path": static_path})],
-                                   template_path=static_path,
-                                   static_path=static_path,
-                                   debug=is_debug,
-                                   login_url="/login",
-                                   cookie_secret=settings.COOKIE_SECRET,
-                                   redis_connection=redis_connection,
-                                   data_manager=data_manager)
-
-
-if __name__ == '__main__':
-    tornado.options.define("port", default=8888, type=int)
-    tornado.options.define("debug", default=False, type=bool)
-    tornado.options.define("static", default=settings.STATIC_ASSETS_PATH, type=str)
-
-    tornado.options.parse_command_line()
-
-    root_path = os.path.dirname(__file__)
-    static_path = os.path.abspath(os.path.join(root_path, tornado.options.options.static))
-
-    url = urlparse.urlparse(settings.REDIS_URL)
-    redis_connection = redis.StrictRedis(host=url.hostname, port=url.port, db=0, password=url.password)
-    data_manager = data.Manager(redis_connection, settings.INTERNAL_DB_CONNECTION_STRING,
-                                      settings.MAX_CONNECTIONS)
-
-    logging.info("re:dash web server stating on port: %d...", tornado.options.options.port)
-    logging.info("UI assets path: %s...", static_path)
-
-    application = get_application(static_path, tornado.options.options.debug,
-                                  redis_connection, data_manager)
-
-    application.listen(tornado.options.options.port)
-    tornado.ioloop.IOLoop.instance().start()

rd_service/settings_example.py

@@ -1,41 +0,0 @@
-"""
-Example settings module. You should make your own copy as settings.py and enter the real settings.
-"""
-
-import django.conf
-
-REDIS_URL = "redis://localhost:6379"
-
-# Either "pg" or "mysql"
-CONNECTION_ADAPTER = "mysql"
-# Connection string for the database that is used to run queries against
-# -- example mysql CONNECTION_STRING = "Server=;User=;Pwd=;Database="
-# -- example pg    CONNECTION_STRING = "user= password= host= port=5439 dbname="
-CONNECTION_STRING = "user= password= host= port=5439 dbname="
-# Connection string for the operational databases (where we store the queries, results, etc)
-INTERNAL_DB_CONNECTION_STRING = "dbname=postgres"
-# Google Apps domain to allow access from; any user with email in this Google Apps will be allowed
-# access
-GOOGLE_APPS_DOMAIN = ""
-# Email addresses of specific users not from the above set Google Apps Domain, that you want to
-# allow access to re:dash
-ALLOWED_USERS = []
-# Email addresses of admin users
-ADMINS = []
-STATIC_ASSETS_PATH = "../rd_ui/dist/"
-WORKERS_COUNT = 2
-MAX_CONNECTIONS = 3
-COOKIE_SECRET = "c292a0a3aa32397cdb050e233733900f"
-LOG_LEVEL = "INFO"
-ANALYTICS = ""
-
-# Configuration of the operational database for the Django models
-django.conf.settings.configure(DATABASES = { 'default': {
-        'ENGINE': 'dbpool.db.backends.postgresql_psycopg2',
-        'OPTIONS': {'MAX_CONNS': 10, 'MIN_CONNS': 1},
-        'NAME': 'postgres',
-        'USER': '',
-        'PASSWORD': '',
-        'HOST': '',
-        'PORT': '',
-        },}, TIME_ZONE = 'UTC')

rd_ui/Gruntfile.js

@@ -170,7 +170,7 @@ module.exports = function (grunt) {
       }
     },
     useminPrepare: {
-      html: '<%= yeoman.app %>/index.html',
+      html: ['<%= yeoman.app %>/index.html', '<%= yeoman.app %>/login.html'],
       options: {
         dest: '<%= yeoman.dist %>'
       }
@@ -231,7 +231,7 @@ module.exports = function (grunt) {
         files: [{
           expand: true,
           cwd: '<%= yeoman.app %>',
-          src: ['*.html', 'views/*.html'],
+          src: ['*.html', 'views/**/*.html'],
           dest: '<%= yeoman.dist %>'
         }]
       }
@@ -249,6 +249,7 @@ module.exports = function (grunt) {
             '.htaccess',
             'bower_components/**/*',
             'images/{,*/}*.{gif,webp}',
+            'styles/{,*/}*.{png,gif}',
             'fonts/*'
           ]
         }, {

rd_ui/app/index.html

@@ -4,7 +4,7 @@
 <!--[if IE 8]>         <html class="no-js lt-ie9" ng-app="redash" ng-controller='MainCtrl'> <![endif]-->
 <!--[if gt IE 8]><!--> <html class="no-js" ng-app="redash" ng-controller='MainCtrl'> <!--<![endif]-->
 <head>
-    <title ng-bind="'re:dash | ' + pageTitle"></title>
+    <title ng-bind="'{{name}} | ' + pageTitle"></title>
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
 
@@ -14,6 +14,7 @@
     <link rel="stylesheet" href="/bower_components/gridster/dist/jquery.gridster.css">
     <link rel="stylesheet" href="/bower_components/pivottable/examples/pivot.css">
     <link rel="stylesheet" href="/bower_components/cornelius/src/cornelius.css">
+    <link rel="stylesheet" href="/bower_components/select2/select2.css">
     <link rel="stylesheet" href="/styles/redash.css">
     <!-- endbuild -->
 </head>
@@ -29,8 +30,9 @@
             <span class="icon-bar"></span>
             <span class="icon-bar"></span>
         </button>
-        <a class="navbar-brand" href="/"><strong>re:dash</strong></a>
+        <a class="navbar-brand" href="/"><strong>{{name}}</strong></a>
     </div>
+    {% raw %}
     <div class="collapse navbar-collapse navbar-ex1-collapse">
         <ul class="nav navbar-nav">
             <li class="active" ng-show="pageTitle"><a class="page-title" ng-bind="pageTitle"></a></li>
@@ -42,32 +44,36 @@
                           <a href="#" ng-bind="name"></a>
                           <ul class="dropdown-menu">
                             <li ng-repeat="dashboard in group" role="presentation">
-                                <a role="menu-item" ng-href="/dashboard/{{!dashboard.slug}}" ng-bind="dashboard.name"></a>
+                                <a role="menu-item" ng-href="/dashboard/{{dashboard.slug}}" ng-bind="dashboard.name"></a>
                             </li>
                           </ul>
                         </li>
                     </span>
                     <li ng-repeat="dashboard in otherDashboards">
-                        <a role="menu-item" ng-href="/dashboard/{{!dashboard.slug}}" ng-bind="dashboard.name"></a>
+                        <a role="menu-item" ng-href="/dashboard/{{dashboard.slug}}" ng-bind="dashboard.name"></a>
                     </li>
-                    <li class="divider"></li>
-                    <li><a data-toggle="modal" href="#new_dashboard_dialog">New Dashboard</a></li>
+                    <li class="divider" ng-show="currentUser.hasPermission('create_dashboard')"></li>
+                    <li><a data-toggle="modal" href="#new_dashboard_dialog" ng-show="currentUser.hasPermission('create_dashboard')">New Dashboard</a></li>
                 </ul>
             </li>
-            <li class="dropdown">
+            <li class="dropdown" ng-show="currentUser.hasPermission('view_query')">
                 <a href="#" class="dropdown-toggle" data-toggle="dropdown">Queries <b class="caret"></b></a>
                 <ul class="dropdown-menu">
-                    <li><a href="/queries/new">New Query</a></li>
+                    <li ng-show="currentUser.hasPermission('create_query')"><a href="/queries/new">New Query</a></li>
                     <li><a href="/queries">Queries</a></li>
                 </ul>
             </li>
         </ul>
         <ul class="nav navbar-nav navbar-right">
-            <p class="navbar-text avatar">
-                <img ng-src="{{!currentUser.gravatar_url}}" class="img-circle" alt="{{!currentUser.name}}" width="40" height="40"/>
+            <p class="navbar-text avatar" ng-show="currentUser.id" ng-cloak>
+                <img ng-src="{{currentUser.gravatar_url}}" class="img-circle" alt="{{currentUser.name}}"/>
+                <a target="_self" href="/logout" id="logout" title="Logout">
+                    <span class="glyphicon glyphicon-log-out"></span>
+                </a>
             </p>
         </ul>
     </div>
+    {% endraw %}
 
     </div>
 </nav>
@@ -95,37 +101,56 @@
 <script src="/bower_components/angular-ui-codemirror/ui-codemirror.js"></script>
 <script src="/bower_components/highcharts/highcharts.js"></script>
 <script src="/bower_components/highcharts/modules/exporting.js"></script>
-<script src="/scripts/ng-highchart.js"></script>
-<script src="/scripts/smart-table.js"></script>
-<script src="/scripts/ui-bootstrap-tpls-0.5.0.min.js"></script>
 <script src="/bower_components/gridster/dist/jquery.gridster.js"></script>
 <script src="/bower_components/angular-growl/build/angular-growl.js"></script>
 <script src="/bower_components/pivottable/examples/pivot.js"></script>
 <script src="/bower_components/cornelius/src/cornelius.js"></script>
 <script src="/bower_components/mousetrap/mousetrap.js"></script>
 <script src="/bower_components/mousetrap/plugins/global-bind/mousetrap-global-bind.js"></script>
+<script src="/bower_components/select2/select2.js"></script>
+<script src="/bower_components/angular-ui-select2/src/select2.js"></script>
+
+<script src="/scripts/ng_highchart.js"></script>
+<script src="/scripts/ng_smart_table.js"></script>
+<script src="/scripts/ui-bootstrap-tpls-0.5.0.min.js"></script>
+
 <!-- endbuild -->
 
 <!-- build:js({.tmp,app}) /scripts/scripts.js -->
 <script src="/scripts/app.js"></script>
-<script src="/scripts/controllers.js"></script>
-<script src="/scripts/admin_controllers.js"></script>
-<script src="/scripts/directives.js"></script>
-<script src="/scripts/services.js"></script>
-<script src="/scripts/filters.js"></script>
+<script src="/scripts/services/services.js"></script>
+<script src="/scripts/services/resources.js"></script>
 <script src="/scripts/services/notifications.js"></script>
 <script src="/scripts/services/dashboards.js"></script>
-<script src="/scripts/query_fiddle/renderers.js"></script>
+<script src="/scripts/controllers/controllers.js"></script>
+<script src="/scripts/controllers/dashboard.js"></script>
+<script src="/scripts/controllers/admin_controllers.js"></script>
+<script src="/scripts/controllers/query_view.js"></script>
+<script src="/scripts/controllers/query_source.js"></script>
+<script src="/scripts/visualizations/base.js"></script>
+<script src="/scripts/visualizations/chart.js"></script>
+<script src="/scripts/visualizations/cohort.js"></script>
+<script src="/scripts/visualizations/table.js"></script>
+<script src="/scripts/visualizations/pivot.js"></script>
+<script src="/scripts/directives/directives.js"></script>
+<script src="/scripts/directives/query_directives.js"></script>
+<script src="/scripts/directives/dashboard_directives.js"></script>
+<script src="/scripts/filters.js"></script>
 <!-- endbuild -->
 
 <script>
-    var currentUser = {% raw user %};
+    var currentUser = {{ user|safe }};
 
     currentUser.canEdit = function(object) {
-        return object.user && (object.user.indexOf(currentUser.name) != -1);
+        var user_id = object.user_id || (object.user && object.user.id);
+        return user_id && (user_id == currentUser.id);
     };
 
-    {% raw analytics %}
+    currentUser.hasPermission = function(permission) {
+        return this.permissions.indexOf(permission) != -1;
+    }
+
+    {{ analytics|safe }}
 </script>
 
 </body>

rd_ui/app/login.html

@@ -0,0 +1,85 @@
+<!DOCTYPE html>
+<!--[if lt IE 7]>      <html class="no-js lt-ie9 lt-ie8 lt-ie7"> <![endif]-->
+<!--[if IE 7]>         <html class="no-js lt-ie9 lt-ie8"> <![endif]-->
+<!--[if IE 8]>         <html class="no-js lt-ie9"> <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js"> <!--<![endif]-->
+<head>
+    <title>{{name}} Login</title>
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+
+    <!-- build:css /styles/main_login.css -->
+    <link rel="stylesheet" href="/bower_components/bootstrap/dist/css/bootstrap.css">
+    <link rel="stylesheet" href="/styles/redash.css">
+    <link rel="stylesheet" href="/styles/login.css">
+    <!-- endbuild -->
+</head>
+<body>
+
+<nav class="navbar navbar-inverse navbar-fixed-top" role="navigation">
+    <div class="container">
+    <div class="navbar-header">
+        <button type="button" class="navbar-toggle" data-toggle="collapse"
+                data-target=".navbar-ex1-collapse">
+            <span class="sr-only">Toggle navigation</span>
+            <span class="icon-bar"></span>
+            <span class="icon-bar"></span>
+            <span class="icon-bar"></span>
+        </button>
+        <a class="navbar-brand" href="/"><strong>{{name}}</strong></a>
+    </div>
+    </div>
+</nav>
+
+<div class="container">
+    <div class="row">
+
+        <div class="main">
+            <form role="form" method="post" name="login">
+                <div class="form-group">
+                    <label for="inputUsernameEmail">Username or email</label>
+                    <input type="text" class="form-control" id="inputUsernameEmail" name="username" value="{{username}}">
+                </div>
+                <div class="form-group">
+                    <!--<a class="pull-right" href="#">Forgot password?</a>-->
+                    <label for="inputPassword">Password</label>
+                    <input type="password" class="form-control" id="inputPassword" name="password">
+                </div>
+                <div class="checkbox pull-right">
+                    <label>
+                        <input type="checkbox" name="remember">
+                        Remember me </label>
+                </div>
+
+                <button type="submit" class="btn btn btn-primary">
+                    Log In
+                </button>
+            </form>
+
+            {% if show_google_openid %}
+
+            <div class="login-or">
+                <hr class="hr-or">
+                <span class="span-or">or</span>
+            </div>
+
+            <div class="row">
+                <div class="col-xs-6 col-sm-6 col-md-6">
+                    <a href="/google_auth/login?next={{next}}" class="btn btn-lg btn-info btn-block">Google</a>
+                </div>
+            </div>
+
+            {% endif %}
+        </div>
+
+    </div>
+</div>
+
+<script src="/bower_components/jquery/jquery.js"></script>
+
+<script>
+    {{ analytics|safe }}
+</script>
+
+</body>
+</html>

rd_ui/app/scripts/admin_controllers.js

@@ -1,23 +0,0 @@
-(function () {
-    var AdminStatusCtrl = function ($scope, $http, $timeout) {
-        $scope.$parent.pageTitle = "System Status";
-
-        var refresh = function () {
-            $scope.refresh_time = moment().add('minutes', 1);
-            $http.get('/status.json').success(function (data) {
-                $scope.workers = data.workers;
-                delete data.workers;
-                $scope.manager = data.manager;
-                delete data.manager;
-                $scope.status = data;
-            });
-
-            $timeout(refresh, 59 * 1000);
-        };
-
-        refresh();
-    }
-
-    angular.module('redash.admin_controllers', [])
-        .controller('AdminStatusCtrl', ['$scope', '$http', '$timeout', AdminStatusCtrl])
-})();

rd_ui/app/scripts/app.js

@@ -5,8 +5,10 @@ angular.module('redash', [
     'redash.filters',
     'redash.services',
     'redash.renderers',
+    'redash.visualization',
     'ui.codemirror',
     'highchart',
+    'ui.select2',
     'angular-growl',
     'angularMoment',
     'ui.bootstrap',
@@ -16,6 +18,11 @@ angular.module('redash', [
 ]).config(['$routeProvider', '$locationProvider', '$compileProvider', 'growlProvider',
     function($routeProvider, $locationProvider, $compileProvider, growlProvider) {
 
+        function getQuery(Query, $route) {
+            var query = Query.get({'id': $route.current.params.queryId });
+            return query.$promise;
+        };
+
         $compileProvider.aHrefSanitizationWhitelist(/^\s*(https?|http|data):/);
         $locationProvider.html5Mode(true);
         growlProvider.globalTimeToLive(2000);
@@ -30,14 +37,30 @@ angular.module('redash', [
             reloadOnSearch: false
         });
         $routeProvider.when('/queries/new', {
-            templateUrl: '/views/queryfiddle.html',
-            controller: 'QueryFiddleCtrl',
-            reloadOnSearch: false
+            templateUrl: '/views/query.html',
+            controller: 'QuerySourceCtrl',
+            reloadOnSearch: false,
+            resolve: {
+                'query': ['Query', function newQuery(Query) {
+                    return Query.newQuery();
+                }]
+            }
         });
         $routeProvider.when('/queries/:queryId', {
-            templateUrl: '/views/queryfiddle.html',
-            controller: 'QueryFiddleCtrl',
-            reloadOnSearch: false
+            templateUrl: '/views/query.html',
+            controller: 'QueryViewCtrl',
+            reloadOnSearch: false,
+            resolve: {
+                'query': ['Query', '$route', getQuery]
+            }
+        });
+        $routeProvider.when('/queries/:queryId/source', {
+            templateUrl: '/views/query.html',
+            controller: 'QuerySourceCtrl',
+            reloadOnSearch: false,
+            resolve: {
+                'query': ['Query', '$route', getQuery]
+            }
         });
         $routeProvider.when('/admin/status', {
             templateUrl: '/views/admin_status.html',
@@ -51,9 +74,6 @@ angular.module('redash', [
             redirectTo: '/'
         });
 
-        Highcharts.setOptions({
-            colors: ["#4572A7", "#AA4643", "#89A54E", "#80699B", "#3D96AE",
-                        "#DB843D", "#92A8CD", "#A47D7C", "#B5CA92"]
-        });
+
     }
 ]);

rd_ui/app/scripts/controllers.js

@@ -1,395 +0,0 @@
-(function () {
-    var DashboardCtrl = function ($scope, $routeParams, $http, Dashboard) {
-        $scope.dashboard = Dashboard.get({slug: $routeParams.dashboardSlug}, function(dashboard) {
-            $scope.$parent.pageTitle = dashboard.name;
-        });
-    };
-
-    var WidgetCtrl = function ($scope, $http, $location, Query) {
-        $scope.deleteWidget = function() {
-            if (!confirm('Are you sure you want to remove "' + $scope.widget.visualization.name + '" from the dashboard?')) {
-                return;
-            }
-
-            $http.delete('/api/widgets/' + $scope.widget.id).success(function() {
-                $scope.dashboard.widgets = _.map($scope.dashboard.widgets, function(row) {
-                    return _.filter(row, function(widget) {
-                        return widget.id != $scope.widget.id;
-                    })
-                });
-            });
-        };
-
-        $scope.open = function(query) {
-            $location.path('/queries/' + query.id);
-        }
-
-        $scope.query = new Query($scope.widget.visualization.query);
-        $scope.queryResult = $scope.query.getQueryResult();
-
-        $scope.updateTime = (new Date($scope.queryResult.getUpdatedAt())).toISOString();
-        $scope.nextUpdateTime = moment(new Date(($scope.query.updated_at + $scope.query.ttl + $scope.query.runtime + 300) * 1000)).fromNow();
-
-        $scope.updateTime = '';
-    }
-
-    var QueryFiddleCtrl = function ($scope, $window, $location, $routeParams, $http, $location, growl, notifications, Query, Visualization) {
-        var DEFAULT_TAB = 'table';
-        var pristineHash = null;
-        var leavingPageText = "You will lose your changes if you leave";
-
-        $scope.dirty = undefined;
-        $scope.newVisualization = undefined;
-
-        $window.onbeforeunload = function(){
-            if (currentUser.canEdit($scope.query) && $scope.dirty) {
-                return leavingPageText;
-            }
-        }
-
-        Mousetrap.bindGlobal("meta+s", function(e) {
-            e.preventDefault();
-
-            if (currentUser.canEdit($scope.query)) {
-                $scope.saveQuery();
-            }
-        });
-
-        $scope.$on('$locationChangeStart', function(event, next, current) {
-            if (next.split("#")[0] == current.split("#")[0]) {
-                return;
-            }
-
-            if (!currentUser.canEdit($scope.query)) {
-                return;
-            }
-
-            if($scope.dirty &&
-                !confirm(leavingPageText + "\n\nAre you sure you want to leave this page?")) {
-                event.preventDefault();
-            } else {
-                Mousetrap.unbind("meta+s");
-            }
-        });
-
-        $scope.$parent.pageTitle = "Query Fiddle";
-
-        $scope.$watch(function() {return $location.hash()}, function(hash) {
-            $scope.selectedTab = hash || DEFAULT_TAB;
-        });
-
-        $scope.lockButton = function (lock) {
-            $scope.queryExecuting = lock;
-        };
-
-        $scope.formatQuery = function() {
-            $scope.editorOptions.readOnly = 'nocursor';
-
-            $http.post('/api/queries/format', {'query': $scope.query.query}).success(function(response) {
-                $scope.query.query = response;
-                $scope.editorOptions.readOnly = false;
-            })
-        }
-
-        $scope.saveQuery = function (duplicate, oldId) {
-            if (!oldId) {
-                oldId = $scope.query.id;
-            }
-            delete $scope.query.latest_query_data;
-            $scope.query.$save(function (q) {
-                pristineHash = q.getHash();
-                $scope.dirty = false;
-
-                if (duplicate) {
-                    growl.addInfoMessage("Query duplicated.", {ttl: 2000});
-                } else{
-                    growl.addSuccessMessage("Query saved.", {ttl: 2000});
-                }
-
-                if (oldId != q.id) {
-                    if (oldId == undefined) {
-                        $location.path($location.path().replace('new', q.id)).replace();
-                    } else {
-                        // TODO: replace this with a safer method
-                        $location.path($location.path().replace(oldId, q.id)).replace();
-                    }
-                }
-            }, function(httpResponse) {
-                growl.addErrorMessage("Query could not be saved");
-            });
-        };
-
-        $scope.duplicateQuery = function () {
-            var oldId = $scope.query.id;
-            $scope.query.id = null;
-            $scope.query.ttl = -1;
-
-            $scope.saveQuery(true, oldId);
-        };
-
-        // Query Editor:
-        $scope.editorOptions = {
-            mode: 'text/x-sql',
-            lineWrapping: true,
-            lineNumbers: true,
-            readOnly: false,
-            matchBrackets: true,
-            autoCloseBrackets: true
-        };
-
-        $scope.refreshOptions = [
-            {value: -1, name: 'No Refresh'},
-        ]
-
-        _.each(_.range(1, 13), function(i) {
-            $scope.refreshOptions.push({value: i*3600, name: 'Every ' + i + 'h'});
-        })
-
-        $scope.refreshOptions.push({value: 24*3600, name: 'Every 24h'});
-        $scope.refreshOptions.push({value: 7*24*3600, name: 'Once a week'});
-
-        $scope.$watch('queryResult && queryResult.getError()', function (newError, oldError) {
-            if (newError == undefined) {
-                return;
-            }
-
-            if (oldError == undefined && newError != undefined) {
-                $scope.lockButton(false);
-            }
-        });
-
-        $scope.$watch('queryResult && queryResult.getData()', function (data, oldData) {
-            if (!data) {
-                return;
-            }
-
-            if ($scope.queryResult.getId() == null) {
-                $scope.dataUri = "";
-            } else {
-                $scope.dataUri = '/api/queries/' + $scope.query.id + '/results/' + $scope.queryResult.getId() + '.csv';
-                $scope.dataFilename = $scope.query.name.replace(" ", "_") + moment($scope.queryResult.getUpdatedAt()).format("_YYYY_MM_DD") + ".csv";
-            }
-        });
-
-        $scope.$watch("queryResult && queryResult.getStatus()", function (status) {
-            if (!status) {
-                return;
-            }
-
-            if (status == "done") {
-                if ($scope.query.id && $scope.query.latest_query_data_id != $scope.queryResult.getId() &&
-                    $scope.query.query_hash == $scope.queryResult.query_result.query_hash) {
-                    Query.save({'id': $scope.query.id, 'latest_query_data_id': $scope.queryResult.getId()})
-                }
-                $scope.query.latest_query_data_id = $scope.queryResult.getId();
-
-                notifications.showNotification("re:dash", $scope.query.name + " updated.");
-
-                $scope.lockButton(false);
-            }
-        });
-
-        if ($routeParams.queryId != undefined) {
-            $scope.query = Query.get({id: $routeParams.queryId}, function(q) {
-                pristineHash = q.getHash();
-                $scope.dirty = false;
-                $scope.queryResult = $scope.query.getQueryResult();
-            });
-        } else {
-            $scope.query = new Query({query: "", name: "New Query", ttl: -1, user: currentUser.name});
-            $scope.lockButton(false);
-        }
-
-        $scope.$watch('query.name', function() {
-            $scope.$parent.pageTitle = $scope.query.name;
-        });
-
-        $scope.$watch(function() {
-            return $scope.query.getHash();
-        }, function(newHash) {
-            $scope.dirty = (newHash !== pristineHash);
-        });
-
-        $scope.executeQuery = function() {
-            $scope.queryResult = $scope.query.getQueryResult(0);
-            $scope.lockButton(true);
-            $scope.cancelling = false;
-        };
-
-        $scope.cancelExecution = function() {
-            $scope.cancelling = true;
-            $scope.queryResult.cancelExecution();
-        };
-
-        $scope.deleteVisualization = function($e, vis) {
-            $e.preventDefault();
-            if (confirm('Are you sure you want to delete ' + vis.name + ' ?')) {
-                Visualization.delete(vis);
-                if ($scope.selectedTab == vis.id) {
-                    $scope.selectedTab = DEFAULT_TAB;
-                }
-                $scope.query.visualizations =
-                    $scope.query.visualizations.filter(function(v) {
-                        return vis.id !== v.id;
-                    });
-            }
-        };
-    }
-
-    var QueriesCtrl = function($scope, $http, $location, $filter, Query) {
-        $scope.$parent.pageTitle = "All Queries";
-        $scope.gridConfig = {
-            isPaginationEnabled: true,
-            itemsByPage: 50,
-            maxSize: 8,
-            isGlobalSearchActivated: true
-        }
-
-        $scope.allQueries = [];
-        $scope.queries = [];
-
-        var dateFormatter = function (value) {
-            if (!value) return "-";
-            return value.format("DD/MM/YY HH:mm");
-        }
-
-        var filterQueries = function() {
-            $scope.queries = _.filter($scope.allQueries, function(query) {
-                if (!$scope.selectedTab) {
-                    return false;
-                }
-
-                if ($scope.selectedTab.key == 'my') {
-                    return query.user == currentUser.name && query.name != 'New Query';
-                } else if ($scope.selectedTab.key == 'drafts') {
-                    return query.user == currentUser.name && query.name == 'New Query';
-                }
-
-                return query.name != 'New Query';
-            });
-        }
-
-        Query.query(function(queries) {
-            $scope.allQueries = _.map(queries, function(query) {
-                query.created_at = moment(query.created_at);
-                query.last_retrieved_at = moment(query.last_retrieved_at);
-                return query;
-            });
-
-            filterQueries();
-        });
-
-        $scope.gridColumns = [
-            {
-                "label": "Name",
-                "map": "name",
-                "cellTemplateUrl": "/views/queries_query_name_cell.html"
-            },
-            {
-                'label': 'Created By',
-                'map': 'user'
-            },
-            {
-                'label': 'Created At',
-                'map': 'created_at',
-                'formatFunction': dateFormatter
-            },
-            {
-                'label': 'Runtime (avg)',
-                'map': 'avg_runtime',
-                'formatFunction': function(value) {
-                    return $filter('durationHumanize')(value);
-                }
-            },
-            {
-                'label': 'Runtime (min)',
-                'map': 'min_runtime',
-                'formatFunction': function(value) {
-                    return $filter('durationHumanize')(value);
-                }
-            },
-            {
-                'label': 'Runtime (max)',
-                'map': 'max_runtime',
-                'formatFunction': function(value) {
-                    return $filter('durationHumanize')(value);
-                }
-            },
-            {
-                'label': 'Last Executed At',
-                'map': 'last_retrieved_at',
-                'formatFunction': dateFormatter
-            },
-            {
-                'label': 'Times Executed',
-                'map': 'times_retrieved'
-            },
-            {
-                'label': 'Update Schedule',
-                'map': 'ttl',
-                'formatFunction': function(value) {
-                    return $filter('refreshRateHumanize')(value);
-                }
-            }
-        ]
-        $scope.tabs = [{"name": "My Queries", "key": "my"}, {"key": "all", "name": "All Queries"}, {"key": "drafts", "name": "Drafts"}];
-
-        $scope.$watch('selectedTab', function(tab) {
-            if (tab) {
-                $scope.$parent.pageTitle =  tab.name;
-            }
-
-            filterQueries();
-        });
-    }
-
-    var MainCtrl = function ($scope, Dashboard, notifications) {
-        $scope.dashboards = [];
-        $scope.reloadDashboards = function() {
-            Dashboard.query(function (dashboards) {
-                $scope.dashboards = _.sortBy(dashboards, "name");
-                $scope.allDashboards = _.groupBy($scope.dashboards, function(d) {
-                    parts = d.name.split(":");
-                    if (parts.length == 1) {
-                        return "Other";
-                    }
-                    return parts[0];
-                });
-                $scope.otherDashboards = $scope.allDashboards['Other'] || [];
-                $scope.groupedDashboards = _.omit($scope.allDashboards, 'Other');
-            });
-        }
-
-        $scope.reloadDashboards();
-
-        $scope.currentUser = currentUser;
-        $scope.newDashboard = {
-            'name': null,
-            'layout': null
-        }
-
-        $(window).click(function () {
-            notifications.getPermissions();
-        });
-    }
-
-    var IndexCtrl = function($scope, Dashboard) {
-        $scope.$parent.pageTitle = "Home";
-
-        $scope.archiveDashboard = function(dashboard) {
-            if (confirm('Are you sure you want to delete "' + dashboard.name + '" dashboard?')) {
-                dashboard.$delete(function() {
-                    $scope.$parent.reloadDashboards();
-                });
-            }
-        }
-    }
-
-    angular.module('redash.controllers', [])
-        .controller('DashboardCtrl', ['$scope', '$routeParams', '$http', 'Dashboard', DashboardCtrl])
-        .controller('WidgetCtrl', ['$scope', '$http', '$location', 'Query', WidgetCtrl])
-        .controller('QueriesCtrl', ['$scope', '$http', '$location', '$filter', 'Query', QueriesCtrl])
-        .controller('QueryFiddleCtrl', ['$scope', '$window', '$location', '$routeParams', '$http', '$location', 'growl', 'notifications', 'Query', 'Visualization', QueryFiddleCtrl])
-        .controller('IndexCtrl', ['$scope', 'Dashboard', IndexCtrl])
-        .controller('MainCtrl', ['$scope', 'Dashboard', 'notifications', MainCtrl]);
-})();

rd_ui/app/scripts/controllers/admin_controllers.js

@@ -0,0 +1,24 @@
+(function () {
+  var AdminStatusCtrl = function ($scope, Events, $http, $timeout) {
+    Events.record(currentUser, "view", "page", "admin/status");
+    $scope.$parent.pageTitle = "System Status";
+
+    var refresh = function () {
+      $scope.refresh_time = moment().add('minutes', 1);
+      $http.get('/status.json').success(function (data) {
+        $scope.workers = data.workers;
+        delete data.workers;
+        $scope.manager = data.manager;
+        delete data.manager;
+        $scope.status = data;
+      });
+
+      $timeout(refresh, 59 * 1000);
+    };
+
+    refresh();
+  }
+
+  angular.module('redash.admin_controllers', [])
+         .controller('AdminStatusCtrl', ['$scope', 'Events', '$http', '$timeout', AdminStatusCtrl])
+})();

rd_ui/app/scripts/controllers/controllers.js

@@ -0,0 +1,157 @@
+(function () {
+    var QueriesCtrl = function($scope, $http, $location, $filter, Query) {
+        $scope.$parent.pageTitle = "All Queries";
+        $scope.gridConfig = {
+            isPaginationEnabled: true,
+            itemsByPage: 50,
+            maxSize: 8,
+            isGlobalSearchActivated: true
+        }
+
+        $scope.allQueries = [];
+        $scope.queries = [];
+
+        var dateFormatter = function (value) {
+            if (!value) return "-";
+            return value.format("DD/MM/YY HH:mm");
+        }
+
+        var filterQueries = function() {
+            $scope.queries = _.filter($scope.allQueries, function(query) {
+                if (!$scope.selectedTab) {
+                    return false;
+                }
+
+                if ($scope.selectedTab.key == 'my') {
+                    return query.user.id == currentUser.id && query.name != 'New Query';
+                } else if ($scope.selectedTab.key == 'drafts') {
+                    return query.user.id == currentUser.id && query.name == 'New Query';
+                }
+
+                return query.name != 'New Query';
+            });
+        }
+
+        Query.query(function(queries) {
+            $scope.allQueries = _.map(queries, function(query) {
+                query.created_at = moment(query.created_at);
+                query.last_retrieved_at = moment(query.last_retrieved_at);
+                return query;
+            });
+
+            filterQueries();
+        });
+
+        $scope.gridColumns = [
+            {
+                "label": "Name",
+                "map": "name",
+                "cellTemplateUrl": "/views/queries_query_name_cell.html"
+            },
+            {
+                'label': 'Created By',
+                'map': 'user.name'
+            },
+            {
+                'label': 'Created At',
+                'map': 'created_at',
+                'formatFunction': dateFormatter
+            },
+            {
+                'label': 'Runtime (avg)',
+                'map': 'avg_runtime',
+                'formatFunction': function(value) {
+                    return $filter('durationHumanize')(value);
+                }
+            },
+            {
+                'label': 'Runtime (min)',
+                'map': 'min_runtime',
+                'formatFunction': function(value) {
+                    return $filter('durationHumanize')(value);
+                }
+            },
+            {
+                'label': 'Runtime (max)',
+                'map': 'max_runtime',
+                'formatFunction': function(value) {
+                    return $filter('durationHumanize')(value);
+                }
+            },
+            {
+                'label': 'Last Executed At',
+                'map': 'last_retrieved_at',
+                'formatFunction': dateFormatter
+            },
+            {
+                'label': 'Times Executed',
+                'map': 'times_retrieved'
+            },
+            {
+                'label': 'Update Schedule',
+                'map': 'ttl',
+                'formatFunction': function(value) {
+                    return $filter('refreshRateHumanize')(value);
+                }
+            }
+        ]
+        $scope.tabs = [{"name": "My Queries", "key": "my"}, {"key": "all", "name": "All Queries"}, {"key": "drafts", "name": "Drafts"}];
+
+        $scope.$watch('selectedTab', function(tab) {
+            if (tab) {
+                $scope.$parent.pageTitle =  tab.name;
+            }
+
+            filterQueries();
+        });
+    }
+
+    var MainCtrl = function ($scope, Dashboard, notifications) {
+        $scope.dashboards = [];
+        $scope.reloadDashboards = function() {
+            Dashboard.query(function (dashboards) {
+                $scope.dashboards = _.sortBy(dashboards, "name");
+                $scope.allDashboards = _.groupBy($scope.dashboards, function(d) {
+                    parts = d.name.split(":");
+                    if (parts.length == 1) {
+                        return "Other";
+                    }
+                    return parts[0];
+                });
+                $scope.otherDashboards = $scope.allDashboards['Other'] || [];
+                $scope.groupedDashboards = _.omit($scope.allDashboards, 'Other');
+            });
+        }
+
+        $scope.reloadDashboards();
+
+        $scope.currentUser = currentUser;
+        $scope.newDashboard = {
+            'name': null,
+            'layout': null
+        }
+
+        $(window).click(function () {
+            notifications.getPermissions();
+        });
+    }
+
+    var IndexCtrl = function($scope, Events, Dashboard) {
+        Events.record(currentUser, "view", "page", "homepage");
+        $scope.$parent.pageTitle = "Home";
+
+        $scope.archiveDashboard = function(dashboard) {
+            if (confirm('Are you sure you want to delete "' + dashboard.name + '" dashboard?')) {
+                Events.record(currentUser, "archive", "dashboard", dashboard.id);
+                dashboard.$delete(function() {
+                    $scope.$parent.reloadDashboards();
+                });
+            }
+        }
+    }
+
+    angular.module('redash.controllers', [])
+        .controller('QueriesCtrl', ['$scope', '$http', '$location', '$filter', 'Query', QueriesCtrl])
+        .controller('IndexCtrl', ['$scope', 'Events', 'Dashboard', IndexCtrl])
+        .controller('MainCtrl', ['$scope', 'Dashboard', 'notifications', MainCtrl]);
+})();

rd_ui/app/scripts/controllers/dashboard.js

@@ -0,0 +1,89 @@
+(function() {
+  var DashboardCtrl = function($scope, Events, $routeParams, $http, $timeout, Dashboard) {
+    Events.record(currentUser, "view", "dashboard", dashboard.id);
+
+    $scope.refreshEnabled = false;
+    $scope.refreshRate = 60;
+    $scope.dashboard = Dashboard.get({
+      slug: $routeParams.dashboardSlug
+    }, function(dashboard) {
+      $scope.$parent.pageTitle = dashboard.name;
+    });
+
+    var autoRefresh = function() {
+      if ($scope.refreshEnabled) {
+        $timeout(function() {
+          Dashboard.get({
+            slug: $routeParams.dashboardSlug
+          }, function(dashboard) {
+            var newWidgets = _.groupBy(_.flatten(dashboard.widgets), 'id');
+
+            _.each($scope.dashboard.widgets, function(row) {
+              _.each(row, function(widget, i) {
+                var newWidget = newWidgets[widget.id];
+                if (newWidget && newWidget[0].visualization.query.latest_query_data_id != widget.visualization.query.latest_query_data_id) {
+                  row[i] = newWidget[0];
+                }
+              });
+            });
+
+            autoRefresh();
+          });
+
+        }, $scope.refreshRate);
+      };
+    }
+
+    $scope.triggerRefresh = function() {
+      $scope.refreshEnabled = !$scope.refreshEnabled;
+
+      Events.record(currentUser, "autorefresh", "dashboard", dashboard.id, {'enable': $scope.refreshEnabled});
+
+      if ($scope.refreshEnabled) {
+        var refreshRate = _.min(_.flatten($scope.dashboard.widgets), function(widget) {
+          return widget.visualization.query.ttl;
+        }).visualization.query.ttl;
+
+        $scope.refreshRate = _.max([120, refreshRate * 2]) * 1000;
+
+        autoRefresh();
+      }
+    };
+  };
+
+  var WidgetCtrl = function($scope, Events, $http, $location, Query) {
+    $scope.deleteWidget = function() {
+      if (!confirm('Are you sure you want to remove "' + $scope.widget.visualization.name + '" from the dashboard?')) {
+        return;
+      }
+
+      Events.record(currentUser, "delete", "widget", $scope.widget.id);
+
+      $http.delete('/api/widgets/' + $scope.widget.id).success(function() {
+        $scope.dashboard.widgets = _.map($scope.dashboard.widgets, function(row) {
+          return _.filter(row, function(widget) {
+            return widget.id != $scope.widget.id;
+          })
+        });
+      });
+    };
+
+    // TODO: fire event for query view for each query
+    Events.record(currentUser, "view", "widget", $scope.widget.id);
+    Events.record(currentUser, "view", "query", $scope.widget.visualization.query.id);
+    Events.record(currentUser, "view", "visualization", $scope.widget.visualization.id);
+
+    $scope.query = new Query($scope.widget.visualization.query);
+    $scope.queryResult = $scope.query.getQueryResult();
+
+    $scope.updateTime = (new Date($scope.queryResult.getUpdatedAt())).toISOString();
+    $scope.nextUpdateTime = moment(new Date(($scope.query.updated_at + $scope.query.ttl + $scope.query.runtime + 300) * 1000)).fromNow();
+
+    $scope.updateTime = '';
+  };
+
+  angular.module('redash.controllers')
+    .controller('DashboardCtrl', ['$scope', 'Events', '$routeParams', '$http', '$timeout', 'Dashboard', DashboardCtrl])
+    .controller('WidgetCtrl', ['$scope', 'Events', '$http', '$location', 'Query', WidgetCtrl])
+
+})();

rd_ui/app/scripts/controllers/query_source.js

@@ -0,0 +1,105 @@
+(function() {
+  'use strict';
+
+  function QuerySourceCtrl(Events, $controller, $scope, $location, Query, Visualization, KeyboardShortcuts) {
+    // extends QueryViewCtrl
+    $controller('QueryViewCtrl', {$scope: $scope});
+    // TODO:
+    // This doesn't get inherited. Setting it on this didn't work either (which is weird).
+    // Obviously it shouldn't be repeated, but we got bigger fish to fry.
+    var DEFAULT_TAB = 'table';
+
+    Events.record(currentUser, 'view_source', 'query', $scope.query.id);
+
+    var isNewQuery = !$scope.query.id,
+        queryText = $scope.query.query,
+        // ref to QueryViewCtrl.saveQuery
+        saveQuery = $scope.saveQuery,
+        shortcuts = {
+          'meta+s': function () {
+            if ($scope.canEdit) {
+              $scope.saveQuery();
+            }
+          }
+        };
+
+    $scope.sourceMode = true;
+    $scope.canEdit = currentUser.canEdit($scope.query);
+    $scope.isDirty = false;
+
+    $scope.newVisualization = undefined;
+
+    KeyboardShortcuts.bind(shortcuts);
+
+    // @override
+    $scope.saveQuery = function(options, data) {
+      var savePromise = saveQuery(options, data);
+
+      savePromise.then(function(savedQuery) {
+        queryText = savedQuery.query;
+        $scope.isDirty = $scope.query.query !== queryText;
+
+        if (isNewQuery) {
+          // redirect to new created query (keep hash)
+          $location.path(savedQuery.getSourceLink()).replace();
+        }
+      });
+
+      return savePromise;
+    };
+
+    $scope.duplicateQuery = function() {
+      Events.record(currentUser, 'fork', 'query', $scope.query.id);
+      $scope.query.id = null;
+      $scope.query.ttl = -1;
+
+      $scope.saveQuery({
+        successMessage: 'Query forked',
+        errorMessage: 'Query could not be forked'
+      }).then(function redirect(savedQuery) {
+        // redirect to forked query (clear hash)
+        $location.url(savedQuery.getSourceLink()).replace()
+      });
+    };
+
+    $scope.deleteVisualization = function($e, vis) {
+      $e.preventDefault();
+      if (confirm('Are you sure you want to delete ' + vis.name + ' ?')) {
+        Events.record(currentUser, 'delete', 'visualization', vis.id);
+
+        Visualization.delete(vis);
+        if ($scope.selectedTab == vis.id) {
+          $scope.selectedTab = DEFAULT_TAB;
+          $location.hash($scope.selectedTab);
+        }
+        $scope.query.visualizations =
+          $scope.query.visualizations.filter(function(v) {
+            return vis.id !== v.id;
+          });
+      }
+    };
+
+    $scope.$watch('query.query', function(newQueryText) {
+      $scope.isDirty = (newQueryText !== queryText);
+    });
+
+    $scope.$on('$destroy', function destroy() {
+      KeyboardShortcuts.unbind(shortcuts);
+    });
+
+    if (isNewQuery) {
+      // save new query when creating a visualization
+      var unbind = $scope.$watch('selectedTab == "add"', function(triggerSave) {
+        if (triggerSave) {
+          unbind();
+          $scope.saveQuery();
+        }
+      });
+    }
+  }
+
+  angular.module('redash.controllers').controller('QuerySourceCtrl', [
+    'Events', '$controller', '$scope', '$location', 'Query',
+    'Visualization', 'KeyboardShortcuts', QuerySourceCtrl
+    ]);
+})();

rd_ui/app/scripts/controllers/query_view.js

@@ -0,0 +1,155 @@
+(function() {
+  'use strict';
+
+  function QueryViewCtrl($scope, Events, $route, $location, notifications, growl, Query, DataSource) {
+    var DEFAULT_TAB = 'table';
+
+    $scope.query = $route.current.locals.query;
+    Events.record(currentUser, 'view', 'query', $scope.query.id);
+    $scope.queryResult = $scope.query.getQueryResult();
+    $scope.queryExecuting = false;
+
+    $scope.isQueryOwner = currentUser.id === $scope.query.user.id;
+    $scope.canViewSource = currentUser.hasPermission('view_source');
+
+    $scope.dataSources = DataSource.get(function(dataSources) {
+      $scope.query.data_source_id = $scope.query.data_source_id || dataSources[0].id;
+    });
+
+    $scope.lockButton = function(lock) {
+      $scope.queryExecuting = lock;
+    };
+
+    $scope.saveQuery = function(options, data) {
+      if (data) {
+        data.id = $scope.query.id;
+      } else {
+        data = $scope.query;
+      }
+
+      options = _.extend({}, {
+        successMessage: 'Query saved',
+        errorMessage: 'Query could not be saved'
+      }, options);
+
+      delete $scope.query.latest_query_data;
+
+      return Query.save(data, function() {
+        growl.addSuccessMessage(options.successMessage);
+      }, function(httpResponse) {
+        growl.addErrorMessage(options.errorMessage);
+      }).$promise;
+    }
+
+    $scope.saveDescription = function() {
+      Events.record(currentUser, 'edit_description', 'query', $scope.query.id);
+      $scope.saveQuery(undefined, {'description': $scope.query.description});
+    };
+
+    $scope.saveName = function() {
+      Events.record(currentUser, 'edit_name', 'query', $scope.query.id);
+      $scope.saveQuery(undefined, {'name': $scope.query.name});
+    };
+
+    $scope.executeQuery = function() {
+      $scope.queryResult = $scope.query.getQueryResult(0);
+      $scope.lockButton(true);
+      $scope.cancelling = false;
+      Events.record(currentUser, 'execute', 'query', $scope.query.id);
+    };
+
+    $scope.cancelExecution = function() {
+      $scope.cancelling = true;
+      $scope.queryResult.cancelExecution();
+      Events.record(currentUser, 'cancel_execute', 'query', $scope.query.id);
+    };
+
+    $scope.updateDataSource = function() {
+      Events.record(currentUser, 'update_data_source', 'query', $scope.query.id);
+      $scope.query.latest_query_data = null;
+      $scope.query.latest_query_data_id = null;
+      Query.save({
+          'id': $scope.query.id,
+          'data_source_id': $scope.query.data_source_id,
+          'latest_query_data_id': null
+      });
+
+      $scope.executeQuery();
+    };
+
+    $scope.setVisualizationTab = function (visualization) {
+      $scope.selectedTab = visualization.id;
+      $location.hash(visualization.id);
+    };
+
+    $scope.$watch('query.name', function() {
+      $scope.$parent.pageTitle = $scope.query.name;
+    });
+
+    $scope.$watch('queryResult && queryResult.getError()', function(newError, oldError) {
+      if (newError == undefined) {
+        return;
+      }
+
+      if (oldError == undefined && newError != undefined) {
+        $scope.lockButton(false);
+      }
+    });
+
+    $scope.$watch('queryResult && queryResult.getData()', function(data, oldData) {
+      if (!data) {
+        return;
+      }
+
+      $scope.filters = $scope.queryResult.getFilters();
+
+      if ($scope.queryResult.getId() == null) {
+        $scope.dataUri = "";
+      } else {
+        $scope.dataUri =
+          '/api/queries/' + $scope.query.id + '/results/' +
+          $scope.queryResult.getId() + '.csv';
+
+        $scope.dataFilename =
+          $scope.query.name.replace(" ", "_") +
+          moment($scope.queryResult.getUpdatedAt()).format("_YYYY_MM_DD") +
+          ".csv";
+      }
+    });
+
+    $scope.$watch("queryResult && queryResult.getStatus()", function(status) {
+      if (!status) {
+        return;
+      }
+
+      if (status == "done") {
+        if ($scope.query.id &&
+          $scope.query.latest_query_data_id != $scope.queryResult.getId() &&
+          $scope.query.query_hash == $scope.queryResult.query_result.query_hash) {
+          Query.save({
+            'id': $scope.query.id,
+            'latest_query_data_id': $scope.queryResult.getId()
+          })
+        }
+        $scope.query.latest_query_data_id = $scope.queryResult.getId();
+
+        notifications.showNotification("re:dash", $scope.query.name + " updated.");
+
+        $scope.lockButton(false);
+      }
+    });
+
+    $scope.$watch(function() {
+      return $location.hash()
+    }, function(hash) {
+      if (hash == 'pivot') {
+        Events.record(currentUser, 'pivot', 'query', $scope.query && $scope.query.id);
+      }
+      $scope.selectedTab = hash || DEFAULT_TAB;
+    });
+  };
+
+  angular.module('redash.controllers')
+    .controller('QueryViewCtrl',
+      ['$scope', 'Events', '$route', '$location', 'notifications', 'growl', 'Query', 'DataSource', QueryViewCtrl]);
+})();

rd_ui/app/scripts/directives.js

@@ -1,399 +0,0 @@
-(function() {
-    'use strict';
-
-    var directives = angular.module('redash.directives', []);
-
-    directives.directive('rdTab', ['$location', function($location) {
-        return {
-            restrict: 'E',
-            scope: {
-                'id': '@',
-                'name': '@'
-            },
-            transclude: true,
-            template: '<li class="rd-tab" ng-class="{active: id==selectedTab}"><a href="#{{id}}">{{name}}<span ng-transclude></span></a></li>',
-            replace: true,
-            link: function(scope) {
-                scope.$watch(function(){return scope.$parent.selectedTab}, function(tab) {
-                    scope.selectedTab = tab;
-                });
-            }
-        }
-    }]);
-
-    directives.directive('rdTabs', ['$location', '$rootScope', function($location, $rootScope) {
-        return {
-            restrict: 'E',
-            scope: {
-                tabsCollection: '=',
-                selectedTab: '='
-            },
-            template: '<ul class="nav nav-tabs"><li ng-class="{active: tab==selectedTab}" ng-repeat="tab in tabsCollection"><a href="#{{tab.key}}">{{tab.name}}</a></li></ul>',
-            replace: true,
-            link: function($scope, element, attrs) {
-                $scope.selectTab = function(tabKey) {
-                    $scope.selectedTab  = _.find($scope.tabsCollection, function(tab) { return tab.key == tabKey; });
-                }
-
-                $scope.$watch(function() { return $location.hash()}, function(hash) {
-                    if (hash) {
-                        $scope.selectTab($location.hash());
-                    } else {
-                        $scope.selectTab($scope.tabsCollection[0].key);
-                    }
-                });
-            }
-        }
-    }]);
-
-    directives.directive('editVisulatizationForm', ['Visualization', 'growl', function(Visualization, growl) {
-        return {
-            restrict: 'E',
-            templateUrl: '/views/edit_visualization.html',
-            replace: true,
-            scope: {
-                query: '=',
-                vis: '=?'
-            },
-            link: function(scope, element, attrs) {
-                scope.advancedMode = false;
-                scope.visTypes = {
-                    'Chart': Visualization.prototype.TYPES.CHART,
-                    'Cohort': Visualization.prototype.TYPES.COHORT,
-                    'Table': Visualization.prototype.TYPES.GRID
-                };
-                scope.seriesTypes = {
-                    'Line': Visualization.prototype.SERIES_TYPES.LINE,
-                    'Bar': Visualization.prototype.SERIES_TYPES.BAR,
-                    'Area': Visualization.prototype.SERIES_TYPES.AREA
-                };
-
-                if (!scope.vis) {
-                    // create new visualization
-                    // wait for query to load to populate with defaults
-                    var unwatch = scope.$watch('query', function(q) {
-                        if (q && q.id) {
-                            unwatch();
-                            scope.vis = {
-                                'query_id': q.id,
-                                'type': Visualization.prototype.TYPES.CHART,
-                                'name': q.name,
-                                'description': q.description,
-                                'options': newOptions()
-                            };
-                        }
-                    }, true);
-                }
-
-                function newOptions(chartType) {
-                    if (chartType === Visualization.prototype.TYPES.COHORT) {
-                        // empty config at the moment
-                        return {};
-                    }
-
-                    // Chart
-                    return {
-                        'series': {
-                            'type': Visualization.prototype.SERIES_TYPES.LINE
-                        }
-                    };
-                }
-
-                scope.toggleAdvancedMode = function() {
-                    scope.advancedMode = !scope.advancedMode;
-                };
-
-                scope.typeChanged = function() {
-                    scope.vis.options = newOptions();
-                };
-
-                scope.submit = function() {
-                    Visualization.save(scope.vis, function success(result) {
-                        growl.addSuccessMessage("Visualization saved");
-
-                        scope.vis = result;
-
-                        var visIds = _.pluck(scope.query.visualizations, 'id');
-                        var index = visIds.indexOf(result.id);
-                        if (index > -1) {
-                            scope.query.visualizations[index] = result;
-                        } else {
-                            scope.query.visualizations.push(result);
-                        }
-                    }, function error() {
-                        growl.addErrorMessage("Visualization could not be saved");
-                    });
-                };
-            }
-        }
-    }]);
-
-    directives.directive('editDashboardForm', ['$http', '$location', '$timeout', 'Dashboard', function($http, $location, $timeout, Dashboard) {
-        return {
-            restrict: 'E',
-            scope: {
-                dashboard: '='
-            },
-            templateUrl: '/views/edit_dashboard.html',
-            replace: true,
-            link: function($scope, element, attrs) {
-                var gridster = element.find(".gridster ul").gridster({
-                    widget_margins: [5, 5],
-                    widget_base_dimensions: [260, 100],
-                    min_cols: 2,
-                    max_cols: 2,
-                    serialize_params: function($w, wgd) {
-                        return {
-                            col: wgd.col,
-                            row: wgd.row,
-                            id: $w.data('widget-id')
-                        }
-                    }
-                }).data('gridster');
-
-                var gsItemTemplate = '<li data-widget-id="{id}" class="widget panel panel-default gs-w">' +
-                                 '<div class="panel-heading">{name}' +
-                                 '</div></li>';
-
-                $scope.$watch('dashboard.widgets', function(widgets) {
-                    $timeout(function () {
-                        gridster.remove_all_widgets();
-
-                        if (widgets && widgets.length) {
-                            var layout = [];
-
-                            _.each(widgets, function(row, rowIndex) {
-                                _.each(row, function(widget, colIndex) {
-                                    layout.push({
-                                        id: widget.id,
-                                        col: colIndex+1,
-                                        row: rowIndex+1,
-                                        ySize: 1,
-                                        xSize: widget.width,
-                                        name: widget.visualization.name
-                                    });
-                                });
-                            });
-
-                            _.each(layout, function(item) {
-                                var el = gsItemTemplate.replace('{id}', item.id).replace('{name}', item.name);
-                                gridster.add_widget(el, item.xSize, item.ySize, item.col, item.row);
-
-                            });
-                        }
-                    });
-                }, true);
-
-                $scope.saveDashboard = function() {
-                    $scope.saveInProgress = true;
-                    // TODO: we should use the dashboard service here.
-                    if ($scope.dashboard.id) {
-                        var positions = $(element).find('.gridster ul').data('gridster').serialize();
-                        var layout = [];
-                        _.each(_.sortBy(positions, function (pos) {
-                            return pos.row * 10 + pos.col;
-                        }), function (pos) {
-                            var row = pos.row - 1;
-                            var col = pos.col - 1;
-                            layout[row] = layout[row] || [];
-                            if (col > 0 && layout[row][col - 1] == undefined) {
-                                layout[row][col - 1] = pos.id;
-                            } else {
-                                layout[row][col] = pos.id;
-                            }
-
-                        });
-                        $scope.dashboard.layout = layout;
-
-                        layout = JSON.stringify(layout);
-                        $http.post('/api/dashboards/' + $scope.dashboard.id, {'name': $scope.dashboard.name, 'layout': layout}).success(function(response) {
-                            $scope.dashboard = new Dashboard(response);
-                            $scope.saveInProgress = false;
-                            $(element).modal('hide');
-                        })
-                    } else {
-                        $http.post('/api/dashboards', {'name': $scope.dashboard.name}).success(function(response) {
-                            $(element).modal('hide');
-                            $location.path('/dashboard/' + response.slug).replace();
-                        })
-                    }
-                }
-
-            }
-        }
-    }]);
-
-    directives.directive('newWidgetForm', ['$http', 'Query', function($http, Query) {
-        return {
-            restrict: 'E',
-            scope: {
-                dashboard: '='
-            },
-            templateUrl: '/views/new_widget_form.html',
-            replace: true,
-            link: function($scope, element, attrs) {
-                $scope.widgetSizes = [{name: 'Regular', value: 1}, {name: 'Double', value: 2}];
-
-                var reset = function() {
-                    $scope.saveInProgress = false;
-                    $scope.widgetSize = 1;
-                    $scope.queryId = null;
-                    $scope.selectedVis = null;
-
-                }
-
-                reset();
-
-                $scope.toggleView = function(viewName) {
-                    $scope.currentView = ($scope.currentView == viewName) ? '' : viewName;
-                };
-
-                $scope.loadVisualizations = function() {
-                    if (!$scope.queryId) {
-                        return;
-                    }
-
-                    Query.get({
-                        id: $scope.queryId
-                    }, function(query) {
-                        if (query) {
-                            $scope.query = query;
-                            if(query.visualizations.length) {
-                                $scope.selectedVis = query.visualizations[0];
-                            }
-                        }
-                    });
-                };
-
-                $scope.saveWidget = function() {
-                    $scope.saveInProgress = true;
-
-                    var widget = {
-                        'visualization_id': $scope.selectedVis.id,
-                        'dashboard_id': $scope.dashboard.id,
-                        'options': {},
-                        'width': $scope.widgetSize
-                    }
-
-                    $http.post('/api/widgets', widget).success(function(response) {
-                        // update dashboard layout
-                        $scope.dashboard.layout = response['layout'];
-                        if (response['new_row']) {
-                            $scope.dashboard.widgets.push([response['widget']]);
-                        } else {
-                            $scope.dashboard.widgets[$scope.dashboard.widgets.length-1].push(response['widget']);
-                        }
-
-                        // close the dialog
-                        $('#add_query_dialog').modal('hide');
-                        reset();
-                    })
-                }
-
-            }
-        }
-    }])
-
-    // From: http://jsfiddle.net/joshdmiller/NDFHg/
-    directives.directive('editInPlace', function () {
-        return {
-            restrict: 'E',
-            scope: {
-                value: '=',
-                ignoreBlanks: '=',
-                editable: '='
-            },
-            template: function(tElement, tAttrs) {
-                var elType = tAttrs.editor || 'input';
-                var placeholder = tAttrs.placeholder || 'Click to edit';
-                return '<span ng-click="editable && edit()" ng-bind="value" ng-class="{editable: editable}"></span>' +
-                       '<span ng-click="editable && edit()" ng-show="editable && !value" ng-class="{editable: editable}">' + placeholder + '</span>' +
-                       '<{elType} ng-model="value" class="form-control" rows="2"></{elType}>'.replace('{elType}', elType);
-            },
-            link: function ($scope, element, attrs) {
-                // Let's get a reference to the input element, as we'll want to reference it.
-                var inputElement = angular.element(element.children()[2]);
-
-                // This directive should have a set class so we can style it.
-                element.addClass('edit-in-place');
-
-                // Initially, we're not editing.
-                $scope.editing = false;
-
-                // ng-click handler to activate edit-in-place
-                $scope.edit = function () {
-                    if ($scope.ignoreBlanks) {
-                        $scope.oldValue = $scope.value;
-                    }
-
-                    $scope.editing = true;
-
-                    // We control display through a class on the directive itself. See the CSS.
-                    element.addClass('active');
-
-                    // And we must focus the element.
-                    // `angular.element()` provides a chainable array, like jQuery so to access a native DOM function,
-                    // we have to reference the first element in the array.
-                    inputElement[0].focus();
-                };
-
-                $(inputElement).blur(function() {
-                    if ($scope.ignoreBlanks && _.isEmpty($scope.value)) {
-                        $scope.value = $scope.oldValue;
-                    }
-                    $scope.editing = false;
-                    element.removeClass('active');
-                })
-            }
-        };
-    });
-
-    // http://stackoverflow.com/a/17904092/1559840
-    directives.directive('jsonText', function() {
-        return {
-            restrict: 'A',
-            require: 'ngModel',
-            link: function(scope, element, attr, ngModel) {
-              function into(input) {
-                return JSON.parse(input);
-              }
-              function out(data) {
-                return JSON.stringify(data, undefined, 2);
-              }
-              ngModel.$parsers.push(into);
-              ngModel.$formatters.push(out);
-
-            }
-        };
-    });
-
-    directives.directive('rdTimer', ['$timeout', function ($timeout) {
-        return {
-            restrict: 'E',
-            scope: { timestamp: '=' },
-            template: '{{currentTime}}',
-            controller: ['$scope' ,function ($scope) {
-                $scope.currentTime = "00:00:00";
-                var currentTimeout = null;
-
-                var updateTime = function() {
-                    $scope.currentTime = moment(moment() - moment($scope.timestamp)).utc().format("HH:mm:ss")
-                    currentTimeout = $timeout(updateTime, 1000);
-                }
-
-                var cancelTimer = function() {
-                    if (currentTimeout) {
-                        $timeout.cancel(currentTimeout);
-                        currentTimeout = null;
-                    }
-                }
-
-                updateTime();
-
-                $scope.$on('$destroy', function () {
-                    cancelTimer();
-                });
-            }]
-        };
-    }]);
-})();

rd_ui/app/scripts/directives/dashboard_directives.js

@@ -0,0 +1,189 @@
+(function() {
+  'use strict'
+
+  var directives = angular.module('redash.directives');
+
+  directives.directive('editDashboardForm', ['Events', '$http', '$location', '$timeout', 'Dashboard',
+    function(Events, $http, $location, $timeout, Dashboard) {
+      return {
+        restrict: 'E',
+        scope: {
+          dashboard: '='
+        },
+        templateUrl: '/views/edit_dashboard.html',
+        replace: true,
+        link: function($scope, element, attrs) {
+          var gridster = element.find(".gridster ul").gridster({
+            widget_margins: [5, 5],
+            widget_base_dimensions: [260, 100],
+            min_cols: 2,
+            max_cols: 2,
+            serialize_params: function($w, wgd) {
+              return {
+                col: wgd.col,
+                row: wgd.row,
+                id: $w.data('widget-id')
+              }
+            }
+          }).data('gridster');
+
+          var gsItemTemplate = '<li data-widget-id="{id}" class="widget panel panel-default gs-w">' +
+            '<div class="panel-heading">{name}' +
+            '</div></li>';
+
+          $scope.$watch('dashboard.widgets && dashboard.widgets.length', function(widgets_length) {
+            $timeout(function() {
+              gridster.remove_all_widgets();
+
+              if ($scope.dashboard.widgets && $scope.dashboard.widgets.length) {
+                var layout = [];
+
+                _.each($scope.dashboard.widgets, function(row, rowIndex) {
+                  _.each(row, function(widget, colIndex) {
+                    layout.push({
+                      id: widget.id,
+                      col: colIndex + 1,
+                      row: rowIndex + 1,
+                      ySize: 1,
+                      xSize: widget.width,
+                      name: widget.visualization.query.name
+                    });
+                  });
+                });
+
+                _.each(layout, function(item) {
+                  var el = gsItemTemplate.replace('{id}', item.id).replace('{name}', item.name);
+                  gridster.add_widget(el, item.xSize, item.ySize, item.col, item.row);
+                });
+              }
+            });
+          });
+
+          $scope.saveDashboard = function() {
+            $scope.saveInProgress = true;
+            // TODO: we should use the dashboard service here.
+            if ($scope.dashboard.id) {
+              var positions = $(element).find('.gridster ul').data('gridster').serialize();
+              var layout = [];
+              _.each(_.sortBy(positions, function(pos) {
+                return pos.row * 10 + pos.col;
+              }), function(pos) {
+                var row = pos.row - 1;
+                var col = pos.col - 1;
+                layout[row] = layout[row] || [];
+                if (col > 0 && layout[row][col - 1] == undefined) {
+                  layout[row][col - 1] = pos.id;
+                } else {
+                  layout[row][col] = pos.id;
+                }
+
+              });
+              $scope.dashboard.layout = layout;
+
+              layout = JSON.stringify(layout);
+              $http.post('/api/dashboards/' + $scope.dashboard.id, {
+                'name': $scope.dashboard.name,
+                'layout': layout
+              }).success(function(response) {
+                $scope.dashboard = new Dashboard(response);
+                $scope.saveInProgress = false;
+                $(element).modal('hide');
+              });
+              Events.record(currentUser, 'edit', 'dashboard', $scope.dashboard.id);
+            } else {
+
+              $http.post('/api/dashboards', {
+                'name': $scope.dashboard.name
+              }).success(function(response) {
+                $(element).modal('hide');
+                $location.path('/dashboard/' + response.slug).replace();
+              });
+              Events.record(currentUser, 'create', 'dashboard');
+            }
+          }
+
+        }
+      }
+    }
+  ]);
+
+  directives.directive('newWidgetForm', ['Query', 'Widget', 'growl',
+    function(Query, Widget, growl) {
+      return {
+        restrict: 'E',
+        scope: {
+          dashboard: '='
+        },
+        templateUrl: '/views/new_widget_form.html',
+        replace: true,
+        link: function($scope, element, attrs) {
+          $scope.widgetSizes = [{
+            name: 'Regular',
+            value: 1
+          }, {
+            name: 'Double',
+            value: 2
+          }];
+
+          var reset = function() {
+            $scope.saveInProgress = false;
+            $scope.widgetSize = 1;
+            $scope.queryId = null;
+            $scope.selectedVis = null;
+            $scope.query = null;
+
+          }
+
+          reset();
+
+          $scope.loadVisualizations = function() {
+            if (!$scope.queryId) {
+              return;
+            }
+
+            Query.get({
+              id: $scope.queryId
+            }, function(query) {
+              if (query) {
+                $scope.query = query;
+                if (query.visualizations.length) {
+                  $scope.selectedVis = query.visualizations[0];
+                }
+              }
+            });
+          };
+
+          $scope.saveWidget = function() {
+            $scope.saveInProgress = true;
+
+            var widget = new Widget({
+              'visualization_id': $scope.selectedVis.id,
+              'dashboard_id': $scope.dashboard.id,
+              'options': {},
+              'width': $scope.widgetSize
+            });
+
+            widget.$save().then(function(response) {
+              // update dashboard layout
+              $scope.dashboard.layout = response['layout'];
+              if (response['new_row']) {
+                $scope.dashboard.widgets.push([response['widget']]);
+              } else {
+                $scope.dashboard.widgets[$scope.dashboard.widgets.length - 1].push(response['widget']);
+              }
+
+              // close the dialog
+              $('#add_query_dialog').modal('hide');
+              reset();
+            }).catch(function() {
+              growl.addErrorMessage("Widget can not be added");
+            }).finally(function() {
+              $scope.saveInProgress = false;
+            });
+          }
+
+        }
+      }
+    }
+  ])
+})();

rd_ui/app/scripts/directives/directives.js

@@ -0,0 +1,219 @@
+(function() {
+    'use strict';
+
+    var directives = angular.module('redash.directives', []);
+
+    directives.directive('alertUnsavedChanges', ['$window', function($window) {
+        return {
+            restrict: 'E',
+            replace: true,
+            scope: {
+                'isDirty': '='
+            },
+            link: function($scope) {
+                var
+
+                unloadMessage = "You will lose your changes if you leave",
+                confirmMessage = unloadMessage + "\n\nAre you sure you want to leave this page?",
+
+                // store original handler (if any)
+                _onbeforeunload = $window.onbeforeunload;
+
+                $window.onbeforeunload = function() {
+                    return $scope.isDirty ? unloadMessage : null;
+                }
+
+                $scope.$on('$locationChangeStart', function(event, next, current) {
+                  if (next.split("#")[0] == current.split("#")[0]) {
+                    return;
+                  }
+
+                  if ($scope.isDirty && !confirm(confirmMessage)) {
+                    event.preventDefault();
+                  }
+                });
+
+                $scope.$on('$destroy', function() {
+                    $window.onbeforeunload = _onbeforeunload;
+                });
+            }
+        }
+    }]);
+
+    directives.directive('rdTab', function() {
+        return {
+            restrict: 'E',
+            scope: {
+                'tabId': '@',
+                'name': '@'
+            },
+            transclude: true,
+            template: '<li class="rd-tab" ng-class="{active: tabId==selectedTab}"><a href="#{{tabId}}">{{name}}<span ng-transclude></span></a></li>',
+            replace: true,
+            link: function(scope) {
+                scope.$watch(function(){return scope.$parent.selectedTab}, function(tab) {
+                    scope.selectedTab = tab;
+                });
+            }
+        }
+    });
+
+    directives.directive('rdTabs', ['$location', function($location) {
+        return {
+            restrict: 'E',
+            scope: {
+                tabsCollection: '=',
+                selectedTab: '='
+            },
+            template: '<ul class="nav nav-tabs"><li ng-class="{active: tab==selectedTab}" ng-repeat="tab in tabsCollection"><a href="#{{tab.key}}">{{tab.name}}</a></li></ul>',
+            replace: true,
+            link: function($scope, element, attrs) {
+                $scope.selectTab = function(tabKey) {
+                    $scope.selectedTab  = _.find($scope.tabsCollection, function(tab) { return tab.key == tabKey; });
+                }
+
+                $scope.$watch(function() { return $location.hash()}, function(hash) {
+                    if (hash) {
+                        $scope.selectTab($location.hash());
+                    } else {
+                        $scope.selectTab($scope.tabsCollection[0].key);
+                    }
+                });
+            }
+        }
+    }]);
+
+    // From: http://jsfiddle.net/joshdmiller/NDFHg/
+    directives.directive('editInPlace', function () {
+        return {
+            restrict: 'E',
+            scope: {
+                value: '=',
+                ignoreBlanks: '=',
+                editable: '=',
+                done: '='
+            },
+            template: function(tElement, tAttrs) {
+                var elType = tAttrs.editor || 'input';
+                var placeholder = tAttrs.placeholder || 'Click to edit';
+                return '<span ng-click="editable && edit()" ng-bind="value" ng-class="{editable: editable}"></span>' +
+                       '<span ng-click="editable && edit()" ng-show="editable && !value" ng-class="{editable: editable}">' + placeholder + '</span>' +
+                       '<{elType} ng-model="value" class="rd-form-control"></{elType}>'.replace('{elType}', elType);
+            },
+            link: function ($scope, element, attrs) {
+                // Let's get a reference to the input element, as we'll want to reference it.
+                var inputElement = angular.element(element.children()[2]);
+
+                // This directive should have a set class so we can style it.
+                element.addClass('edit-in-place');
+
+                // Initially, we're not editing.
+                $scope.editing = false;
+
+                // ng-click handler to activate edit-in-place
+                $scope.edit = function () {
+                    $scope.oldValue = $scope.value;
+
+                    $scope.editing = true;
+
+                    // We control display through a class on the directive itself. See the CSS.
+                    element.addClass('active');
+
+                    // And we must focus the element.
+                    // `angular.element()` provides a chainable array, like jQuery so to access a native DOM function,
+                    // we have to reference the first element in the array.
+                    inputElement[0].focus();
+                };
+
+                function save() {
+                    if ($scope.editing) {
+                        if ($scope.ignoreBlanks && _.isEmpty($scope.value)) {
+                            $scope.value = $scope.oldValue;
+                        }
+                        $scope.editing = false;
+                        element.removeClass('active');
+
+                        if ($scope.value !== $scope.oldValue) {
+                            $scope.done && $scope.done();
+                        }
+                    }
+                }
+
+                $(inputElement).keydown(function(e) {
+                    // 'return' or 'enter' key pressed
+                    // allow 'shift' to break lines
+                    if (e.which === 13 && !e.shiftKey) {
+                        save();
+                    } else if (e.which === 27) {
+                        $scope.value = $scope.oldValue;
+                        $scope.$apply(function() {
+                            $(inputElement[0]).blur();
+                        });
+                    }
+                }).blur(function() {
+                    save();
+                });
+            }
+        };
+    });
+
+    // http://stackoverflow.com/a/17904092/1559840
+    directives.directive('jsonText', function() {
+        return {
+            restrict: 'A',
+            require: 'ngModel',
+            link: function(scope, element, attr, ngModel) {
+              function into(input) {
+                return JSON.parse(input);
+              }
+              function out(data) {
+                return JSON.stringify(data, undefined, 2);
+              }
+              ngModel.$parsers.push(into);
+              ngModel.$formatters.push(out);
+
+              scope.$watch(attr.ngModel, function(newValue) {
+                element[0].value = out(newValue);
+              }, true);
+            }
+        };
+    });
+
+    directives.directive('rdTimer', [function () {
+        return {
+            restrict: 'E',
+            scope: { timestamp: '=' },
+            template: '{{currentTime}}',
+            controller: ['$scope' ,function ($scope) {
+                $scope.currentTime = "00:00:00";
+
+                // We're using setInterval directly instead of $timeout, to avoid using $apply, to
+                // prevent the digest loop being run every second.
+                var currentTimer = setInterval(function() {
+                    $scope.currentTime = moment(moment() - moment($scope.timestamp)).utc().format("HH:mm:ss");
+                    $scope.$digest();
+                }, 1000);
+
+                $scope.$on('$destroy', function () {
+                    if (currentTimer) {
+                        clearInterval(currentTimer);
+                        currentTimer = null;
+                    }
+                });
+            }]
+        };
+    }]);
+
+    directives.directive('rdTimeAgo', function() {
+        return {
+            restrict: 'E',
+            scope: {
+                value: '='
+            },
+            template: '<span>' +
+                        '<span ng-show="value" am-time-ago="value"></span>' +
+                        '<span ng-hide="value">-</span>' +
+                      '</span>'
+        }
+    });
+})();

rd_ui/app/scripts/directives/query_directives.js

@@ -0,0 +1,141 @@
+(function() {
+  'use strict'
+
+  function queryLink() {
+    return {
+      restrict: 'E',
+      scope: {
+        'query': '=',
+        'visualization': '=?'
+      },
+      template: '<a ng-href="{{link}}" class="query-link">{{query.name}}</a>',
+      link: function(scope, element) {
+        scope.link = '/queries/' + scope.query.id;
+        if (scope.visualization) {
+          if (scope.visualization.type === 'TABLE') {
+            // link to hard-coded table tab instead of the (hidden) visualization tab
+            scope.link += '#table';
+          } else {
+            scope.link += '#' + scope.visualization.id;
+          }
+        }
+        // element.find('a').attr('href', link);
+      }
+    }
+  }
+
+  function querySourceLink() {
+    return {
+      restrict: 'E',
+      template: '<span ng-show="query.id && canViewSource">\
+                    <a ng-show="!sourceMode"\
+                      ng-href="{{query.id}}/source#{{selectedTab}}">Show Source\
+                    </a>\
+                    <a ng-show="sourceMode"\
+                      ng-href="/queries/{{query.id}}#{{selectedTab}}">Hide Source\
+                    </a>\
+                </span>'
+    }
+  }
+
+  function queryEditor() {
+    return {
+      restrict: 'E',
+      scope: {
+        'query': '=',
+        'lock': '='
+      },
+      template: '<textarea\
+                  ui-codemirror="editorOptions"\
+                  ng-model="query.query">',
+      link: function($scope) {
+        $scope.editorOptions = {
+            mode: 'text/x-sql',
+            lineWrapping: true,
+            lineNumbers: true,
+            readOnly: false,
+            matchBrackets: true,
+            autoCloseBrackets: true
+        };
+
+        $scope.$watch('lock', function(locked) {
+          $scope.editorOptions.readOnly = locked ? 'nocursor' : false;
+        });
+      }
+    }
+  }
+
+  function queryFormatter($http) {
+    return {
+      restrict: 'E',
+      // don't create new scope to avoid ui-codemirror bug
+      // seehttps://github.com/angular-ui/ui-codemirror/pull/37
+      scope: false,
+      template: '<button type="button" class="btn btn-default btn-xs"\
+                   ng-click="formatQuery()">\
+                    <span class="glyphicon glyphicon-indent-left"></span>\
+                     Format SQL\
+                </button>',
+      link: function($scope) {
+        $scope.formatQuery = function formatQuery() {
+            $scope.queryExecuting = true;
+            $http.post('/api/queries/format', {
+                'query': $scope.query.query
+            }).success(function (response) {
+                $scope.query.query = response;
+            }).finally(function () {
+              $scope.queryExecuting = false;
+            });
+        };
+      }
+    }
+  }
+
+  function queryRefreshSelect() {
+    return {
+      restrict: 'E',
+      template: '<select\
+                  ng-disabled="!isQueryOwner"\
+                  ng-model="query.ttl"\
+                  ng-change="saveQuery()"\
+                  ng-options="c.value as c.name for c in refreshOptions">\
+                  </select>',
+      link: function($scope) {
+        $scope.refreshOptions = [
+            {
+                value: -1,
+                name: 'No Refresh'
+            },
+            {
+                value: 60,
+                name: 'Every minute'
+            },
+        ]
+
+        _.each(_.range(1, 13), function (i) {
+            $scope.refreshOptions.push({
+                value: i * 3600,
+                name: 'Every ' + i + 'h'
+            });
+        })
+
+        $scope.refreshOptions.push({
+            value: 24 * 3600,
+            name: 'Every 24h'
+        });
+        $scope.refreshOptions.push({
+            value: 7 * 24 * 3600,
+            name: 'Once a week'
+        });
+      }
+
+    }
+  }
+
+  angular.module('redash.directives')
+  .directive('queryLink', queryLink)
+  .directive('querySourceLink', querySourceLink)
+  .directive('queryEditor', queryEditor)
+  .directive('queryRefreshSelect', queryRefreshSelect)
+  .directive('queryFormatter', ['$http', queryFormatter]);
+})();

rd_ui/app/scripts/filters.js

@@ -47,4 +47,15 @@ angular.module('redash.filters', []).
             }
             return 12;
         }
+    })
+    
+    .filter('capitalize', function () {
+        return function (text) {
+            if (text) {
+                return text[0].toUpperCase() + text.slice(1).toLowerCase();
+            } else {
+                return null;
+            }
+            
+        }
     });

rd_ui/app/scripts/ng-highchart.js

@@ -1,181 +0,0 @@
-(function(){
-    'use strict';
-
-    var defaultOptions = {
-        title: {
-            "text": null
-        },
-        tooltip: {
-            valueDecimals: 2,
-            formatter: function () {
-                if (moment.isMoment(this.x)) {
-                    var s = '<b>' + moment(this.x).format("DD/MM/YY HH:mm") + '</b>',
-                        pointsCount = this.points.length;
-
-                    $.each(this.points, function (i, point) {
-                        s += '<br/><span style="color:'+point.series.color+'">' + point.series.name + '</span>: ' +
-                            Highcharts.numberFormat(point.y);
-
-                        if (pointsCount > 1 && point.percentage) {
-                            s += " (" + Highcharts.numberFormat(point.percentage) + "%)";
-                        }
-                    });
-                } else {
-                    var s = "<b>" + this.points[0].key + "</b>";
-                    $.each(this.points, function (i, point) {
-                        s+= '<br/><span style="color:'+point.series.color+'">' + point.series.name + '</span>: ' +
-                            Highcharts.numberFormat(point.y);
-                    });
-                }
-
-                return s;
-            },
-            shared: true
-        },
-        xAxis: {
-            type: 'datetime'
-        },
-        yAxis: {
-            title: {
-                text: null
-            }
-        },
-        exporting: {
-            chartOptions: {
-                title: {
-                    text: ''
-                }
-            },
-            buttons: {
-                contextButton: {
-                    menuItems: [
-                        {
-                            text: 'Toggle % Stacking',
-                            onclick: function () {
-                                var newStacking = "normal";
-                                if (this.series[0].options.stacking == "normal") {
-                                    newStacking = "percent";
-                                }
-
-                                _.each(this.series, function (series) {
-                                    series.update({stacking: newStacking}, true);
-                                });
-                            }
-                        }
-                    ]
-                }
-            }
-        },
-        credits: {
-            enabled: false
-        },
-        plotOptions: {
-            "column": {
-                "stacking": "normal",
-                "pointPadding": 0,
-                "borderWidth": 1,
-                "groupPadding": 0,
-                "shadow": false
-            }
-        },
-        series: []
-    };
-
-    angular.module('highchart', [])
-        .directive('chart', ['$timeout', function ($timeout) {
-            return {
-                restrict: 'E',
-                template: '<div></div>',
-                scope: {
-                    options: "=options",
-                    series: "=series"
-                },
-                transclude: true,
-                replace: true,
-
-                link: function (scope, element, attrs) {
-                    var chartsDefaults = {
-                        chart: {
-                            renderTo: element[0],
-                            type: attrs.type || null,
-                            height: attrs.height || null,
-                            width: attrs.width || null
-                        }
-                    };
-
-                    var chartOptions = $.extend(true, {}, defaultOptions, chartsDefaults);
-
-                    // Update when options change
-                    scope.$watch('options', function(newOptions) {
-                        initChart(newOptions);
-                    }, true);
-
-                    //Update when charts data changes
-                    scope.$watch(function () {
-                        return (scope.series && scope.series.length) || 0;
-                    }, function (length) {
-                        if (!length || length == 0) {
-                            scope.chart.showLoading();
-                        } else {
-                            drawChart();
-                        };
-                    }, true);
-
-                    function initChart(options) {
-                        if (scope.chart) {
-                           scope.chart.destroy();
-                        }
-
-                        $.extend(true, chartOptions, options);
-
-                        scope.chart = new Highcharts.Chart(chartOptions);
-                        drawChart();
-                    }
-
-                    function drawChart() {
-                        while(scope.chart.series.length > 0) {
-                            scope.chart.series[0].remove(true);
-                        }
-
-                        // todo series.type
-
-                        if (_.some(scope.series[0].data, function(p) { return angular.isString(p.x) })) {
-                            scope.chart.xAxis[0].update({type: 'category'});
-
-                            // We need to make sure that for each category, each series has a value.
-                            var categories = _.union.apply(this, _.map(scope.series, function(s) { return _.pluck(s.data,'x')}));
-
-                            _.each(scope.series, function(s) {
-                                // TODO: move this logic to Query#getChartData
-                                var yValues = _.groupBy(s.data, 'x');
-
-                                var newData = _.sortBy(_.map(categories, function(category) {
-                                    return {
-                                        name: category,
-                                        y: yValues[category] && yValues[category][0].y
-                                    }
-                                }), 'name');
-
-                                s.data = newData;
-                            });
-                        } else {
-                            scope.chart.xAxis[0].update({type: 'datetime'});
-                        }
-
-                        scope.chart.counters.color = 0;
-
-                        _.each(scope.series, function(s) {
-                            // here we override the series with the visualization config
-                            var _s = $.extend(true, {}, s, chartOptions['series']);
-                            scope.chart.addSeries(_s);
-                        })
-
-                        scope.chart.redraw();
-                        scope.chart.hideLoading();
-                    }
-
-                }
-            };
-
-        }]);
-})();

rd_ui/app/scripts/ng_highchart.js

@@ -0,0 +1,290 @@
+(function () {
+  'use strict';
+
+  Highcharts.setOptions({
+    colors: ["#4572A7", "#AA4643", "#89A54E", "#80699B", "#3D96AE",
+      "#DB843D", "#92A8CD", "#A47D7C", "#B5CA92"]
+  });
+
+  var defaultOptions = {
+    title: {
+      "text": null
+    },
+    xAxis: {
+      type: 'datetime'
+    },
+    yAxis: {
+      title: {
+        text: null
+      }
+    },
+    tooltip: {
+      valueDecimals: 2,
+      formatter: function () {
+        if (!this.points) {
+          this.points = [this.point];
+        }
+        ;
+
+        if (moment.isMoment(this.x)) {
+          var s = '<b>' + moment(this.x).format("DD/MM/YY HH:mm") + '</b>',
+              pointsCount = this.points.length;
+
+          $.each(this.points, function (i, point) {
+            s += '<br/><span style="color:' + point.series.color + '">' + point.series.name + '</span>: ' +
+                Highcharts.numberFormat(point.y);
+
+            if (pointsCount > 1 && point.percentage) {
+              s += " (" + Highcharts.numberFormat(point.percentage) + "%)";
+            }
+          });
+        } else {
+          var points = this.points;
+          var name = points[0].key || points[0].name;
+
+          var s = "<b>" + name + "</b>";
+
+          $.each(points, function (i, point) {
+            if (points.length > 1) {
+              s += '<br/><span style="color:' + point.series.color + '">' + point.series.name + '</span>: ' + Highcharts.numberFormat(point.y);
+            } else {
+              s += ": " + Highcharts.numberFormat(point.y);
+              if (point.percentage < 100) {
+                s += ' (' + Highcharts.numberFormat(point.percentage) + '%)';
+              }
+            }
+          });
+        }
+
+        return s;
+      },
+      shared: true
+    },
+    exporting: {
+      chartOptions: {
+        title: {
+          text: ''
+        }
+      },
+      buttons: {
+        contextButton: {
+          menuItems: [
+            {
+              text: 'Toggle % Stacking',
+              onclick: function () {
+                var newStacking = "normal";
+                if (this.series[0].options.stacking == "normal") {
+                  newStacking = "percent";
+                }
+
+                _.each(this.series, function (series) {
+                  series.update({stacking: newStacking}, true);
+                });
+              }
+            }
+          ]
+        }
+      }
+    },
+    credits: {
+      enabled: false
+    },
+    plotOptions: {
+      area: {
+        marker: {
+          enabled: false,
+          symbol: 'circle',
+          radius: 2,
+          states: {
+            hover: {
+              enabled: true
+            }
+          }
+        }
+      },
+      column: {
+        stacking: "normal",
+        pointPadding: 0,
+        borderWidth: 1,
+        groupPadding: 0,
+        shadow: false
+      },
+      line: {
+        marker: {
+          radius: 1
+        },
+        lineWidth: 2,
+        states: {
+          hover: {
+            lineWidth: 2,
+            marker: {
+              radius: 3
+            }
+          }
+        }
+      },
+      pie: {
+        allowPointSelect: true,
+        cursor: 'pointer',
+        dataLabels: {
+          enabled: true,
+          color: '#000000',
+          connectorColor: '#000000',
+          format: '<b>{point.name}</b>: {point.y} ({point.percentage:.1f} %)'
+        }
+      },
+      scatter: {
+        marker: {
+          radius: 5,
+          states: {
+            hover: {
+              enabled: true,
+              lineColor: 'rgb(100,100,100)'
+            }
+          }
+        },
+        tooltip: {
+          headerFormat: '<b>{series.name}</b><br>',
+          pointFormat: '{point.x}, {point.y}'
+        }
+      }
+    },
+    series: []
+  };
+
+  angular.module('highchart', [])
+      .directive('chart', ['$timeout', function ($timeout) {
+        return {
+          restrict: 'E',
+          template: '<div></div>',
+          scope: {
+            options: "=options",
+            series: "=series"
+          },
+          transclude: true,
+          replace: true,
+
+          link: function (scope, element, attrs) {
+            var chartsDefaults = {
+              chart: {
+                renderTo: element[0],
+                type: attrs.type || null,
+                height: attrs.height || null,
+                width: attrs.width || null
+              }
+            };
+
+            var chartOptions = $.extend(true, {}, defaultOptions, chartsDefaults);
+
+            // $timeout makes sure that this function invoked after the DOM ready. When draw/init
+            // invoked after the DOM is ready, we see first an empty HighCharts objects and later
+            // they get filled up. Which gives the feeling that the charts loading faster (otherwise
+            // we stare at an empty screen until the HighCharts object is ready).
+            $timeout(function () {
+              // Update when options change
+              scope.$watch('options', function (newOptions) {
+                initChart(newOptions);
+              }, true);
+
+              //Update when charts data changes
+              scope.$watchCollection('series', function (series) {
+                if (!series || series.length == 0) {
+                  scope.chart.showLoading();
+                } else {
+                  drawChart();
+                }
+                ;
+              });
+            });
+
+            function initChart(options) {
+              if (scope.chart) {
+                scope.chart.destroy();
+              }
+              ;
+
+              $.extend(true, chartOptions, options);
+
+              scope.chart = new Highcharts.Chart(chartOptions);
+              drawChart();
+            }
+
+            function drawChart() {
+              while (scope.chart.series.length > 0) {
+                scope.chart.series[0].remove(false);
+              };
+
+              if (!('xAxis' in chartOptions && 'type' in chartOptions['xAxis'])) {
+                if (scope.series.length > 0 && _.some(scope.series[0].data, function (p) {
+                  return (angular.isString(p.x) || angular.isDefined(p.name));
+                })) {
+                  chartOptions['xAxis'] = chartOptions['xAxis'] || {};
+                  chartOptions['xAxis']['type'] = 'category';
+                } else {
+                  chartOptions['xAxis'] = chartOptions['xAxis'] || {};
+                  chartOptions['xAxis']['type'] = 'datetime';
+                }
+              }
+
+              if (chartOptions['xAxis']['type'] == 'category' || chartOptions['series']['type']=='pie') {
+                if (!angular.isDefined(scope.series[0].data[0].name)) {
+                  // We need to make sure that for each category, each series has a value.
+                  var categories = _.union.apply(this, _.map(scope.series, function (s) {
+                    return _.pluck(s.data, 'x')
+                  }));
+
+                  _.each(scope.series, function (s) {
+                    // TODO: move this logic to Query#getChartData
+                    var yValues = _.groupBy(s.data, 'x');
+
+                    var newData = _.map(categories, function (category) {
+                      return {
+                        name: category,
+                        y: (yValues[category] && yValues[category][0].y) || 0
+                      }
+                    });
+
+                    if (categories.length == 1) {
+                      newData = _.sortBy(newData, 'y').reverse();
+                    }
+                    ;
+
+                    s.data = newData;
+                  });
+                }
+              }
+
+              scope.chart.counters.color = 0;
+
+              _.each(scope.series, function (s) {
+                // here we override the series with the visualization config
+                s = _.extend(s, chartOptions['series']);
+
+                if (s.type == 'area') {
+                  _.each(s.data, function (p) {
+                    // This is an insane hack: somewhere deep in HighChart's code,
+                    // when you stack areas, it tries to convert the string representation
+                    // of point's x into a number. With the default implementation of toString
+                    // it fails....
+
+                    if (moment.isMoment(p.x)) {
+                      p.x.toString = function () {
+                        return String(this.toDate().getTime());
+                      };
+                    }
+                  });
+                }
+                ;
+
+                scope.chart.addSeries(s, false);
+              });
+
+              scope.chart.redraw();
+              scope.chart.hideLoading();
+            }
+
+          }
+        };
+
+      }]);
+})();

rd_ui/app/scripts/query_fiddle/renderers.js

@@ -1,203 +0,0 @@
-var renderers = angular.module('redash.renderers', []);
-
-renderers.directive('visualizationRenderer', function() {
-    return {
-        restrict: 'E',
-        scope: {
-            visualization: '=',
-            queryResult: '='
-        },
-        template: '<div ng-switch on="visualization.type">' +
-                    '<chart-renderer ng-switch-when="CHART" options="visualization.options" query-result="queryResult"></chart-renderer>' +
-                    '<grid-renderer ng-switch-when="GRID" options="visualization.options" query-result="queryResult"></grid-renderer>' +
-                    '<cohort-renderer ng-switch-when="COHORT" options="visualization.options" query-result="queryResult"></cohort-renderer>' +
-                   '</div>',
-        replace: false
-    }
-});
-
-renderers.directive('chartRenderer', function () {
-    return {
-        restrict: 'E',
-        scope: {
-            queryResult: '=',
-            options: '=?'
-        },
-        template: "<chart options='chartOptions' series='chartSeries' class='graph'></chart>",
-        replace: false,
-        controller: ['$scope', function ($scope) {
-            $scope.chartSeries = [];
-            $scope.chartOptions = {};
-
-            $scope.$watch('options', function(chartOptions) {
-                if (chartOptions) {
-                    $scope.chartOptions = chartOptions;
-                }
-            });
-            $scope.$watch('queryResult && queryResult.getData()', function (data) {
-                if (!data || $scope.queryResult.getData() == null) {
-                    $scope.chartSeries.splice(0, $scope.chartSeries.length);
-                } else {
-                    $scope.chartSeries.splice(0, $scope.chartSeries.length);
-
-                    _.each($scope.queryResult.getChartData(), function (s) {
-                        $scope.chartSeries.push(_.extend(s, {'stacking': 'normal'}));
-                    });
-                }
-            });
-        }]
-    }
-})
-
-renderers.directive('gridRenderer', function () {
-    return {
-        restrict: 'E',
-        scope: {
-            queryResult: '=',
-            itemsPerPage: '='
-        },
-        templateUrl: "/views/grid_renderer.html",
-        replace: false,
-        controller: ['$scope', function ($scope) {
-            $scope.gridColumns = [];
-            $scope.gridData = [];
-            $scope.gridConfig = {
-                isPaginationEnabled: true,
-                itemsByPage: $scope.itemsPerPage || 15,
-                maxSize: 8
-            };
-
-            $scope.$watch('queryResult && queryResult.getData()', function (data) {
-                if (!data) {
-                    return;
-                }
-
-                if ($scope.queryResult.getData() == null) {
-                    $scope.gridColumns = [];
-                    $scope.gridData = [];
-                    $scope.filters = [];
-                } else {
-
-
-                    $scope.filters = $scope.queryResult.getFilters();
-
-                    var gridData = _.map($scope.queryResult.getData(), function (row) {
-                        var newRow = {};
-                        _.each(row, function (val, key) {
-                            newRow[$scope.queryResult.getColumnCleanName(key)] = val;
-                        })
-                        return newRow;
-                    });
-
-                    $scope.gridColumns = _.map($scope.queryResult.getColumnCleanNames(), function (col, i) {
-                        var columnDefinition = {
-                            'label': $scope.queryResult.getColumnFriendlyNames()[i],
-                            'map': col
-                        };
-
-                        if (gridData.length > 0) {
-                            var exampleData = gridData[0][col];
-                            if (angular.isNumber(exampleData)) {
-                                columnDefinition['formatFunction'] = 'number';
-                                columnDefinition['formatParameter'] = 2;
-                            } else if (moment.isMoment(exampleData)) {
-                                columnDefinition['formatFunction'] = function(value) {
-                                    return value.format("DD/MM/YY HH:mm");
-                                }
-                            }
-                        }
-
-                        return columnDefinition;
-                    });
-
-                    $scope.gridData = _.clone(gridData);
-
-                    $scope.$watch('filters', function (filters) {
-                        $scope.gridData = _.filter(gridData, function (row) {
-                            return _.reduce(filters, function (memo, filter) {
-                                if (filter.current == 'All') {
-                                    return memo && true;
-                                }
-
-                                return (memo && row[$scope.queryResult.getColumnCleanName(filter.name)] == filter.current);
-                            }, true);
-                        });
-                    }, true);
-                }
-            });
-        }]
-    }
-})
-
-renderers.directive('pivotTableRenderer', function () {
-    return {
-        restrict: 'E',
-        scope: {
-            queryResult: '='
-        },
-        template: "",
-        replace: false,
-        link: function($scope, element, attrs) {
-            $scope.$watch('queryResult && queryResult.getData()', function (data) {
-                if (!data) {
-                    return;
-                }
-
-                if ($scope.queryResult.getData() == null) {
-                } else {
-                    $(element).pivotUI($scope.queryResult.getData(), {
-                         renderers: $.pivotUtilities.renderers
-                    }, true);
-                }
-            });
-        }
-    }
-})
-
-renderers.directive('cohortRenderer', function() {
-    return {
-        restrict: 'E',
-        scope: {
-            queryResult: '='
-        },
-        template: "",
-        replace: false,
-        link: function($scope, element, attrs) {
-            $scope.$watch('queryResult && queryResult.getData()', function (data) {
-                if (!data) {
-                    return;
-                }
-
-                if ($scope.queryResult.getData() == null) {
-
-                } else {
-                    var sortedData = _.sortBy($scope.queryResult.getData(), "date");
-                    var grouped = _.groupBy(sortedData, "date");
-                    var data = _.map(grouped, function(values, date) {
-                       var row = [values[0].total];
-                        _.each(values, function(value) { row.push(value.value); });
-                        return row;
-                    });
-
-                    var initialDate = moment(sortedData[0].date).toDate(),
-                        container = angular.element(element)[0];
-
-                    Cornelius.draw({
-                        initialDate: initialDate,
-                        container: container,
-                        cohort: data,
-                        title: null,
-                        timeInterval: 'daily',
-                        labels: {
-                            time: 'Activation Day',
-                            people: 'Users'
-                        },
-                        formatHeaderLabel: function (i) {
-                            return "Day " + (i - 1);
-                        }
-                    });
-                }
-            });
-        }
-    }
-})

rd_ui/app/scripts/services.js

@@ -1,310 +0,0 @@
-(function () {
-    var QueryResult = function($resource, $timeout) {
-        var QueryResultResource = $resource('/api/query_results/:id', {id: '@id'}, {'post': {'method': 'POST'}});
-        var Job = $resource('/api/jobs/:id', {id: '@id'});
-
-        var updateFunction = function (props) {
-            angular.extend(this, props);
-            if ('query_result' in props) {
-                this.status = "done";
-
-                _.each(this.query_result.data.rows, function (row) {
-                    _.each(row, function (v, k) {
-                        if (_.isString(v) && v.match(/^\d{4}-\d{2}-\d{2}/)) {
-                            row[k] = moment(v);
-                        }
-                    });
-                });
-            } else if (this.job.status == 3) {
-                this.status = "processing";
-            } else {
-                this.status = undefined;
-            }
-        }
-
-        function QueryResult(props) {
-            this.job = {};
-            this.query_result = {};
-            this.status = "waiting";
-
-            this.updatedAt = moment();
-
-            if (props) {
-                updateFunction.apply(this, [props]);
-            }
-        }
-
-        var statuses = {
-            1: "waiting",
-            2: "processing",
-            3: "done",
-            4: "failed"
-        }
-
-        QueryResult.prototype.update = updateFunction;
-
-        QueryResult.prototype.getId = function() {
-            var id = null;
-            if ('query_result' in this) {
-                id = this.query_result.id;
-            }
-            return id;
-        }
-
-        QueryResult.prototype.cancelExecution = function() {
-            Job.delete({id: this.job.id});
-        }
-
-        QueryResult.prototype.getStatus = function() {
-            return this.status || statuses[this.job.status];
-        }
-
-        QueryResult.prototype.getError = function() {
-            // TODO: move this logic to the server...
-            if (this.job.error == "None") {
-                return undefined;
-            }
-
-            return this.job.error;
-        }
-
-        QueryResult.prototype.getUpdatedAt = function() {
-            return this.query_result.retrieved_at || this.job.updated_at*1000.0 || this.updatedAt;
-        }
-
-        QueryResult.prototype.getRuntime = function() {
-            return this.query_result.runtime;
-        }
-
-        QueryResult.prototype.getData = function() {
-            if (!this.query_result.data) {
-                return null;
-            }
-
-            var data = this.query_result.data.rows;
-            return data;
-        }
-
-        QueryResult.prototype.getChartData = function () {
-            var series = {};
-
-            _.each(this.getData(), function (row) {
-                var point = {};
-                var seriesName = undefined;
-                var xValue = 0;
-                var yValues = {};
-
-                _.each(row, function (value, definition) {
-                    var type = definition.split("::")[1];
-                    var name = definition.split("::")[0];
-
-                    if (type == 'x') {
-                        xValue = value;
-                        point[type] = value;
-                    }
-
-                    if (type == 'y') {
-                        yValues[name] = value;
-                        point[type] = value;
-                    }
-
-                    if (type == 'series') {
-                        seriesName = String(value);
-                    }
-                });
-
-                var addPointToSeries = function(seriesName, point) {
-                    if (series[seriesName] == undefined) {
-                        series[seriesName] = {
-                            name: seriesName,
-                            type: 'column',
-                            data: []
-                        }
-                    }
-
-                    series[seriesName]['data'].push(point);
-                }
-
-                if (seriesName === undefined) {
-                    _.each(yValues, function(yValue, seriesName) {
-                        addPointToSeries(seriesName, {'x': xValue, 'y': yValue});
-                    });
-                } else {
-                    addPointToSeries(seriesName, point);
-                }
-            });
-
-            _.each(series, function(series) {
-                series.data = _.sortBy(series.data, 'x');
-            });
-
-            return _.values(series);
-        };
-
-        QueryResult.prototype.getColumns = function () {
-            if (this.columns == undefined) {
-                this.columns = _.map(this.query_result.data.columns, function(v) {
-                    return v.name;
-                })
-            }
-
-            return this.columns;
-        }
-
-        QueryResult.prototype.getColumnCleanName = function (column) {
-            var parts = column.split('::');
-            var name = parts[1];
-            if (parts[0] != '') {
-                // TODO: it's probably time to generalize this.
-                // see also getColumnFriendlyName
-                name = parts[0].replace(/%/g, '__pct').replace(/ /g, '_').replace(/\?/g,'');
-            }
-            return name;
-        }
-
-        QueryResult.prototype.getColumnFriendlyName = function (column) {
-            return this.getColumnCleanName(column).replace('__pct', '%').replace(/_/g, ' ').replace(/(?:^|\s)\S/g, function (a) {
-                return a.toUpperCase();
-            });
-        }
-
-        QueryResult.prototype.getColumnCleanNames = function () {
-            return _.map(this.getColumns(), function (col) {
-                return this.getColumnCleanName(col);
-            }, this);
-        }
-
-        QueryResult.prototype.getColumnFriendlyNames = function () {
-            return _.map(this.getColumns(), function (col) {
-                return this.getColumnFriendlyName(col);
-            }, this);
-        }
-
-        QueryResult.prototype.getFilters = function () {
-            var filterNames = [];
-            _.each(this.getColumns(), function (col) {
-                if (col.split('::')[1] == 'filter') {
-                    filterNames.push(col);
-                }
-            });
-
-            var filterValues = [];
-            _.each(this.getData(), function (row) {
-                _.each(filterNames, function (filter, i) {
-                    if (filterValues[i] == undefined) {
-                        filterValues[i] = [];
-                    }
-                    filterValues[i].push(row[filter]);
-                })
-            });
-
-            var filters = _.map(filterNames, function (filter, i) {
-                var f = {
-                    name: filter,
-                    friendlyName: this.getColumnFriendlyName(filter),
-                    values: _.uniq(filterValues[i])
-                };
-
-                f.current = f.values[0];
-                return f;
-            }, this);
-
-            return filters;
-        };
-
-        var refreshStatus = function(queryResult, query, ttl) {
-            Job.get({'id': queryResult.job.id}, function(response) {
-                queryResult.update(response);
-
-                if (queryResult.getStatus() == "processing" && queryResult.job.query_result_id && queryResult.job.query_result_id != "None") {
-                    QueryResultResource.get({'id': queryResult.job.query_result_id}, function(response) {
-                        queryResult.update(response);
-                    });
-                } else if (queryResult.getStatus() != "failed") {
-                    $timeout(function () {
-                        refreshStatus(queryResult, query, ttl);
-                    }, 3000);
-                }
-            })
-        }
-
-        QueryResult.getById = function (id) {
-            var queryResult = new QueryResult();
-
-            QueryResultResource.get({'id': id}, function (response) {
-                queryResult.update(response);
-            });
-
-            return queryResult;
-        }
-
-        QueryResult.get = function (query, ttl) {
-            var queryResult = new QueryResult();
-
-            QueryResultResource.post({'query': query, 'ttl': ttl}, function (response) {
-                queryResult.update(response);
-
-                if ('job' in response) {
-                    refreshStatus(queryResult, query, ttl);
-                }
-            });
-
-            return queryResult;
-        }
-
-        return QueryResult;
-    };
-
-    var Query = function ($resource, QueryResult) {
-        var Query = $resource('/api/queries/:id', {id: '@id'});
-
-        Query.prototype.getQueryResult = function(ttl) {
-            if (ttl == undefined) {
-                ttl = this.ttl;
-            }
-
-
-            var queryResult = null;
-            if (this.latest_query_data && ttl != 0) {
-                queryResult = new QueryResult({'query_result': this.latest_query_data});
-            } else if (this.latest_query_data_id && ttl != 0) {
-                queryResult = QueryResult.getById(this.latest_query_data_id);
-            } else {
-                queryResult = QueryResult.get(this.query, ttl);
-            }
-
-            return queryResult;
-        };
-
-        Query.prototype.getHash = function() {
-            return [this.name, this.description, this.query].join('!#');
-        };
-
-        return Query;
-    };
-
-    var Visualization = function($resource) {
-        var Visualization = $resource('/api/visualizations/:id', {id: '@id'});
-
-        Visualization.prototype = {
-            TYPES: {
-                'CHART': 'CHART',
-                'COHORT': 'COHORT',
-                'GRID': 'GRID'
-            },
-            SERIES_TYPES: {
-                'LINE': 'line',
-                'BAR': 'bar',
-                'AREA': 'area'
-            }
-        };
-
-        return Visualization;
-    };
-
-    angular.module('redash.services', [])
-        .factory('QueryResult', ['$resource', '$timeout', QueryResult])
-        .factory('Query', ['$resource', 'QueryResult', Query])
-        .factory('Visualization', ['$resource', Visualization])
-
-})();

rd_ui/app/scripts/services/dashboards.js

@@ -2,7 +2,7 @@
     var Dashboard = function($resource) {
         var resource = $resource('/api/dashboards/:slug', {slug: '@slug'});
         resource.prototype.canEdit = function() {
-            return currentUser.is_admin || currentUser.canEdit(this);
+            return currentUser.hasPermission('admin') || currentUser.canEdit(this);
         }
         return resource;
     }

rd_ui/app/scripts/services/notifications.js

@@ -1,5 +1,5 @@
 (function () {
-    var notifications = function () {
+    var notifications = function (Events) {
         var notificationService = {};
         var lastNotification = null;
 
@@ -40,6 +40,7 @@
             notification.onclick = function () {
                 window.focus();
                 this.cancel();
+                Events.record(currentUser, 'click', 'notification');
             };
 
             notification.show()
@@ -49,5 +50,5 @@
     }
 
     angular.module('redash.services')
-        .factory('notifications', notifications);
+        .factory('notifications', ['Events', notifications]);
 })();

rd_ui/app/scripts/services/resources.js

@@ -0,0 +1,374 @@
+(function () {
+  var QueryResult = function ($resource, $timeout) {
+    var QueryResultResource = $resource('/api/query_results/:id', {id: '@id'}, {'post': {'method': 'POST'}});
+    var Job = $resource('/api/jobs/:id', {id: '@id'});
+
+    var updateFunction = function (props) {
+      angular.extend(this, props);
+      if ('query_result' in props) {
+        this.status = "done";
+        this.filters = undefined;
+        this.filterFreeze = undefined;
+
+        _.each(this.query_result.data.rows, function (row) {
+          _.each(row, function (v, k) {
+            if (_.isString(v) && v.match(/^\d{4}-\d{2}-\d{2}/)) {
+              row[k] = moment(v);
+            }
+          });
+        });
+      } else if (this.job.status == 3) {
+        this.status = "processing";
+      } else {
+        this.status = undefined;
+      }
+    }
+
+    function QueryResult(props) {
+      this.job = {};
+      this.query_result = {};
+      this.status = "waiting";
+      this.filters = undefined;
+      this.filterFreeze = undefined;
+
+      this.updatedAt = moment();
+
+      if (props) {
+        updateFunction.apply(this, [props]);
+      }
+    }
+
+    var statuses = {
+      1: "waiting",
+      2: "processing",
+      3: "done",
+      4: "failed"
+    }
+
+    QueryResult.prototype.update = updateFunction;
+
+    QueryResult.prototype.getId = function () {
+      var id = null;
+      if ('query_result' in this) {
+        id = this.query_result.id;
+      }
+      return id;
+    }
+
+    QueryResult.prototype.cancelExecution = function () {
+      Job.delete({id: this.job.id});
+    }
+
+    QueryResult.prototype.getStatus = function () {
+      return this.status || statuses[this.job.status];
+    }
+
+    QueryResult.prototype.getError = function () {
+      // TODO: move this logic to the server...
+      if (this.job.error == "None") {
+        return undefined;
+      }
+
+      return this.job.error;
+    }
+
+    QueryResult.prototype.getUpdatedAt = function () {
+      return this.query_result.retrieved_at || this.job.updated_at * 1000.0 || this.updatedAt;
+    }
+
+    QueryResult.prototype.getRuntime = function () {
+      return this.query_result.runtime;
+    }
+
+    QueryResult.prototype.getRawData = function () {
+      if (!this.query_result.data) {
+        return null;
+      }
+
+      var data = this.query_result.data.rows;
+
+      return data;
+    }
+
+    QueryResult.prototype.getData = function () {
+      if (!this.query_result.data) {
+        return null;
+      }
+
+      var filterValues = function (filters) {
+        if (!filters) {
+          return null;
+        }
+
+        return _.reduce(filters, function (str, filter) {
+          return str + filter.current;
+        }, "")
+      }
+
+      var filters = this.getFilters();
+      var filterFreeze = filterValues(filters);
+
+      if (this.filterFreeze != filterFreeze) {
+        this.filterFreeze = filterFreeze;
+
+        if (filters) {
+          this.filteredData = _.filter(this.query_result.data.rows, function (row) {
+            return _.reduce(filters, function (memo, filter) {
+              if (!_.isArray(filter.current)) {
+                filter.current = [filter.current];
+              };
+
+              return (memo && _.some(filter.current, function(v) {
+                // We compare with either the value or the String representation of the value,
+                // because Select2 casts true/false to "true"/"false".
+                return v == row[filter.name] || String(row[filter.name]) == v
+              }));
+            }, true);
+          });
+        } else {
+          this.filteredData = this.query_result.data.rows;
+        }
+      }
+
+      return this.filteredData;
+    }
+
+    QueryResult.prototype.getChartData = function () {
+      var series = {};
+
+      _.each(this.getData(), function (row) {
+        var point = {};
+        var seriesName = undefined;
+        var xValue = 0;
+        var yValues = {};
+
+        _.each(row, function (value, definition) {
+          var type = definition.split("::")[1];
+          var name = definition.split("::")[0];
+
+          if (type == 'x') {
+            xValue = value;
+            point[type] = value;
+          }
+          if (type == 'y') {
+            if (value == null) {
+              value = 0;
+            }
+            yValues[name] = value;
+            point[type] = value;
+          }
+
+          if (type == 'series') {
+            seriesName = String(value);
+          }
+
+          if (type == 'multi-filter') {
+            seriesName = String(value);
+          }
+        });
+
+        var addPointToSeries = function (seriesName, point) {
+          if (series[seriesName] == undefined) {
+            series[seriesName] = {
+              name: seriesName,
+              type: 'column',
+              data: []
+            }
+          }
+
+          series[seriesName]['data'].push(point);
+        }
+
+        if (seriesName === undefined) {
+          _.each(yValues, function (yValue, seriesName) {
+            addPointToSeries(seriesName, {'x': xValue, 'y': yValue});
+          });
+        } else {
+          addPointToSeries(seriesName, point);
+        }
+      });
+
+      _.each(series, function (series) {
+        series.data = _.sortBy(series.data, 'x');
+      });
+
+      return _.values(series);
+    };
+
+    QueryResult.prototype.getColumns = function () {
+      if (this.columns == undefined && this.query_result.data) {
+        this.columns = _.map(this.query_result.data.columns, function (v) {
+          return v.name;
+        });
+      }
+
+      return this.columns;
+    }
+
+    QueryResult.prototype.getColumnCleanName = function (column) {
+      var parts = column.split('::');
+      var name = parts[1];
+      if (parts[0] != '') {
+        // TODO: it's probably time to generalize this.
+        // see also getColumnFriendlyName
+        name = parts[0].replace(/%/g, '__pct').replace(/ /g, '_').replace(/\?/g, '');
+      }
+      return name;
+    }
+
+    QueryResult.prototype.getColumnFriendlyName = function (column) {
+      return this.getColumnCleanName(column).replace('__pct', '%').replace(/_/g, ' ').replace(/(?:^|\s)\S/g, function (a) {
+        return a.toUpperCase();
+      });
+    }
+
+    QueryResult.prototype.getColumnCleanNames = function () {
+      return _.map(this.getColumns(), function (col) {
+        return this.getColumnCleanName(col);
+      }, this);
+    }
+
+    QueryResult.prototype.getColumnFriendlyNames = function () {
+      return _.map(this.getColumns(), function (col) {
+        return this.getColumnFriendlyName(col);
+      }, this);
+    }
+
+    QueryResult.prototype.getFilters = function () {
+      if (!this.filters) {
+        this.prepareFilters();
+      }
+
+      return this.filters;
+    };
+
+    QueryResult.prototype.prepareFilters = function () {
+      var filters = [];
+      var filterTypes = ['filter', 'multi-filter'];
+      _.each(this.getColumns(), function (col) {
+        var type = col.split('::')[1]
+        if (_.contains(filterTypes, type)) {
+          // filter found
+          var filter = {
+            name: col,
+            friendlyName: this.getColumnFriendlyName(col),
+            values: [],
+            multiple: (type=='multi-filter')
+          }
+          filters.push(filter);
+        }
+      }, this);
+
+      _.each(this.getRawData(), function (row) {
+        _.each(filters, function (filter) {
+          filter.values.push(row[filter.name]);
+          if (filter.values.length == 1) {
+            filter.current = row[filter.name];
+          }
+        })
+      });
+
+      _.each(filters, function(filter) {
+        filter.values = _.uniq(filter.values);
+      });
+
+      this.filters = filters;
+    }
+
+    var refreshStatus = function (queryResult, query, ttl) {
+      Job.get({'id': queryResult.job.id}, function (response) {
+        queryResult.update(response);
+
+        if (queryResult.getStatus() == "processing" && queryResult.job.query_result_id && queryResult.job.query_result_id != "None") {
+          QueryResultResource.get({'id': queryResult.job.query_result_id}, function (response) {
+            queryResult.update(response);
+          });
+        } else if (queryResult.getStatus() != "failed") {
+          $timeout(function () {
+            refreshStatus(queryResult, query, ttl);
+          }, 3000);
+        }
+      })
+    }
+
+    QueryResult.getById = function (id) {
+      var queryResult = new QueryResult();
+
+      QueryResultResource.get({'id': id}, function (response) {
+        queryResult.update(response);
+      });
+
+      return queryResult;
+    }
+
+    QueryResult.get = function (data_source_id, query, ttl) {
+      var queryResult = new QueryResult();
+
+      QueryResultResource.post({'data_source_id': data_source_id, 'query': query, 'ttl': ttl}, function (response) {
+        queryResult.update(response);
+
+        if ('job' in response) {
+          refreshStatus(queryResult, query, ttl);
+        }
+      });
+
+      return queryResult;
+    }
+
+    return QueryResult;
+  };
+
+  var Query = function ($resource, QueryResult, DataSource) {
+    var Query = $resource('/api/queries/:id', {id: '@id'});
+
+    Query.newQuery = function () {
+      return new Query({
+        query: "",
+        name: "New Query",
+        ttl: -1,
+        user: currentUser
+      });
+    };
+
+    Query.prototype.getSourceLink = function () {
+      return '/queries/' + this.id + '/source';
+    };
+
+    Query.prototype.getQueryResult = function (ttl) {
+      if (ttl == undefined) {
+        ttl = this.ttl;
+      }
+
+      var queryResult = null;
+      if (this.latest_query_data && ttl != 0) {
+        queryResult = new QueryResult({'query_result': this.latest_query_data});
+      } else if (this.latest_query_data_id && ttl != 0) {
+        queryResult = QueryResult.getById(this.latest_query_data_id);
+      } else if (this.data_source_id) {
+        queryResult = QueryResult.get(this.data_source_id, this.query, ttl);
+      }
+
+      return queryResult;
+    };
+
+    return Query;
+  };
+
+  var DataSource = function ($resource) {
+    var DataSourceResource = $resource('/api/data_sources/:id', {id: '@id'}, {'get': {'method': 'GET', 'cache': true, 'isArray': true}});
+
+    return DataSourceResource;
+  }
+
+  var Widget = function ($resource) {
+    var WidgetResource = $resource('/api/widgets/:id', {id: '@id'});
+
+    return WidgetResource;
+  }
+
+  angular.module('redash.services')
+      .factory('QueryResult', ['$resource', '$timeout', QueryResult])
+      .factory('Query', ['$resource', 'QueryResult', 'DataSource', Query])
+      .factory('DataSource', ['$resource', DataSource])
+      .factory('Widget', ['$resource', Widget]);
+})();

rd_ui/app/scripts/services/services.js

@@ -0,0 +1,52 @@
+(function () {
+  'use strict'
+
+  function KeyboardShortcuts() {
+    this.bind = function bind(keymap) {
+      _.forEach(keymap, function (fn, key) {
+        Mousetrap.bindGlobal(key, function (e) {
+          e.preventDefault();
+          fn();
+        });
+      });
+
+    }
+
+    this.unbind = function unbind(keymap) {
+      _.forEach(keymap, function (fn, key) {
+        Mousetrap.unbind(key);
+      });
+    }
+  }
+
+  function Events($http) {
+    this.events = [];
+
+    this.post = _.debounce(function() {
+      var events = this.events;
+      this.events = [];
+
+      $http.post('/api/events', events);
+
+    }, 1000);
+
+    this.record = function (user, action, object_type, object_id, additional_properties) {
+
+      var event = {
+        "user_id": user.id,
+        "action": action,
+        "object_type": object_type,
+        "object_id": object_id,
+        "timestamp": Date.now()/1000.0
+      };
+      _.extend(event, additional_properties);
+      this.events.push(event);
+
+      this.post();
+    };
+  }
+
+  angular.module('redash.services', [])
+      .service('KeyboardShortcuts', [KeyboardShortcuts])
+      .service('Events', ['$http', Events])
+})();

rd_ui/app/scripts/visualizations/base.js

@@ -0,0 +1,179 @@
+(function () {
+  var VisualizationProvider = function () {
+    this.visualizations = {};
+    this.visualizationTypes = {};
+    var defaultConfig = {
+      defaultOptions: {},
+      skipTypes: false,
+      editorTemplate: null
+    }
+
+    this.registerVisualization = function (config) {
+      var visualization = _.extend({}, defaultConfig, config);
+
+      // TODO: this is prone to errors; better refactor.
+      if (_.isEmpty(this.visualizations)) {
+        this.defaultVisualization = visualization;
+      }
+
+      this.visualizations[config.type] = visualization;
+
+      if (!config.skipTypes) {
+        this.visualizationTypes[config.name] = config.type;
+      }
+      ;
+    };
+
+    this.getSwitchTemplate = function (property) {
+      var pattern = /(<[a-zA-Z0-9-]*?)( |>)/
+
+      var mergedTemplates = _.reduce(this.visualizations, function (templates, visualization) {
+        if (visualization[property]) {
+          var ngSwitch = '$1 ng-switch-when="' + visualization.type + '" $2';
+          var template = visualization[property].replace(pattern, ngSwitch);
+
+          return templates + "\n" + template;
+        }
+
+        return templates;
+      }, "");
+
+      mergedTemplates = '<div ng-switch on="visualization.type">' + mergedTemplates + "</div>";
+
+      return mergedTemplates;
+    }
+
+    this.$get = ['$resource', function ($resource) {
+      var Visualization = $resource('/api/visualizations/:id', {id: '@id'});
+      Visualization.visualizations = this.visualizations;
+      Visualization.visualizationTypes = this.visualizationTypes;
+      Visualization.renderVisualizationsTemplate = this.getSwitchTemplate('renderTemplate');
+      Visualization.editorTemplate = this.getSwitchTemplate('editorTemplate');
+      Visualization.defaultVisualization = this.defaultVisualization;
+
+      return Visualization;
+    }];
+  };
+
+  var VisualizationRenderer = function (Visualization) {
+    return {
+      restrict: 'E',
+      scope: {
+        visualization: '=',
+        queryResult: '='
+      },
+      // TODO: using switch here (and in the options editor) might introduce errors and bad
+      // performance wise. It's better to eventually show the correct template based on the
+      // visualization type and not make the browser render all of them.
+      template: '<filters></filters>\n' + Visualization.renderVisualizationsTemplate,
+      replace: false,
+      link: function (scope) {
+        scope.select2Options = {
+          width: '50%'
+        }
+        scope.$watch('queryResult && queryResult.getFilters()', function (filters) {
+          if (filters) {
+            scope.filters = filters;
+          }
+        });
+      }
+    }
+  };
+
+  var VisualizationOptionsEditor = function (Visualization) {
+    return {
+      restrict: 'E',
+      template: Visualization.editorTemplate,
+      replace: false
+    }
+  };
+
+  var Filters = function () {
+    return {
+      restrict: 'E',
+      templateUrl: '/views/visualizations/filters.html'
+    }
+  }
+
+  var EditVisualizationForm = function (Events, Visualization, growl) {
+    return {
+      restrict: 'E',
+      templateUrl: '/views/visualizations/edit_visualization.html',
+      replace: true,
+      scope: {
+        query: '=',
+        queryResult: '=',
+        visualization: '=?',
+        onNewSuccess: '=?'
+      },
+      link: function (scope, element, attrs) {
+        scope.editRawOptions = currentUser.hasPermission('edit_raw_chart');
+        scope.visTypes = Visualization.visualizationTypes;
+
+        scope.newVisualization = function (q) {
+          return {
+            'query_id': q.id,
+            'type': Visualization.defaultVisualization.type,
+            'name': Visualization.defaultVisualization.name,
+            'description': q.description || '',
+            'options': Visualization.defaultVisualization.defaultOptions
+          };
+        }
+
+        if (!scope.visualization) {
+          // create new visualization
+          // wait for query to load to populate with defaults
+          var unwatch = scope.$watch('query', function (q) {
+            if (q && q.id) {
+              unwatch();
+
+              scope.visualization = scope.newVisualization(q);
+            }
+          }, true);
+        }
+
+        scope.$watch('visualization.type', function (type, oldType) {
+          // if not edited by user, set name to match type
+          if (type && oldType != type && scope.visualization && !scope.visForm.name.$dirty) {
+            // poor man's titlecase
+            scope.visualization.name = scope.visualization.type[0] + scope.visualization.type.slice(1).toLowerCase();
+          }
+        });
+
+        scope.submit = function () {
+          if (scope.visualization.id) {
+            Events.record(currentUser, "update", "visualization", scope.visualization.id, {'type': scope.visualization.type});
+          } else {
+            Events.record(currentUser, "create", "visualization", null, {'type': scope.visualization.type});
+          }
+
+          Visualization.save(scope.visualization, function success(result) {
+            growl.addSuccessMessage("Visualization saved");
+
+            scope.visualization = scope.newVisualization(scope.query);
+
+            var visIds = _.pluck(scope.query.visualizations, 'id');
+            var index = visIds.indexOf(result.id);
+            if (index > -1) {
+              scope.query.visualizations[index] = result;
+            } else {
+              // new visualization
+              scope.query.visualizations.push(result);
+              scope.onNewSuccess && scope.onNewSuccess(result);
+            }
+          }, function error() {
+            growl.addErrorMessage("Visualization could not be saved");
+          });
+        };
+      }
+    }
+  };
+
+
+  angular.module('redash.visualization', [])
+      .provider('Visualization', VisualizationProvider)
+      .directive('visualizationRenderer', ['Visualization', VisualizationRenderer])
+      .directive('visualizationOptionsEditor', ['Visualization', VisualizationOptionsEditor])
+      .directive('filters', Filters)
+      .directive('editVisulatizationForm', ['Events', 'Visualization', 'growl', EditVisualizationForm])
+})();

rd_ui/app/scripts/visualizations/chart.js

@@ -0,0 +1,123 @@
+(function () {
+  var chartVisualization = angular.module('redash.visualization');
+
+  chartVisualization.config(['VisualizationProvider', function (VisualizationProvider) {
+    var renderTemplate = '<chart-renderer options="visualization.options" query-result="queryResult"></chart-renderer>';
+    var editTemplate = '<chart-editor></chart-editor>';
+    var defaultOptions = {
+      'series': {
+        'type': 'column',
+        'stacking': null
+      }
+    };
+
+    VisualizationProvider.registerVisualization({
+      type: 'CHART',
+      name: 'Chart',
+      renderTemplate: renderTemplate,
+      editorTemplate: editTemplate,
+      defaultOptions: defaultOptions
+    });
+  }]);
+
+  chartVisualization.directive('chartRenderer', function () {
+    return {
+      restrict: 'E',
+      scope: {
+        queryResult: '=',
+        options: '=?'
+      },
+      template: "<chart options='chartOptions' series='chartSeries' class='graph'></chart>",
+      replace: false,
+      controller: ['$scope', function ($scope) {
+        $scope.chartSeries = [];
+        $scope.chartOptions = {};
+
+        $scope.$watch('options', function (chartOptions) {
+          if (chartOptions) {
+            $scope.chartOptions = chartOptions;
+          }
+        });
+        $scope.$watch('queryResult && queryResult.getData()', function (data) {
+          if (!data || $scope.queryResult.getData() == null) {
+            $scope.chartSeries.splice(0, $scope.chartSeries.length);
+          } else {
+            $scope.chartSeries.splice(0, $scope.chartSeries.length);
+
+            _.each($scope.queryResult.getChartData(), function (s) {
+              $scope.chartSeries.push(_.extend(s, {'stacking': 'normal'}));
+            });
+          }
+        });
+      }]
+    }
+  });
+
+  chartVisualization.directive('chartEditor', function () {
+    return {
+      restrict: 'E',
+      templateUrl: '/views/visualizations/chart_editor.html',
+      link: function (scope, element, attrs) {
+        scope.seriesTypes = {
+          'Line': 'line',
+          'Column': 'column',
+          'Area': 'area',
+          'Scatter': 'scatter',
+          'Pie': 'pie'
+        };
+
+        scope.stackingOptions = {
+          "None": "none",
+          "Normal": "normal",
+          "Percent": "percent"
+        };
+
+        scope.xAxisOptions = {
+          "Date/Time": "datetime",
+          "Linear": "linear",
+          "Category": "category"
+        };
+
+        scope.xAxisType = "datetime";
+        scope.stacking = "none";
+
+        var chartOptionsUnwatch = null;
+
+        scope.$watch('visualization', function (visualization) {
+          if (visualization && visualization.type == 'CHART') {
+            if (scope.visualization.options.series.stacking === null) {
+              scope.stacking = "none";
+            } else if (scope.visualization.options.series.stacking === undefined) {
+              scope.stacking = "normal";
+            } else {
+              scope.stacking = scope.visualization.options.series.stacking;
+            }
+
+            chartOptionsUnwatch = scope.$watch("stacking", function (stacking) {
+              if (stacking == "none") {
+                scope.visualization.options.series.stacking = null;
+              } else {
+                scope.visualization.options.series.stacking = stacking;
+              }
+            });
+
+            xAxisUnwatch = scope.$watch("xAxisType", function (xAxisType) {
+              scope.visualization.options.xAxis = scope.visualization.options.xAxis || {};
+              scope.visualization.options.xAxis.type = xAxisType;
+            });
+          } else {
+            if (chartOptionsUnwatch) {
+              chartOptionsUnwatch();
+              chartOptionsUnwatch = null;
+            }
+
+            if (xAxisUnwatch) {
+              xAxisUnwatch();
+              xAxisUnwatch = null;
+            }
+          }
+        });
+      }
+    }
+  });
+}());

rd_ui/app/scripts/visualizations/cohort.js

@@ -0,0 +1,60 @@
+(function () {
+    var cohortVisualization = angular.module('redash.visualization');
+
+    cohortVisualization.config(['VisualizationProvider', function(VisualizationProvider) {
+        VisualizationProvider.registerVisualization({
+            type: 'COHORT',
+            name: 'Cohort',
+            renderTemplate: '<cohort-renderer options="visualization.options" query-result="queryResult"></cohort-renderer>'
+        });
+    }]);
+
+    cohortVisualization.directive('cohortRenderer', function() {
+        return {
+            restrict: 'E',
+            scope: {
+                queryResult: '='
+            },
+            template: "",
+            replace: false,
+            link: function($scope, element, attrs) {
+                $scope.$watch('queryResult && queryResult.getData()', function (data) {
+                    if (!data) {
+                        return;
+                    }
+
+                    if ($scope.queryResult.getData() == null) {
+
+                    } else {
+                        var sortedData = _.sortBy($scope.queryResult.getData(), "date");
+                        var grouped = _.groupBy(sortedData, "date");
+                        var data = _.map(grouped, function(values, date) {
+                           var row = [values[0].total];
+                            _.each(values, function(value) { row.push(value.value); });
+                            return row;
+                        });
+
+                        var initialDate = moment(sortedData[0].date).toDate(),
+                            container = angular.element(element)[0];
+
+                        Cornelius.draw({
+                            initialDate: initialDate,
+                            container: container,
+                            cohort: data,
+                            title: null,
+                            timeInterval: 'daily',
+                            labels: {
+                                time: 'Activation Day',
+                                people: 'Users'
+                            },
+                            formatHeaderLabel: function (i) {
+                                return "Day " + (i - 1);
+                            }
+                        });
+                    }
+                });
+            }
+        }
+    });
+
+}());

rd_ui/app/scripts/visualizations/pivot.js

@@ -0,0 +1,29 @@
+var renderers = angular.module('redash.renderers', []);
+
+renderers.directive('pivotTableRenderer', function () {
+    return {
+        restrict: 'E',
+        scope: {
+            queryResult: '='
+        },
+        template: "",
+        replace: false,
+        link: function($scope, element, attrs) {
+            $scope.$watch('queryResult && queryResult.getData()', function (data) {
+                if (!data) {
+                    return;
+                }
+
+                if ($scope.queryResult.getData() == null) {
+                } else {
+                    // We need to give the pivot table its own copy of the data, because its change
+                    // it which interferes with other visualizations.
+                    var data = $.extend(true, [], $scope.queryResult.getData());
+                    $(element).pivotUI(data, {
+                         renderers: $.pivotUtilities.renderers
+                    }, true);
+                }
+            });
+        }
+    }
+});

rd_ui/app/scripts/visualizations/table.js

@@ -0,0 +1,90 @@
+(function () {
+    var tableVisualization = angular.module('redash.visualization');
+
+    tableVisualization.config(['VisualizationProvider', function(VisualizationProvider) {
+        VisualizationProvider.registerVisualization({
+            type: 'TABLE',
+            name: 'Table',
+            renderTemplate: '<grid-renderer options="visualization.options" query-result="queryResult"></grid-renderer>',
+            skipTypes: true
+        });
+    }]);
+
+    tableVisualization.directive('gridRenderer', function () {
+        return {
+            restrict: 'E',
+            scope: {
+                queryResult: '=',
+                itemsPerPage: '='
+            },
+            templateUrl: "/views/grid_renderer.html",
+            replace: false,
+            controller: ['$scope', function ($scope) {
+                $scope.gridColumns = [];
+                $scope.gridData = [];
+                $scope.gridConfig = {
+                    isPaginationEnabled: true,
+                    itemsByPage: $scope.itemsPerPage || 15,
+                    maxSize: 8
+                };
+
+                $scope.$watch('queryResult && queryResult.getData()', function (data) {
+                    if (!data) {
+                        return;
+                    }
+
+                    if ($scope.queryResult.getData() == null) {
+                        $scope.gridColumns = [];
+                        $scope.gridData = [];
+                        $scope.filters = [];
+                    } else {
+                        $scope.filters = $scope.queryResult.getFilters();
+
+                        var prepareGridData = function(data) {
+                            var gridData = _.map(data, function (row) {
+                                var newRow = {};
+                                _.each(row, function (val, key) {
+                                    newRow[$scope.queryResult.getColumnCleanName(key)] = val;
+                                })
+                                return newRow;
+                            });
+
+                            return gridData;
+                        };
+
+                        $scope.gridData = prepareGridData($scope.queryResult.getData());
+
+                        $scope.gridColumns = _.map($scope.queryResult.getColumnCleanNames(), function (col, i) {
+                            var columnDefinition = {
+                                'label': $scope.queryResult.getColumnFriendlyNames()[i],
+                                'map': col
+                            };
+
+                            var rawData = $scope.queryResult.getRawData();
+
+                            if (rawData.length > 0) {
+                                var exampleData = rawData[0][col];
+                                if (angular.isNumber(exampleData)) {
+                                    columnDefinition['formatFunction'] = 'number';
+                                    columnDefinition['formatParameter'] = 2;
+                                } else if (moment.isMoment(exampleData)) {
+                                    columnDefinition['formatFunction'] = function(value) {
+                                        // TODO: this is very hackish way to determine if we need
+                                        // to show the value as a time or date only. Better solution
+                                        // is to complete #70 and use the information it returns.
+                                        if (value._i.match(/^\d{4}-\d{2}-\d{2}T/)) {
+                                            return value.format("DD/MM/YY HH:mm");
+                                        }
+                                        return value.format("DD/MM/YY");
+                                    }
+                                }
+                            }
+
+                            return columnDefinition;
+                        });
+                    }
+                });
+            }]
+        }
+    })
+}());

rd_ui/app/styles/login.css

@@ -0,0 +1,37 @@
+.main {
+    max-width: 320px;
+    margin: 0 auto;
+}
+
+.login-or {
+    position: relative;
+    font-size: 18px;
+    color: #aaa;
+    margin-top: 10px;
+            margin-bottom: 10px;
+    padding-top: 10px;
+    padding-bottom: 10px;
+}
+
+.span-or {
+    display: block;
+    position: absolute;
+    left: 50%;
+    top: -2px;
+    margin-left: -25px;
+    background-color: #fff;
+    width: 50px;
+    text-align: center;
+}
+
+.hr-or {
+    background-color: #cdcdcd;
+    height: 1px;
+    margin-top: 0px !important;
+    margin-bottom: 0px !important;
+}
+
+/*h3 {*/
+    /*text-align: center;*/
+    /*line-height: 300%;*/
+/*}*/

rd_ui/app/styles/redash.css

@@ -25,7 +25,33 @@ a.navbar-brand {
     margin-top: 5px;
     margin-bottom: 5px;
 }
+.avatar img {
+    width: 40px;
+    height: 40px;
+}
 
+#logout {
+    color: white;
+    position: relative;
+    left: -9px;
+    bottom: -11px;
+}
+
+.details-toggle {
+    cursor: pointer;
+}
+.details-toggle::before {
+    content: '▸';
+    margin-right: 5px;
+}
+.details-toggle.open::before {
+    content: '▾';
+    margin-right: 5px;
+}
+
+.edit-in-place span {
+    white-space: pre-line;
+}
 .edit-in-place span.editable {
     cursor: pointer;
 }
@@ -60,10 +86,15 @@ a.navbar-brand {
     margin-bottom: 0px;
 }
 
-.panel-heading > a {
+.panel-heading > a,
+.panel-heading .query-link {
     color: inherit;
 }
 
+.panel-heading .query-link:hover {
+    text-decoration: none;
+}
+
 /* angular-growl */
 .growl {
     position: fixed;
@@ -213,4 +244,36 @@ to add those CSS styles here. */
     color: white;
     background-color: #FF8080;
     border-radius: 50%;
+}
+
+/* light version of bootstrap's form-control */
+.rd-form-control {
+    display: block;
+    padding: 6px 12px;
+    line-height: 1.428571429;
+    color: #555555;
+    vertical-align: middle;
+    background-color: #ffffff;
+    border: 1px solid #cccccc;
+    border-radius: 4px;
+    -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+    box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+    -webkit-transition: border-color ease-in-out 0.15s, box-shadow ease-in-out 0.15s;
+    transition: border-color ease-in-out 0.15s, box-shadow ease-in-out 0.15s;
+}
+.rd-form-control {
+     width: 100%;
+}
+visualization-renderer > div {
+    overflow: auto;
+}
+
+/*
+bootstrap's hidden-xs class adds display:block when not hidden
+use this class when you need to keep the original display value
+*/
+@media (max-width: 767px) {
+    .rd-hidden-xs {
+        display: none !important;
+    }
 }

rd_ui/app/views/dashboard.html

@@ -4,6 +4,8 @@
 <div class="container">
     <h2 id="dashboard_title">
         {{dashboard.name}}
+
+            <button type="button" class="btn btn-default btn-xs" ng-class="{active: refreshEnabled}" tooltip="Enable/Disable Auto Refresh" ng-click="triggerRefresh()"><span class="glyphicon glyphicon-refresh"></span></button>
             <span ng-show="dashboard.canEdit()">
                 <button type="button" class="btn btn-default btn-xs" data-toggle="modal" href="#edit_dashboard_dialog" tooltip="Edit Dashboard (Name/Layout)"><span
                         class="glyphicon glyphicon-cog"></span></button>
@@ -21,9 +23,10 @@
 
             <div class="panel panel-default">
                 <div class="panel-heading">
-                    <h3 class="panel-title" style="cursor: pointer;" ng-click="open(query)">
+                    <h3 class="panel-title">
                         <p>
-                            <span ng-bind="query.name"></span>
+                            <span ng-hide="currentUser.hasPermission('view_query')">{{query.name}}</span>
+                            <query-link query="query" visualization="widget.visualization" ng-show="currentUser.hasPermission('view_query')"></query-link>
                         </p>
                         <div class="text-muted" ng-bind="query.description"></div>
                     </h3>
@@ -37,7 +40,7 @@
                           tooltip-placement="bottom">Updated: <span am-time-ago="queryResult.getUpdatedAt()"></span></span>
 
                     <span class="pull-right">
-                        <a class="btn btn-default btn-xs" ng-href="/queries/{{query.id}}"><span class="glyphicon glyphicon-link"></span></a>
+                        <a class="btn btn-default btn-xs" ng-href="/queries/{{query.id}}#{{widget.visualization.id}}" ng-show="currentUser.hasPermission('view_query')"><span class="glyphicon glyphicon-link"></span></a>
                         <button type="button" class="btn btn-default btn-xs" ng-show="dashboard.canEdit()" ng-click="deleteWidget()" title="Remove Widget"><span class="glyphicon glyphicon-trash"></span></button>
                     </span>
 

rd_ui/app/views/edit_visualization.html

@@ -1,31 +0,0 @@
-<form role="form" name="visForm" ng-submit="submit()">
-  <div class="form-group">
-    <label class="control-label">Name</label>
-    <input type="text" class="form-control" ng-model="vis.name" placeholder="{{query.name}}">
-  </div>
-
-  <div class="form-group">
-    <label class="control-label">Description</label>
-    <textarea class="form-control" ng-model="vis.description" placeholder="{{query.description}}"></textarea>
-  </div>
-
-  <div class="form-group">
-    <label class="control-label">Visualization Type</label>
-    <select required ng-model="vis.type" ng-options="value as key for (key, value) in visTypes" class="form-control" ng-change="typeChanged()"></select>
-  </div>
-
-  <div class="form-group" ng-show="vis.type == visTypes.Chart">
-    <label class="control-label">Chart Type</label>
-    <select required ng-model="vis.options.series.type" ng-options="value as key for (key, value) in seriesTypes" class="form-control"></select>
-  </div>
-
-  <div class="form-group">
-    <a class="link" ng-click="toggleAdvancedMode()">Advanced</a>
-    <textarea json-text class="form-control" rows="5" ng-model="vis.options" ng-show="advancedMode"></textarea>
-  </div>
-
-  <div class="form-group">
-    <button type="submit" class="btn btn-primary">Save</button>
-  </div>
-
-</form>

rd_ui/app/views/grid_renderer.html

@@ -1,17 +1,4 @@
 <div>
-    <div class="btn-group pull-right" ng-repeat="filter in filters">
-        <button type="button" class="btn btn-default dropdown-toggle" data-toggle="dropdown">
-            {{filter.friendlyName}}: {{filter.current}}<span class="caret"></span>
-        </button>
-        <ul class="dropdown-menu" role="menu">
-            <li ng-repeat="value in filter.values">
-                <a href="#" ng-click="filter.current = value">{{value}}</a>
-            </li>
-            <li class="divider"></li>
-            <li><a href="#" ng-click="filter.current = 'All'">All</a></li>
-        </ul>
-    </div>
-
     <smart-table rows="gridData" columns="gridColumns"
                  config="gridConfig"
                  class="table table-condensed table-hover"></smart-table>

rd_ui/app/views/index.html

@@ -3,7 +3,7 @@
     <div class="list-group" ng-repeat="(name, dashboards) in allDashboards">
         <div class="list-group-item active">
             {{name}}
-            <button type="button" class="btn btn-sm btn-link" data-toggle="modal" href="#new_dashboard_dialog" tooltip="New Dashboard"><span class="glyphicon glyphicon-plus-sign"></span></button>
+            <button ng-show="currentUser.hasPermission('create_dashboard')" type="button" class="btn btn-sm btn-link" data-toggle="modal" href="#new_dashboard_dialog" tooltip="New Dashboard"><span class="glyphicon glyphicon-plus-sign"></span></button>
         </div>
         <div class="list-group-item" ng-repeat="dashboard in dashboards" >
             <button type="button" class="close delete-button" aria-hidden="true" ng-show="dashboard.canEdit()" ng-click="archiveDashboard(dashboard)" tooltip="Delete Dashboard">&times;</button>
@@ -11,7 +11,7 @@
         </div>
     </div>
 
-    <div ng-show="currentUser.is_admin">
+    <div ng-show="currentUser.hasPermission('admin')">
         <div class="list-group">
             <div class="list-group-item active">Admin</div>
             <a href="/admin/status" class="list-group-item">Status</a>

rd_ui/app/views/new_widget_form.html

@@ -12,7 +12,7 @@
                         <input class="form-control" placeholder="Query Id" ng-model="queryId">
                       </div>
                       <button type="submit" class="btn btn-primary" ng-disabled="!queryId">
-                        <span class="glyphicon glyphicon-refresh"></span> Load
+                        Load visualizations
                       </button>
                     </form>
                 </p>
@@ -23,20 +23,13 @@
                       <select ng-model="selectedVis" ng-options="vis as vis.name group by vis.type for vis in query.visualizations" class="form-control"></select>
                   </div>
 
-                  <div class="form-group">
-                    <a ng-click="toggleView('addNew')" class="link">&plus; Add New</a>
-                    <div class="well" ng-show="currentView=='addNew'">
-                      <edit-visulatization-form query="query"></edit-visulatization-form>
-                    </div>
-                  </div>
-
                   <div class="form-group">
                       <label for="">Widget Size</label>
                       <select class="form-control" ng-model="widgetSize" ng-options="c.value as c.name for c in widgetSizes"></select>
                   </div>
                 </div>
             </div>
-            <div class="modal-footer">
+            <div class="modal-footer" ng-if="selectedVis">
                 <button type="button" class="btn btn-default" ng-disabled="saveInProgress" data-dismiss="modal">Close</button>
                 <button type="button" class="btn btn-primary" ng-disabled="saveInProgress" ng-click="saveWidget()">Add to Dashboard</button>
             </div>

rd_ui/app/views/query.html

@@ -0,0 +1,182 @@
+
+<div class="container">
+
+    <alert-unsaved-changes ng-if="canEdit" is-dirty="isDirty"></alert-unsaved-changes>
+
+    <div class="row">
+        <div class="col-lg-12">
+            <div class="row">
+                <div class="col-lg-10">
+                    <h2>
+                        <edit-in-place editable="isQueryOwner" done="saveName" ignore-blanks='true' value="query.name"></edit-in-place>
+                    </h2>
+                    <p>
+                        <em>
+                            <edit-in-place editable="isQueryOwner" done="saveDescription" editor="textarea" placeholder="No description" ignore-blanks='false' value="query.description"></edit-in-place>
+                        </em>
+                    </p>
+                </div>
+
+                <div class="col-lg-2">
+                    <div class="rd-hidden-xs pull-right">
+                        <query-source-link></query-source-link>
+                    </div>
+                </div>
+            </div>
+
+            <div class="visible-xs">
+                <p>
+                    <span class="text-muted">Last update </span>
+                    <strong>
+                        <rd-time-ago value="queryResult.query_result.retrieved_at"></rd-time-ago>
+                    </strong>
+                    &nbsp;
+                    <span class="text-muted">Created By </span>
+                    <strong ng-hide="isQueryOwner">{{query.user.name}}</strong>
+                    <strong ng-show="isQueryOwner">You</strong>
+                    &nbsp;
+                    <span class="text-muted">Runtime </span>
+                    <strong ng-show="!queryExecuting">{{queryResult.getRuntime() | durationHumanize}}</strong>
+                    <span ng-show="queryExecuting">Running&hellip;</span>
+                    &nbsp;
+                    <span class="text-muted">Rows </span>
+                    <strong>{{queryResult.getData().length}}</strong>
+                </p>
+                <p>
+                    <query-source-link></query-source-link>
+                </p>
+            </div>
+        </div>
+    </div>
+
+    <hr>
+
+    <div class="row">
+        <div class="col-lg-12">
+            <div ng-show="sourceMode">
+                <p>
+                    <button type="button" class="btn btn-primary btn-xs" ng-disabled="queryExecuting" ng-click="executeQuery()">
+                        <span class="glyphicon glyphicon-play"></span> Execute
+                    </button>
+                    <query-formatter></query-formatter>
+                    <span class="pull-right">
+                        <button class="btn btn-xs btn-default rd-hidden-xs" ng-click="duplicateQuery()">
+                            <span class="glyphicon glyphicon-share-alt"></span> Fork
+                        </button>
+
+                        <button class="btn btn-success btn-xs" ng-show="canEdit" ng-click="saveQuery()">
+                            <span class="glyphicon glyphicon-floppy-disk"> </span> Save<span ng-show="isDirty">&#42;</span>
+                        </button>
+                    </span>
+                </p>
+            </div>
+
+            <!-- code editor -->
+            <div ng-show="sourceMode">
+                <p>
+                    <query-editor query="query" lock="queryExecuting"></query-editor>
+                </p>
+                <hr>
+            </div>
+        </div>
+
+    </div>
+
+    <div class="row">
+        <div class="col-lg-3 rd-hidden-xs">
+            <p>
+                <span class="glyphicon glyphicon-time"></span>
+                <span class="text-muted">Last update </span>
+                <strong>
+                    <rd-time-ago value="queryResult.query_result.retrieved_at"></rd-time-ago>
+                </strong>
+            </p>
+            <p>
+                <span class="glyphicon glyphicon-user"></span>
+                <span class="text-muted">Created By </span>
+                <strong ng-hide="isQueryOwner">{{query.user.name}}</strong>
+                <strong ng-show="isQueryOwner">You</strong>
+            </p>
+            <p>
+                <span class="glyphicon glyphicon-play"></span>
+                <span class="text-muted">Runtime </span>
+                <strong ng-show="!queryExecuting">{{queryResult.getRuntime() | durationHumanize}}</strong>
+                <span ng-show="queryExecuting">Running&hellip;</span>
+            </p>
+            <p>
+                <span class="glyphicon glyphicon-align-justify"></span>
+                <span class="text-muted">Rows </span><strong>{{queryResult.getData().length}}</strong>
+            </p>
+            <p>
+                <span class="glyphicon glyphicon-refresh"></span>
+                <span class="text-muted">Refresh Interval</span>
+                <query-refresh-select></query-refresh-select>
+            </p>
+
+            <p>
+                <span class="glyphicon glyphicon-hdd"></span>
+                <span class="text-muted">Data Source</span>
+                <select ng-disabled="!isQueryOwner" ng-model="query.data_source_id" ng-change="updateDataSource()" ng-options="ds.id as ds.name for ds in dataSources"></select>
+            </p>
+
+            <hr>
+
+            <p>
+                <a class="btn btn-primary btn-sm" ng-disabled="queryExecuting || !queryResult.getData()" ng-href="{{dataUri}}" download="{{dataFilename}}" target="_self">
+                    <span class="glyphicon glyphicon-cloud-download"></span>
+                    <span class="rd-hidden-xs">Download Dataset</span>
+                </a>
+            </p>
+        </div>
+
+        <div class="col-lg-9">
+            <!-- alerts -->
+            <div class="alert alert-info" ng-show="queryResult.getStatus() == 'processing'">
+                Executing query&hellip; <rd-timer timestamp="queryResult.getUpdatedAt()"></rd-timer>
+                <button type="button" class="btn btn-warning btn-xs pull-right" ng-disabled="cancelling" ng-click="cancelExecution()">Cancel</button>
+            </div>
+            <div class="alert alert-info" ng-show="queryResult.getStatus() == 'waiting'">
+                Query in queue&hellip; <rd-timer timestamp="queryResult.getUpdatedAt()"></rd-timer>
+                <button type="button" class="btn btn-warning btn-xs pull-right" ng-disabled="cancelling" ng-click="cancelExecution()">Cancel</button>
+            </div>
+            <div class="alert alert-danger" ng-show="queryResult.getError()">Error running query: <strong>{{queryResult.getError()}}</strong></div>
+
+            <!-- tabs and data -->
+            <div ng-show="queryResult.getStatus() == 'done'">
+                <div class="row">
+                    <div class="col-lg-12">
+                        <ul class="nav nav-tabs">
+                            <rd-tab tab-id="table" name="Table"></rd-tab>
+                            <rd-tab tab-id="pivot" name="Pivot Table"></rd-tab>
+                            <rd-tab tab-id="{{vis.id}}" name="{{vis.name}}" ng-if="vis.type!='TABLE'" ng-repeat="vis in query.visualizations">
+                                <span class="remove" ng-click="deleteVisualization($event, vis)" ng-show="canEdit"> &times;</span>
+                            </rd-tab>
+                            <rd-tab tab-id="add" name="&plus; New" removeable="true" ng-show="canEdit"></rd-tab>
+                        </ul>
+                    </div>
+                </div>
+                <div class="row">
+                    <div class="col-lg-12">
+                        <div ng-show="selectedTab == 'table'" >
+                            <filters></filters>
+                            <grid-renderer query-result="queryResult" items-per-page="50"></grid-renderer>
+                        </div>
+
+                        <pivot-table-renderer ng-show="selectedTab == 'pivot'" query-result="queryResult"></pivot-table-renderer>
+
+                        <div ng-show="selectedTab == vis.id" ng-repeat="vis in query.visualizations">
+                            <visualization-renderer visualization="vis" query-result="queryResult"></visualization-renderer>
+
+                            <edit-visulatization-form visualization="vis" query="query" query-result="queryResult" ng-show="canEdit"></edit-visulatization-form>
+                        </div>
+
+                        <div ng-show="selectedTab == 'add'">
+                            <visualization-renderer visualization="newVisualization" query-result="queryResult"></visualization-renderer>
+                            <edit-visulatization-form visualization="newVisualization" query="query" ng-show="canEdit" on-new-success="setVisualizationTab"></edit-visulatization-form>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+    </div>
+</div>

rd_ui/app/views/queryfiddle.html

@@ -1,97 +0,0 @@
-<div class="container">
-    <div class="row">
-        <div class="panel panel-default">
-            <div class="panel-heading">
-                <h3 class="panel-title">
-                    <p>
-                     <edit-in-place editable="currentUser.canEdit(query)" ignore-blanks='true' value="query.name"></edit-in-place>
-                 </p>
-                </h3>
-                <p>
-                    <edit-in-place editable="currentUser.canEdit(query)" editor="textarea" placeholder="No description" ignore-blanks='false' value="query.description" class="text-muted"></edit-in-place>
-                </p>
-            </div>
-            <div class="panel-body">
-                <textarea ui-codemirror="editorOptions" ng-model="query.query"></textarea>
-
-                <div>
-                    <a class="btn btn-default" ng-disabled="queryExecuting || !queryResult.getData()" ng-href="{{dataUri}}" download="{{dataFilename}}" target="_self">
-                        <span class="glyphicon glyphicon-floppy-disk"></span> Download Data Set
-                    </a>
-                    <button type="button" class="btn btn-default center-x" ng-click="formatQuery()"><span class="glyphicon glyphicon-ok"></span> Format SQL</button>
-
-                    <div class="btn-group pull-right">
-                        <button type="button" class="btn btn-default" ng-click="duplicateQuery()">Duplicate</button>
-                        <button type="button" class="btn btn-default" ng-disabled="!currentUser.canEdit(query)" ng-click="saveQuery()">Save
-                            <span ng-show="dirty">&#42;</span>
-                        </button>
-                        <button type="button" class="btn btn-primary" ng-disabled="queryExecuting" ng-click="executeQuery()">Execute</button>
-                    </div>
-                </div>
-
-            </div>
-            <div class="panel-footer">
-                <span ng-show="queryResult.getRuntime()>=0">Query runtime: {{queryResult.getRuntime() | durationHumanize}} | </span>
-                <span ng-show="queryResult.query_result.retrieved_at">Last update time: <span am-time-ago="queryResult.query_result.retrieved_at"></span> | </span>
-                <span ng-show="queryResult.getStatus() == 'done'">Rows: {{queryResult.getData().length}} | </span>
-                Created by: {{query.user}}
-                <div class="pull-right">Refresh query: <select ng-model="query.ttl" ng-options="c.value as c.name for c in refreshOptions"></select><br></div>
-            </div>
-        </div>
-
-        <div class="alert alert-info" ng-show="queryResult.getStatus() == 'processing'">
-            Executing query... <rd-timer timestamp="queryResult.getUpdatedAt()"></rd-timer>
-            <button type="button" class="btn btn-warning btn-xs pull-right" ng-disabled="cancelling" ng-click="cancelExecution()">Cancel</button>
-        </div>
-        <div class="alert alert-info" ng-show="queryResult.getStatus() == 'waiting'">
-            Query in queue... <rd-timer timestamp="queryResult.getUpdatedAt()"></rd-timer>
-            <button type="button" class="btn btn-warning btn-xs pull-right" ng-disabled="cancelling" ng-click="cancelExecution()">Cancel</button>
-        </div>
-        <div class="alert alert-danger" ng-show="queryResult.getError()">Error running query: <strong>{{queryResult.getError()}}</strong></div>
-    </div>
-
-    <div class="row" ng-show="queryResult.getStatus() == 'done'">
-        <ul class="nav nav-tabs">
-            <rd-tab id="table" name="Table"></rd-tab>
-            <rd-tab id="pivot" name="Pivot Table"></rd-tab>
-            <rd-tab id="{{vis.id}}" name="{{vis.name}}" ng-repeat="vis in query.visualizations">
-                <span class="remove" ng-click="deleteVisualization($event, vis)"> &times;</span>
-            </rd-tab>
-            <rd-tab id="add" name="&plus;New" removeable="true"></rd-tab>
-        </ul>
-
-        <div class="col-lg-12" ng-show="selectedTab == 'table'">
-            <grid-renderer query-result="queryResult" items-per-page="50"></grid-renderer>
-        </div>
-
-        <div class="col-lg-12" ng-show="selectedTab == 'pivot'">
-            <pivot-table-renderer query-result="queryResult"></pivot-table-renderer>
-        </div>
-
-        <div class="col-lg-12" ng-show="selectedTab == vis.id" ng-repeat="vis in query.visualizations">
-            <div class="row">
-                <p>
-                    <div class="col-lg-6">
-                        <edit-visulatization-form vis="vis" query="query"></edit-visulatization-form>
-                    </div>
-                    <div class="col-lg-6">
-                        <visualization-renderer visualization="vis" query-result="queryResult"></visualization-renderer>
-                    </div>
-                </p>
-            </div>
-        </div>
-
-        <div class="col-lg-12" ng-show="selectedTab == 'add'">
-            <div class="row">
-                <p>
-                    <div class="col-lg-6">
-                        <edit-visulatization-form vis="newVisualization" query="query"></edit-visulatization-form>
-                    </div>
-                    <div class="col-lg-6">
-                        <visualization-renderer visualization="newVisualization" query-result="queryResult"></visualization-renderer>
-                    </div>
-                </p>
-            </div>
-        </div>
-    </div>
-</div>

rd_ui/app/views/visualizations/chart_editor.html

@@ -0,0 +1,14 @@
+<div>
+  <div class="form-group">
+      <label class="control-label">Chart Type</label>
+      <select required ng-model="visualization.options.series.type" ng-options="value as key for (key, value) in seriesTypes" class="form-control"></select>
+  </div>
+
+  <div class="form-group">
+      <label class="control-label">Stacking</label>
+      <select required ng-model="stacking" ng-options="value as key for (key, value) in stackingOptions" class="form-control"></select>
+
+      <label class="control-label">X Axis Type</label>
+      <select required ng-model="xAxisType" ng-options="value as key for (key, value) in xAxisOptions" class="form-control"></select>
+  </div>
+</div>

rd_ui/app/views/visualizations/cohort_editor.html

@@ -0,0 +1,15 @@
+<div class="form-group">
+    <label class="control-label">Time Label</label>
+    <input type="text" class="form-control" ng-model="cohortOptions.timeLabel">
+    <label class="control-label">People Label</label>
+    <input type="text" class="form-control" ng-model="cohortOptions.peopleLabel">
+
+    <label class="control-label">Bucket Column</label>
+    <select ng-model="bucket_column" ng-options="value as key for (key, value) in columns" class="form-control"></select>
+    <label class="control-label">Bucket Total Value Column</label>
+    <select ng-model="total_column" ng-options="value as key for (key, value) in columns" class="form-control"></select>
+    <label class="control-label">Day Number Column</label>
+    <select ng-model="value_column" ng-options="value as key for (key, value) in columns" class="form-control"></select>
+    <label class="control-label">Day Value Column</label>
+    <select ng-model="day_column" ng-options="value as key for (key, value) in columns" class="form-control"></select>
+</div>

rd_ui/app/views/visualizations/edit_visualization.html

@@ -0,0 +1,27 @@
+<div>
+  <span ng-click="visEdit=!visEdit" class="details-toggle" ng-class="{open: visEdit}">Edit</span>
+
+  <form ng-if="visEdit" role="form" name="visForm" ng-submit="submit()">
+    <div class="form-group">
+      <label class="control-label">Name</label>
+      <input name="name" type="text" class="form-control" ng-model="visualization.name" placeholder="{{visualization.type | capitalize}}">
+    </div>
+
+    <div class="form-group">
+      <label class="control-label">Visualization Type</label>
+      <select required ng-model="visualization.type" ng-options="value as key for (key, value) in visTypes" class="form-control" ng-change="typeChanged()"></select>
+    </div>
+
+    <visualization-options-editor></visualization-options-editor>
+
+    <div class="form-group" ng-if="editRawOptions">
+      <label class="control-label">Advanced</label>
+      <textarea json-text ng-model="visualization.options" class="form-control" rows="10"></textarea>
+    </div>
+
+    <div class="form-group">
+      <button type="submit" class="btn btn-primary">Save</button>
+    </div>
+
+  </form>
+</div>

rd_ui/app/views/visualizations/filters.html

@@ -0,0 +1,8 @@
+<div class="well well-sm" ng-show="filters">
+    <div ng-repeat="filter in filters">
+        {{filter.friendlyName}}:
+        <select ui-select2='select2Options' ng-model="filter.current" ng-multiple="{{filter.multiple}}">
+            <option ng-repeat="value in filter.values" value="{{value}}">{{value}}</option>
+        </select>
+    </div>
+</div>

rd_ui/bower.json

@@ -13,20 +13,20 @@
     "angular-ui-codemirror": "0.0.5",
     "highcharts": "3.0.1",
     "underscore": "1.5.1",
-    "angular-resource": "1.0.7",
+    "angular-resource": "1.2.15",
     "angular-growl": "0.3.1",
     "angular-route": "1.2.7",
-    "pivottable": "git@github.com:arikfr/pivottable.git#master",
-    "cornelius": "git@github.com:restorando/cornelius.git",
+    "pivottable": "https://github.com/arikfr/pivottable.git",
+    "cornelius": "https://github.com/restorando/cornelius.git",
     "gridster": "0.2.0",
-    "mousetrap": "~1.4.6"
+    "mousetrap": "~1.4.6",
+    "angular-ui-select2": "~0.0.5"
   },
   "devDependencies": {
     "angular-mocks": "~1.0.7",
     "angular-scenario": "~1.0.7"
   },
   "resolutions": {
-    "angular": "~1.2.7",
-    "jquery": "~1.9.1"
+    "angular": "1.2.7"
   }
 }

redash/__init__.py

@@ -0,0 +1,61 @@
+import json
+import urlparse
+import logging
+from flask import Flask, make_response
+from flask.ext.restful import Api
+from flask_peewee.db import Database
+
+import redis
+from statsd import StatsClient
+import events
+from redash import settings, utils
+
+__version__ = '0.3.6'
+
+
+def setup_logging():
+    handler = logging.StreamHandler()
+    formatter = logging.Formatter('[%(asctime)s][PID:%(process)d][%(levelname)s][%(name)s] %(message)s')
+    handler.setFormatter(formatter)
+    logging.getLogger().addHandler(handler)
+    logging.getLogger().setLevel(settings.LOG_LEVEL)
+
+    events.setup_logging(settings.EVENTS_LOG_PATH, settings.EVENTS_CONSOLE_OUTPUT)
+
+setup_logging()
+
+app = Flask(__name__,
+            template_folder=settings.STATIC_ASSETS_PATH,
+            static_folder=settings.STATIC_ASSETS_PATH,
+            static_path='/static')
+
+api = Api(app)
+
+# configure our database
+settings.DATABASE_CONFIG.update({'threadlocals': True})
+app.config['DATABASE'] = settings.DATABASE_CONFIG
+db = Database(app)
+
+from redash.authentication import setup_authentication
+auth = setup_authentication(app)
+
+@api.representation('application/json')
+def json_representation(data, code, headers=None):
+    resp = make_response(json.dumps(data, cls=utils.JSONEncoder), code)
+    resp.headers.extend(headers or {})
+    return resp
+
+
+redis_url = urlparse.urlparse(settings.REDIS_URL)
+if redis_url.path:
+    redis_db = redis_url.path[1]
+else:
+    redis_db = 0
+
+redis_connection = redis.StrictRedis(host=redis_url.hostname, port=redis_url.port, db=redis_db, password=redis_url.password)
+statsd_client = StatsClient(host=settings.STATSD_HOST, port=settings.STATSD_PORT, prefix=settings.STATSD_PREFIX)
+
+from redash import data
+data_manager = data.Manager(redis_connection, statsd_client)
+
+from redash import controllers

redash/authentication.py

@@ -0,0 +1,107 @@
+import functools
+import hashlib
+import hmac
+import time
+import logging
+
+from flask import request, make_response, redirect, url_for
+from flask.ext.googleauth import GoogleAuth, login
+from flask.ext.login import LoginManager, login_user, current_user
+from werkzeug.contrib.fixers import ProxyFix
+
+from models import AnonymousUser
+from redash import models, settings
+
+
+login_manager = LoginManager()
+logger = logging.getLogger('authentication')
+
+
+def sign(key, path, expires):
+    if not key:
+        return None
+
+    h = hmac.new(str(key), msg=path, digestmod=hashlib.sha1)
+    h.update(str(expires))
+
+    return h.hexdigest()
+
+
+class HMACAuthentication(object):
+    @staticmethod
+    def api_key_authentication():
+        signature = request.args.get('signature')
+        expires = float(request.args.get('expires') or 0)
+        query_id = request.view_args.get('query_id', None)
+
+        # TODO: 3600 should be a setting
+        if signature and query_id and time.time() < expires <= time.time() + 3600:
+            query = models.Query.get(models.Query.id == query_id)
+            calculated_signature = sign(query.api_key, request.path, expires)
+
+            if query.api_key and signature == calculated_signature:
+                login_user(models.ApiUser(query.api_key), remember=False)
+                return True
+
+        return False
+
+    def required(self, fn):
+        @functools.wraps(fn)
+        def decorated(*args, **kwargs):
+            if current_user.is_authenticated():
+                return fn(*args, **kwargs)
+
+            if self.api_key_authentication():
+                return fn(*args, **kwargs)
+
+            return make_response(redirect(url_for("login", next=request.url)))
+
+        return decorated
+
+
+def validate_email(email):
+    if not settings.GOOGLE_APPS_DOMAIN:
+        return True
+
+    return email in settings.ALLOWED_EXTERNAL_USERS or email.endswith("@%s" % settings.GOOGLE_APPS_DOMAIN)
+
+
+def create_and_login_user(app, user):
+    if not validate_email(user.email):
+        return
+
+    try:
+        user_object = models.User.get(models.User.email == user.email)
+        if user_object.name != user.name:
+            logger.debug("Updating user name (%r -> %r)", user_object.name, user.name)
+            user_object.name = user.name
+            user_object.save()
+    except models.User.DoesNotExist:
+        logger.debug("Creating user object (%r)", user.name)
+        user_object = models.User.create(name=user.name, email=user.email,
+                                  is_admin=(user.email in settings.ADMINS))
+
+    login_user(user_object, remember=True)
+
+login.connect(create_and_login_user)
+
+
+@login_manager.user_loader
+def load_user(user_id):
+    return models.User.select().where(models.User.id == user_id).first()
+
+
+def setup_authentication(app):
+    if settings.GOOGLE_OPENID_ENABLED:
+        openid_auth = GoogleAuth(app, url_prefix="/google_auth")
+        # If we don't have a list of external users, we can use Google's federated login, which limits
+        # the domain with which you can sign in.
+        if not settings.ALLOWED_EXTERNAL_USERS and settings.GOOGLE_APPS_DOMAIN:
+            openid_auth._OPENID_ENDPOINT = "https://www.google.com/a/%s/o8/ud?be=o8" % settings.GOOGLE_APPS_DOMAIN
+
+    login_manager.init_app(app)
+    login_manager.anonymous_user = AnonymousUser
+    app.wsgi_app = ProxyFix(app.wsgi_app)
+    app.secret_key = settings.COOKIE_SECRET
+
+    return HMACAuthentication()

redash/controllers.py

@@ -0,0 +1,425 @@
+"""
+Flask-restful based API implementation for re:dash.
+
+Currently the Flask server is used to serve the static assets (and the Angular.js app),
+but this is only due to configuration issues and temporary.
+"""
+import csv
+import hashlib
+import json
+import numbers
+import cStringIO
+import datetime
+
+from flask import render_template, send_from_directory, make_response, request, jsonify, redirect, \
+    session, url_for
+from flask.ext.restful import Resource, abort
+from flask_login import current_user, login_user, logout_user
+
+import sqlparse
+import events
+from permissions import require_permission
+from redash import settings, utils
+from redash import data
+
+from redash import app, auth, api, redis_connection, data_manager
+from redash import models
+
+
+@app.route('/ping', methods=['GET'])
+def ping():
+    return 'PONG.'
+
+
+@app.route('/admin/<anything>')
+@app.route('/dashboard/<anything>')
+@app.route('/queries')
+@app.route('/queries/<query_id>')
+@app.route('/queries/<query_id>/<anything>')
+@app.route('/')
+@auth.required
+def index(**kwargs):
+    email_md5 = hashlib.md5(current_user.email.lower()).hexdigest()
+    gravatar_url = "https://www.gravatar.com/avatar/%s?s=40" % email_md5
+
+    user = {
+        'gravatar_url': gravatar_url,
+        'is_admin': current_user.is_admin,
+        'id': current_user.id,
+        'name': current_user.name,
+        'email': current_user.email,
+        'permissions': current_user.permissions
+    }
+
+    return render_template("index.html", user=json.dumps(user), name=settings.NAME,
+                           analytics=settings.ANALYTICS)
+
+
+@app.route('/login', methods=['GET', 'POST'])
+def login():
+    if current_user.is_authenticated():
+        return redirect(request.args.get('next') or '/')
+
+    if not settings.PASSWORD_LOGIN_ENABLED:
+        blueprint = app.extensions['googleauth'].blueprint
+        return redirect(url_for("%s.login" % blueprint.name, next=request.args.get('next')))
+
+    if request.method == 'POST':
+        user = models.User.select().where(models.User.email == request.form['username']).first()
+        if user and user.verify_password(request.form['password']):
+            remember = ('remember' in request.form)
+            login_user(user, remember=remember)
+            return redirect(request.args.get('next') or '/')
+
+    return render_template("login.html",
+                           name=settings.NAME,
+                           analytics=settings.ANALYTICS,
+                           next=request.args.get('next'),
+                           username=request.form.get('username', ''),
+                           show_google_openid=settings.GOOGLE_OPENID_ENABLED)
+
+
+@app.route('/logout')
+def logout():
+    logout_user()
+    session.pop('openid', None)
+
+    return redirect('/login')
+
+@app.route('/status.json')
+@auth.required
+@require_permission('admin')
+def status_api():
+    status = {}
+    info = redis_connection.info()
+    status['redis_used_memory'] = info['used_memory_human']
+
+    status['queries_count'] = models.Query.select().count()
+    status['query_results_count'] = models.QueryResult.select().count()
+    status['dashboards_count'] = models.Dashboard.select().count()
+    status['widgets_count'] = models.Widget.select().count()
+
+    status['workers'] = [redis_connection.hgetall(w)
+                         for w in redis_connection.smembers('workers')]
+
+    manager_status = redis_connection.hgetall('manager:status')
+    status['manager'] = manager_status
+    status['manager']['queue_size'] = redis_connection.zcard('jobs')
+
+    return jsonify(status)
+
+
+@app.route('/api/queries/format', methods=['POST'])
+@auth.required
+def format_sql_query():
+    arguments = request.get_json(force=True)
+    query = arguments.get("query", "")
+
+    return sqlparse.format(query, reindent=True, keyword_case='upper')
+
+
+class BaseResource(Resource):
+    decorators = [auth.required]
+
+    def __init__(self, *args, **kwargs):
+        super(BaseResource, self).__init__(*args, **kwargs)
+        self._user = None
+
+    @property
+    def current_user(self):
+        return current_user._get_current_object()
+
+
+class EventAPI(BaseResource):
+    def post(self):
+        events_list = request.get_json(force=True)
+        for event in events_list:
+            events.record_event(event)
+
+
+api.add_resource(EventAPI, '/api/events', endpoint='events')
+
+
+class DataSourceListAPI(BaseResource):
+    def get(self):
+        data_sources = [ds.to_dict() for ds in models.DataSource.select()]
+        return data_sources
+
+api.add_resource(DataSourceListAPI, '/api/data_sources', endpoint='data_sources')
+
+
+class DashboardListAPI(BaseResource):
+    def get(self):
+        dashboards = [d.to_dict() for d in
+                      models.Dashboard.select().where(models.Dashboard.is_archived==False)]
+
+        return dashboards
+
+    @require_permission('create_dashboard')
+    def post(self):
+        dashboard_properties = request.get_json(force=True)
+        dashboard = models.Dashboard(name=dashboard_properties['name'],
+                                     user=self.current_user,
+                                     layout='[]')
+        dashboard.save()
+        return dashboard.to_dict()
+
+
+class DashboardAPI(BaseResource):
+    def get(self, dashboard_slug=None):
+        try:
+            dashboard = models.Dashboard.get_by_slug(dashboard_slug)
+        except models.Dashboard.DoesNotExist:
+            abort(404)
+
+        return dashboard.to_dict(with_widgets=True)
+
+    @require_permission('edit_dashboard')
+    def post(self, dashboard_slug):
+        dashboard_properties = request.get_json(force=True)
+        # TODO: either convert all requests to use slugs or ids
+        dashboard = models.Dashboard.get_by_id(dashboard_slug)
+        dashboard.layout = dashboard_properties['layout']
+        dashboard.name = dashboard_properties['name']
+        dashboard.save()
+
+        return dashboard.to_dict(with_widgets=True)
+
+    @require_permission('edit_dashboard')
+    def delete(self, dashboard_slug):
+        dashboard = models.Dashboard.get_by_slug(dashboard_slug)
+        dashboard.is_archived = True
+        dashboard.save()
+
+api.add_resource(DashboardListAPI, '/api/dashboards', endpoint='dashboards')
+api.add_resource(DashboardAPI, '/api/dashboards/<dashboard_slug>', endpoint='dashboard')
+
+
+class WidgetListAPI(BaseResource):
+    @require_permission('edit_dashboard')
+    def post(self):
+        widget_properties = request.get_json(force=True)
+        widget_properties['options'] = json.dumps(widget_properties['options'])
+        widget_properties.pop('id', None)
+        widget_properties['dashboard'] = widget_properties.pop('dashboard_id')
+        widget_properties['visualization'] = widget_properties.pop('visualization_id')
+        widget = models.Widget(**widget_properties)
+        widget.save()
+
+        layout = json.loads(widget.dashboard.layout)
+        new_row = True
+
+        if len(layout) == 0 or widget.width == 2:
+            layout.append([widget.id])
+        elif len(layout[-1]) == 1:
+            neighbour_widget = models.Widget.get(models.Widget.id == layout[-1][0])
+            if neighbour_widget.width == 1:
+                layout[-1].append(widget.id)
+                new_row = False
+            else:
+                layout.append([widget.id])
+        else:
+            layout.append([widget.id])
+
+        widget.dashboard.layout = json.dumps(layout)
+        widget.dashboard.save()
+
+        return {'widget': widget.to_dict(), 'layout': layout, 'new_row': new_row}
+
+
+class WidgetAPI(BaseResource):
+    @require_permission('edit_dashboard')
+    def delete(self, widget_id):
+        widget = models.Widget.get(models.Widget.id == widget_id)
+        # TODO: reposition existing ones
+        layout = json.loads(widget.dashboard.layout)
+        layout = map(lambda row: filter(lambda w: w != widget_id, row), layout)
+        layout = filter(lambda row: len(row) > 0, layout)
+        widget.dashboard.layout = json.dumps(layout)
+        widget.dashboard.save()
+
+        widget.delete_instance()
+
+api.add_resource(WidgetListAPI, '/api/widgets', endpoint='widgets')
+api.add_resource(WidgetAPI, '/api/widgets/<int:widget_id>', endpoint='widget')
+
+
+class QueryListAPI(BaseResource):
+    @require_permission('create_query')
+    def post(self):
+        query_def = request.get_json(force=True)
+        for field in ['id', 'created_at', 'api_key', 'visualizations', 'latest_query_data']:
+            query_def.pop(field, None)
+
+        query_def['user'] = self.current_user
+        query_def['data_source'] = query_def.pop('data_source_id')
+        query = models.Query(**query_def)
+        query.save()
+
+        query.create_default_visualizations()
+
+        return query.to_dict(with_result=False)
+
+    @require_permission('view_query')
+    def get(self):
+        return [q.to_dict(with_result=False, with_stats=True) for q in models.Query.all_queries()]
+
+
+class QueryAPI(BaseResource):
+    @require_permission('edit_query')
+    def post(self, query_id):
+        query_def = request.get_json(force=True)
+        for field in ['id', 'created_at', 'api_key', 'visualizations', 'latest_query_data', 'user']:
+            query_def.pop(field, None)
+
+        if 'latest_query_data_id' in query_def:
+            query_def['latest_query_data'] = query_def.pop('latest_query_data_id')
+
+        if 'data_source_id' in query_def:
+            query_def['data_source'] = query_def.pop('data_source_id')
+
+        models.Query.update_instance(query_id, **query_def)
+
+        query = models.Query.get_by_id(query_id)
+
+        return query.to_dict(with_result=False, with_visualizations=True)
+
+    @require_permission('view_query')
+    def get(self, query_id):
+        q = models.Query.get(models.Query.id == query_id)
+        if q:
+            return q.to_dict(with_visualizations=True)
+        else:
+            abort(404, message="Query not found.")
+
+api.add_resource(QueryListAPI, '/api/queries', endpoint='queries')
+api.add_resource(QueryAPI, '/api/queries/<query_id>', endpoint='query')
+
+
+class VisualizationListAPI(BaseResource):
+    @require_permission('edit_query')
+    def post(self):
+        kwargs = request.get_json(force=True)
+        kwargs['options'] = json.dumps(kwargs['options'])
+        kwargs['query'] = kwargs.pop('query_id')
+
+        vis = models.Visualization(**kwargs)
+        vis.save()
+
+        return vis.to_dict(with_query=False)
+
+
+class VisualizationAPI(BaseResource):
+    @require_permission('edit_query')
+    def post(self, visualization_id):
+        kwargs = request.get_json(force=True)
+        if 'options' in kwargs:
+            kwargs['options'] = json.dumps(kwargs['options'])
+        kwargs.pop('id', None)
+
+        update = models.Visualization.update(**kwargs).where(models.Visualization.id == visualization_id)
+        update.execute()
+
+        vis = models.Visualization.get_by_id(visualization_id)
+
+        return vis.to_dict(with_query=False)
+
+    @require_permission('edit_query')
+    def delete(self, visualization_id):
+        vis = models.Visualization.get(models.Visualization.id == visualization_id)
+        vis.delete_instance()
+
+api.add_resource(VisualizationListAPI, '/api/visualizations', endpoint='visualizations')
+api.add_resource(VisualizationAPI, '/api/visualizations/<visualization_id>', endpoint='visualization')
+
+
+class QueryResultListAPI(BaseResource):
+    @require_permission('execute_query')
+    def post(self):
+        params = request.json
+
+        models.ActivityLog(
+            user=self.current_user,
+            type=models.ActivityLog.QUERY_EXECUTION,
+            activity=params['query']
+        ).save()
+
+        if params['ttl'] == 0:
+            query_result = None
+        else:
+            query_result = models.QueryResult.get_latest(params['data_source_id'], params['query'], int(params['ttl']))
+
+        if query_result:
+            return {'query_result': query_result.to_dict()}
+        else:
+            data_source = models.DataSource.get_by_id(params['data_source_id'])
+            job = data_manager.add_job(params['query'], data.Job.HIGH_PRIORITY, data_source)
+            return {'job': job.to_dict()}
+
+
+class QueryResultAPI(BaseResource):
+    @require_permission('view_query')
+    def get(self, query_result_id):
+        query_result = models.QueryResult.get_by_id(query_result_id)
+        if query_result:
+            return {'query_result': query_result.to_dict()}
+        else:
+            abort(404)
+
+
+class CsvQueryResultsAPI(BaseResource):
+    @require_permission('view_query')
+    def get(self, query_id, query_result_id=None):
+        if not query_result_id:
+            query = models.Query.get(models.Query.id == query_id)
+            if query:
+                query_result_id = query._data['latest_query_data']
+
+        query_result = query_result_id and models.QueryResult.get_by_id(query_result_id)
+        if query_result:
+            s = cStringIO.StringIO()
+
+            query_data = json.loads(query_result.data)
+            writer = csv.DictWriter(s, fieldnames=[col['name'] for col in query_data['columns']])
+            writer.writer = utils.UnicodeWriter(s)
+            writer.writeheader()
+            for row in query_data['rows']:
+                for k, v in row.iteritems():
+                    if isinstance(v, numbers.Number) and (v > 1000 * 1000 * 1000 * 100):
+                        row[k] = datetime.datetime.fromtimestamp(v/1000.0)
+
+                writer.writerow(row)
+
+            return make_response(s.getvalue(), 200, {'Content-Type': "text/csv; charset=UTF-8"})
+        else:
+            abort(404)
+
+api.add_resource(CsvQueryResultsAPI, '/api/queries/<query_id>/results/<query_result_id>.csv',
+                 '/api/queries/<query_id>/results.csv',
+                 endpoint='csv_query_results')
+api.add_resource(QueryResultListAPI, '/api/query_results', endpoint='query_results')
+api.add_resource(QueryResultAPI, '/api/query_results/<query_result_id>', endpoint='query_result')
+
+
+class JobAPI(BaseResource):
+    def get(self, job_id):
+        # TODO: if finished, include the query result
+        job = data.Job.load(data_manager.redis_connection, job_id)
+        return {'job': job.to_dict()}
+
+    def delete(self, job_id):
+        job = data.Job.load(data_manager.redis_connection, job_id)
+        job.cancel()
+
+api.add_resource(JobAPI, '/api/jobs/<job_id>', endpoint='job')
+
+@app.route('/<path:filename>')
+def send_static(filename):
+    return send_from_directory(settings.STATIC_ASSETS_PATH, filename)
+
+
+if __name__ == '__main__':
+    app.run(debug=True)
+
+
+

redash/data/__init__.py

@@ -1,4 +1,2 @@
 from manager import Manager
 from worker import Job
-import models
-import utils

redash/data/manager.py

@@ -0,0 +1,159 @@
+"""
+Data manager. Used to manage and coordinate execution of queries.
+"""
+import time
+import logging
+import peewee
+import qr
+import redis
+import json
+from redash import models
+from redash.data import worker
+from redash.utils import gen_query_hash
+
+
+class JSONPriorityQueue(qr.PriorityQueue):
+    """ Use a JSON serializer to help with cross language support """
+    def __init__(self, key, **kwargs):
+        super(qr.PriorityQueue, self).__init__(key, **kwargs)
+        self.serializer = json
+
+
+class Manager(object):
+    def __init__(self, redis_connection, statsd_client):
+        self.statsd_client = statsd_client
+        self.redis_connection = redis_connection
+        self.workers = []
+        self.queue = JSONPriorityQueue("jobs", **self.redis_connection.connection_pool.connection_kwargs)
+        self.max_retries = 5
+        self.status = {
+            'last_refresh_at': 0,
+            'started_at': time.time()
+        }
+
+        self._save_status()
+
+    def add_job(self, query, priority, data_source):
+        query_hash = gen_query_hash(query)
+        logging.info("[Manager][%s] Inserting job with priority=%s", query_hash, priority)
+        try_count = 0
+        job = None
+
+        while try_count < self.max_retries:
+            try_count += 1
+
+            pipe = self.redis_connection.pipeline()
+            try:
+                pipe.watch('query_hash_job:%s' % query_hash)
+                job_id = pipe.get('query_hash_job:%s' % query_hash)
+                if job_id:
+                    logging.info("[Manager][%s] Found existing job: %s", query_hash, job_id)
+                    job = worker.Job.load(self.redis_connection, job_id)
+                else:
+                    job = worker.Job(self.redis_connection, query=query, priority=priority,
+                                     data_source_id=data_source.id,
+                                     data_source_name=data_source.name,
+                                     data_source_type=data_source.type,
+                                     data_source_options=data_source.options)
+                    pipe.multi()
+                    job.save(pipe)
+                    logging.info("[Manager][%s] Created new job: %s", query_hash, job.id)
+                    self.queue.push(job.id, job.priority)
+                break
+
+            except redis.WatchError:
+                continue
+
+        if not job:
+            logging.error("[Manager][%s] Failed adding job for query.", query_hash)
+
+        return job
+
+    def report_status(self):
+        workers = [self.redis_connection.hgetall(w)
+                   for w in self.redis_connection.smembers('workers')]
+
+        for w in workers:
+            self.statsd_client.gauge('worker_{}.seconds_since_update'.format(w['id']),
+                                     time.time() - float(w['updated_at']))
+            self.statsd_client.gauge('worker_{}.jobs_received'.format(w['id']), int(w['jobs_count']))
+            self.statsd_client.gauge('worker_{}.jobs_done'.format(w['id']), int(w['done_jobs_count']))
+
+        manager_status = self.redis_connection.hgetall('manager:status')
+        self.statsd_client.gauge('manager.seconds_since_refresh',
+                                 time.time() - float(manager_status['last_refresh_at']))
+
+    def refresh_queries(self):
+        # TODO: this will only execute scheduled queries that were executed before. I think this is
+        # a reasonable assumption, but worth revisiting.
+
+        # TODO: move this logic to the model.
+        outdated_queries = models.Query.select(peewee.Func('first_value', models.Query.id)\
+            .over(partition_by=[models.Query.query_hash, models.Query.data_source]))\
+            .join(models.QueryResult)\
+            .where(models.Query.ttl > 0,
+                   (models.QueryResult.retrieved_at +
+                    (models.Query.ttl * peewee.SQL("interval '1 second'"))) <
+                   peewee.SQL("(now() at time zone 'utc')"))
+
+        queries = models.Query.select(models.Query, models.DataSource).join(models.DataSource)\
+            .where(models.Query.id << outdated_queries)
+
+        self.status['last_refresh_at'] = time.time()
+        self._save_status()
+
+        logging.info("Refreshing queries...")
+
+        outdated_queries_count = 0
+        for query in queries:
+            self.add_job(query.query, worker.Job.LOW_PRIORITY, query.data_source)
+            outdated_queries_count += 1
+
+        self.statsd_client.gauge('manager.outdated_queries', outdated_queries_count)
+        self.statsd_client.gauge('manager.queue_size', self.redis_connection.zcard('jobs'))
+
+        logging.info("Done refreshing queries... %d" % outdated_queries_count)
+
+    def store_query_result(self, data_source_id, query, data, run_time, retrieved_at):
+        query_hash = gen_query_hash(query)
+
+        query_result = models.QueryResult.create(query_hash=query_hash,
+                                                 query=query,
+                                                 runtime=run_time,
+                                                 data_source=data_source_id,
+                                                 retrieved_at=retrieved_at,
+                                                 data=data)
+
+        logging.info("[Manager][%s] Inserted query data; id=%s", query_hash, query_result.id)
+
+        # TODO: move this logic to the model?
+        updated_count = models.Query.update(latest_query_data=query_result).\
+            where(models.Query.query_hash==query_hash, models.Query.data_source==data_source_id).\
+            execute()
+
+        logging.info("[Manager][%s] Updated %s queries.", query_hash, updated_count)
+
+        return query_result.id
+
+    def start_workers(self, workers_count):
+        if self.workers:
+            return self.workers
+
+        redis_connection_params = self.redis_connection.connection_pool.connection_kwargs
+        self.workers = [worker.Worker(worker_id, self, redis_connection_params)
+                        for worker_id in xrange(workers_count)]
+
+        for w in self.workers:
+            w.start()
+
+        return self.workers
+
+    def stop_workers(self):
+        for w in self.workers:
+            w.terminate()
+
+        for w in self.workers:
+            w.join()
+
+    def _save_status(self):
+        self.redis_connection.hmset('manager:status', self.status)

redash/data/query_runner.py

@@ -0,0 +1,30 @@
+import json
+
+
+def get_query_runner(connection_type, connection_string):
+    if connection_type == 'mysql':
+        from redash.data import query_runner_mysql
+        runner = query_runner_mysql.mysql(connection_string)
+    elif connection_type == 'graphite':
+        from redash.data import query_runner_graphite
+        connection_params = json.loads(connection_string)
+        if connection_params['auth']:
+            connection_params['auth'] = tuple(connection_params['auth'])
+        else:
+            connection_params['auth'] = None
+        runner = query_runner_graphite.graphite(connection_params)
+    elif connection_type == 'bigquery':
+        from redash.data import query_runner_bigquery
+        connection_params = json.loads(connection_string)
+        runner = query_runner_bigquery.bigquery(connection_params)
+    elif connection_type == 'script':
+        from redash.data import query_runner_script
+        runner = query_runner_script.script(connection_string)
+    elif connection_type == 'url':
+        from redash.data import query_runner_url
+        runner = query_runner_url.url(connection_string)
+    else:
+        from redash.data import query_runner_pg
+        runner = query_runner_pg.pg(connection_string)
+
+    return runner

redash/data/query_runner_bigquery.py

@@ -0,0 +1,98 @@
+import httplib2
+import json
+import logging
+import sys
+
+try:
+    import apiclient.errors
+    from apiclient.discovery import build
+    from apiclient.errors import HttpError
+    from oauth2client.client import SignedJwtAssertionCredentials
+except ImportError:
+    print "Missing dependencies. Please install google-api-python-client and oauth2client."
+    print "You can use pip:   pip install google-api-python-client oauth2client"
+
+from redash.utils import JSONEncoder
+
+def bigquery(connection_string):
+    def load_key(filename):
+        f = file(filename, "rb")
+        try:
+            return f.read()
+        finally:
+            f.close()
+
+    def get_bigquery_service():
+        scope = [
+            "https://www.googleapis.com/auth/bigquery",
+        ]
+
+        credentials = SignedJwtAssertionCredentials(connection_string["serviceAccount"], load_key(connection_string["privateKey"]), scope=scope)
+        http = httplib2.Http()
+        http = credentials.authorize(http)
+
+        return build("bigquery", "v2", http=http)
+
+    def query_runner(query):
+        bigquery_service = get_bigquery_service()
+
+        jobs = bigquery_service.jobs()
+        job_data = {
+            "configuration": {
+                "query": {
+                    "query": query,
+                }
+            }
+        }
+
+        logging.debug("bigquery got query: %s", query)
+
+        project_id = connection_string["projectId"]
+
+        try:
+            insert_response = jobs.insert(projectId=project_id, body=job_data).execute()
+            current_row = 0
+            query_reply = jobs.getQueryResults(projectId=project_id, jobId=insert_response['jobReference']['jobId'], startIndex=current_row).execute()
+
+            rows = []
+            field_names = []
+            for f in query_reply["schema"]["fields"]:
+                field_names.append(f["name"])
+
+            while(("rows" in query_reply) and current_row < query_reply['totalRows']):
+                for row in query_reply["rows"]:
+                    row_data = {}
+                    column_index = 0
+                    for cell in row["f"]:
+                        row_data[field_names[column_index]] = cell["v"]
+                        column_index += 1
+
+                    rows.append(row_data)
+
+                current_row += len(query_reply['rows'])
+                query_reply = jobs.getQueryResults(projectId=project_id, jobId=query_reply['jobReference']['jobId'], startIndex=current_row).execute()
+
+            columns = [{'name': name,
+                        'friendly_name': name,
+                        'type': None} for name in field_names]
+
+            data = {
+                "columns" : columns,
+                "rows" : rows
+            }
+            error = None
+
+            json_data = json.dumps(data, cls=JSONEncoder)
+        except apiclient.errors.HttpError, e:
+            json_data = None
+            error = e.args[1]
+        except KeyboardInterrupt:
+            error = "Query cancelled by user."
+            json_data = None
+        except Exception as e:
+            raise sys.exc_info()[1], None, sys.exc_info()[2]
+
+        return json_data, error
+
+
+    return query_runner

redash/data/query_runner_graphite.py

@@ -0,0 +1,46 @@
+"""
+QueryRunner for Graphite.
+"""
+import json
+import datetime
+import requests
+from redash.utils import JSONEncoder
+
+
+def graphite(connection_params):
+    def transform_result(response):
+        columns = [{'name': 'Time::x'}, {'name': 'value::y'}, {'name': 'name::series'}]
+        rows = []
+
+        for series in response.json():
+            for values in series['datapoints']:
+                timestamp = datetime.datetime.fromtimestamp(int(values[1]))
+                rows.append({'Time::x': timestamp, 'name::series': series['target'], 'value::y': values[0]})
+
+        data = {'columns': columns, 'rows': rows}
+        return json.dumps(data, cls=JSONEncoder)
+
+    def query_runner(query):
+        base_url = "%s/render?format=json&" % connection_params['url']
+        url = "%s%s" % (base_url, "&".join(query.split("\n")))
+        error = None
+        data = None
+
+        try:
+            response = requests.get(url, auth=connection_params['auth'],
+                                    verify=connection_params['verify'])
+
+            if response.status_code == 200:
+                data = transform_result(response)
+            else:
+                error = "Failed getting results (%d)" % response.status_code
+
+        except Exception, ex:
+            data = None
+            error = ex.message
+
+        return data, error
+
+    query_runner.annotate_query = False
+
+    return query_runner

redash/data/query_runner_mysql.py

@@ -10,8 +10,7 @@ import logging
 import json
 import MySQLdb
 import sys
-import select
-from .utils import JSONEncoder
+from redash.utils import JSONEncoder
 
 def mysql(connection_string):
     if connection_string.endswith(';'):
@@ -29,22 +28,31 @@ def mysql(connection_string):
             
             data = cursor.fetchall()
             
-            num_fields = len(cursor.description)
-            column_names = [i[0] for i in cursor.description]
+            cursor_desc = cursor.description
+            if (cursor_desc != None):
+                num_fields = len(cursor_desc)
+                column_names = [i[0] for i in cursor.description]
             
-            rows = [dict(zip(column_names, row)) for row in data]
+                rows = [dict(zip(column_names, row)) for row in data]
 
-            columns = [{'name': col_name,
-                        'friendly_name': col_name,
-                        'type': None} for col_name in column_names]
+                columns = [{'name': col_name,
+                            'friendly_name': col_name,
+                            'type': None} for col_name in column_names]
             
-            data = {'columns': columns, 'rows': rows}
-            json_data = json.dumps(data, cls=JSONEncoder)
-            error = None
+                data = {'columns': columns, 'rows': rows}
+                json_data = json.dumps(data, cls=JSONEncoder)
+                error = None
+            else:
+                json_data = None
+                error = "No data was returned."
+                
             cursor.close()
         except MySQLdb.Error, e:
             json_data = None
-            error = e.message
+            error = e.args[1]
+        except KeyboardInterrupt:
+            error = "Query cancelled by user."
+            json_data = None            
         except Exception as e:
             raise sys.exc_info()[1], None, sys.exc_info()[2]
         finally:

redash/data/query_runner_pg.py

@@ -1,55 +1,63 @@
 """
-QueryRunner is the function that the workers use, to execute queries. This is the Redshift
-(PostgreSQL in fact) version, but easily we can write another to support additional databases
-(MySQL and others).
+QueryRunner is the function that the workers use, to execute queries. This is the PostgreSQL
+version, but easily we can write another to support additional databases (MySQL and others).
 
 Because the worker just pass the query, this can be used with any data store that has some sort of
 query language (for example: HiveQL).
 """
-import logging
 import json
-import psycopg2
 import sys
 import select
-from .utils import JSONEncoder
+import logging
+import psycopg2
 
-def redshift(connection_string):
+from redash.utils import JSONEncoder
+
+
+def pg(connection_string):
     def column_friendly_name(column_name):
         return column_name
- 
+
     def wait(conn):
         while 1:
-            state = conn.poll()
-            if state == psycopg2.extensions.POLL_OK:
-                break
-            elif state == psycopg2.extensions.POLL_WRITE:
-                select.select([], [conn.fileno()], [])
-            elif state == psycopg2.extensions.POLL_READ:
-                select.select([conn.fileno()], [], [])
-            else:
-                raise psycopg2.OperationalError("poll() returned %s" % state)
- 
+            try:
+                state = conn.poll()
+                if state == psycopg2.extensions.POLL_OK:
+                    break
+                elif state == psycopg2.extensions.POLL_WRITE:
+                    select.select([], [conn.fileno()], [])
+                elif state == psycopg2.extensions.POLL_READ:
+                    select.select([conn.fileno()], [], [])
+                else:
+                    raise psycopg2.OperationalError("poll() returned %s" % state)
+            except select.error:
+                raise psycopg2.OperationalError("select.error received")
+
     def query_runner(query):
         connection = psycopg2.connect(connection_string, async=True)
         wait(connection)
- 
+
         cursor = connection.cursor()
- 
+
         try:
             cursor.execute(query)
             wait(connection)
- 
+
             column_names = [col.name for col in cursor.description]
- 
+
             rows = [dict(zip(column_names, row)) for row in cursor]
             columns = [{'name': col.name,
                         'friendly_name': column_friendly_name(col.name),
                         'type': None} for col in cursor.description]
- 
+
             data = {'columns': columns, 'rows': rows}
             json_data = json.dumps(data, cls=JSONEncoder)
             error = None
             cursor.close()
+        except (select.error, OSError, psycopg2.OperationalError) as e:
+            logging.exception(e)
+            error = "Query interrupted. Please retry."
+            json_data = None
         except psycopg2.DatabaseError as e:
             json_data = None
             error = e.message
@@ -61,7 +69,7 @@ def redshift(connection_string):
             raise sys.exc_info()[1], None, sys.exc_info()[2]
         finally:
             connection.close()
- 
+
         return json_data, error
- 
-    return query_runner
+
+    return query_runner

redash/data/query_runner_script.py

@@ -0,0 +1,48 @@
+import json
+import logging
+import sys
+import os
+import subprocess
+
+# We use subprocess.check_output because we are lazy.
+# If someone will really want to run this on Python < 2.7 they can easily update the code to run
+# Popen, check the retcodes and other things and read the standard output to a variable.
+if not "check_output" in subprocess.__dict__:
+    print "ERROR: This runner uses subprocess.check_output function which exists in Python 2.7"
+
+def script(connection_string):
+
+    def query_runner(query):
+        try:
+            json_data = None
+            error = None
+
+            # Poor man's protection against running scripts from output the scripts directory
+            if connection_string.find("../") > -1:
+                return None, "Scripts can only be run from the configured scripts directory"
+
+            query = query.strip()
+
+            script = os.path.join(connection_string, query)
+            if not os.path.exists(script):
+                return None, "Script '%s' not found in script directory" % query
+
+            output = subprocess.check_output(script, shell=False)
+            if output != None:
+                output = output.strip()
+                if output != "":
+                    return output, None
+
+            error = "Error reading output"
+        except subprocess.CalledProcessError as e:
+            return None, str(e)
+        except KeyboardInterrupt:
+            error = "Query cancelled by user."
+            json_data = None
+        except Exception as e:
+            raise sys.exc_info()[1], None, sys.exc_info()[2]
+
+        return json_data, error
+
+    query_runner.annotate_query = False
+    return query_runner

redash/data/query_runner_url.py

@@ -0,0 +1,45 @@
+import json
+import logging
+import sys
+import os
+import urllib2
+
+def url(connection_string):
+
+    def query_runner(query):
+        base_url = connection_string
+
+        try:
+            json_data = None
+            error = None
+
+            query = query.strip()
+
+            if base_url is not None and base_url != "":
+                if query.find("://") > -1:
+                    return None, "Accepting only relative URLs to '%s'" % base_url
+
+            if base_url is None:
+                base_url = ""
+
+            url = base_url + query
+
+            json_data = urllib2.urlopen(url).read().strip()
+
+            if not json_data:
+                error = "Error reading data from '%s'" % url
+
+            return json_data, error
+
+        except urllib2.URLError as e:
+            return None, str(e)
+        except KeyboardInterrupt:
+            error = "Query cancelled by user."
+            json_data = None
+        except Exception as e:
+            raise sys.exc_info()[1], None, sys.exc_info()[2]
+
+        return json_data, error
+
+    query_runner.annotate_query = False
+    return query_runner

redash/data/worker.py

@@ -0,0 +1,370 @@
+"""
+Worker implementation to execute incoming queries.
+"""
+import json
+import logging
+import multiprocessing
+import os
+import uuid
+import datetime
+import time
+import signal
+import setproctitle
+import redis
+from statsd import StatsClient
+from redash.utils import gen_query_hash
+from redash.data.query_runner import get_query_runner
+from redash import settings
+
+
+class RedisObject(object):
+    # The following should be overriden in the inheriting class:
+    fields = {}
+    conversions = {}
+    id_field = ''
+    name = ''
+
+    def __init__(self, redis_connection, **kwargs):
+        self.redis_connection = redis_connection
+        self.values = {}
+
+        if not self.fields:
+            raise ValueError("You must set the fields dictionary, before using RedisObject.")
+
+        if not self.name:
+            raise ValueError("You must set the name, before using RedisObject")
+
+        self.update(**kwargs)
+
+    def __getattr__(self, name):
+        if name in self.values:
+            return self.values[name]
+        else:
+            raise AttributeError
+
+    def update(self, **kwargs):
+        for field, default_value in self.fields.iteritems():
+            value = kwargs.get(field, self.values.get(field, default_value))
+            if callable(value):
+                value = value()
+
+            if value == 'None':
+                value = None
+
+            if field in self.conversions and value:
+                value = self.conversions[field](value)
+
+            self.values[field] = value
+
+    @classmethod
+    def _redis_key(cls, object_id):
+        return '{}:{}'.format(cls.name, object_id)
+
+    def save(self, pipe):
+        if not pipe:
+            pipe = self.redis_connection.pipeline()
+
+        pipe.sadd('{}_set'.format(self.name), self.id)
+        pipe.hmset(self._redis_key(self.id), self.values)
+        pipe.publish(self._redis_key(self.id), json.dumps(self.to_dict()))
+
+        pipe.execute()
+
+    @classmethod
+    def load(cls, redis_connection, object_id):
+        object_dict = redis_connection.hgetall(cls._redis_key(object_id))
+        obj = None
+        if object_dict:
+            obj = cls(redis_connection, **object_dict)
+
+        return obj
+
+
+def fix_unicode(string):
+    if isinstance(string, unicode):
+        return string
+
+    return string.decode('utf-8')
+
+
+class Job(RedisObject):
+    HIGH_PRIORITY = 1
+    LOW_PRIORITY = 2
+
+    WAITING = 1
+    PROCESSING = 2
+    DONE = 3
+    FAILED = 4
+
+    fields = {
+        'id': lambda: str(uuid.uuid1()),
+        'query': None,
+        'priority': None,
+        'query_hash': None,
+        'wait_time': 0,
+        'query_time': 0,
+        'error': None,
+        'updated_at': time.time,
+        'status': WAITING,
+        'process_id': None,
+        'query_result_id': None,
+        'data_source_id': None,
+        'data_source_name': None,
+        'data_source_type': None,
+        'data_source_options': None
+    }
+
+    conversions = {
+        'query': fix_unicode,
+        'priority': int,
+        'updated_at': float,
+        'status': int,
+        'wait_time': float,
+        'query_time': float,
+        'process_id': int,
+        'query_result_id': int
+    }
+
+    name = 'job'
+
+    def __init__(self, redis_connection, query, priority, **kwargs):
+        kwargs['query'] = fix_unicode(query)
+        kwargs['priority'] = priority
+        kwargs['query_hash'] = gen_query_hash(kwargs['query'])
+        self.new_job = 'id' not in kwargs
+        super(Job, self).__init__(redis_connection, **kwargs)
+
+    def to_dict(self):
+        return {
+            'query': self.query,
+            'priority': self.priority,
+            'id': self.id,
+            'wait_time': self.wait_time,
+            'query_time': self.query_time,
+            'updated_at': self.updated_at,
+            'status': self.status,
+            'error': self.error,
+            'query_result_id': self.query_result_id,
+            'process_id': self.process_id,
+            'data_source_name': self.data_source_name,
+            'data_source_type': self.data_source_type
+        }
+
+    def cancel(self):
+        # TODO: Race condition:
+        # it's possible that it will be picked up by worker while processing the cancel order
+        if self.is_finished():
+            return
+
+        if self.status == self.PROCESSING:
+            try:
+                os.kill(self.process_id, signal.SIGINT)
+            except OSError as e:
+                logging.warning("[%s] Tried to cancel job but os.kill failed (pid=%d, error=%s)",
+                                self.id, self.process_id, e)
+
+        self.done(None, "Interrupted/Cancelled while running.")
+
+    def save(self, pipe=None):
+        if not pipe:
+            pipe = self.redis_connection.pipeline()
+
+        if self.new_job:
+            pipe.set('query_hash_job:%s' % self.query_hash, self.id)
+
+        if self.is_finished():
+            pipe.delete('query_hash_job:%s' % self.query_hash)
+
+        super(Job, self).save(pipe)
+
+    def expire(self, expire_time):
+        self.redis_connection.expire(self._redis_key(self.id), expire_time)
+
+    def processing(self, process_id):
+        self.update(status=self.PROCESSING,
+                    process_id=process_id,
+                    wait_time=time.time() - self.updated_at,
+                    updated_at=time.time())
+
+        self.save()
+
+    def is_finished(self):
+        return self.status in (self.FAILED, self.DONE)
+
+    def done(self, query_result_id, error):
+        if error:
+            new_status = self.FAILED
+        else:
+            new_status = self.DONE
+
+        self.update(status=new_status,
+                    query_result_id=query_result_id,
+                    error=error,
+                    query_time=time.time() - self.updated_at,
+                    updated_at=time.time())
+
+        self.save()
+
+    def __str__(self):
+        return "<Job:%s,priority:%d,status:%d>" % (self.id, self.priority, self.status)
+
+
+class Worker(multiprocessing.Process):
+    def __init__(self, worker_id, manager, redis_connection_params, sleep_time=0.1):
+        self.manager = manager
+
+        self.statsd_client = StatsClient(host=settings.STATSD_HOST, port=settings.STATSD_PORT,
+                                         prefix=settings.STATSD_PREFIX)
+
+        self.redis_connection_params = {k: v for k, v in redis_connection_params.iteritems()
+                                        if k in ('host', 'db', 'password', 'port')}
+
+        self.worker_id = None
+        self.continue_working = True
+        self.sleep_time = sleep_time
+        self.child_pid = None
+        self.current_job_id = None
+        self.status = {
+            'jobs_count': 0,
+            'cancelled_jobs_count': 0,
+            'done_jobs_count': 0,
+            'updated_at': time.time(),
+            'started_at': time.time()
+        }
+
+        super(Worker, self).__init__(name="Worker")
+
+    def set_title(self, title=None):
+        base_title = "redash worker:%s" % self.worker_id
+        if title:
+            full_title = "%s - %s" % (base_title, title)
+        else:
+            full_title = base_title
+
+        setproctitle.setproctitle(full_title)
+
+    def run(self):
+        self.worker_id = os.getpid()
+        self.status['id'] = self.worker_id
+        self.name = "Worker:%d" % self.worker_id
+        self.manager.redis_connection.sadd('workers', self._key)
+        self._save_status()
+        self.set_title()
+
+        logging.info("[%s] started.", self.name)
+
+        signal.signal(signal.SIGINT, self._stop)
+        signal.signal(signal.SIGTERM, self._stop)
+
+        self._wait_for_jobs()
+
+    def _stop(self, signum, frame):
+        self.continue_working = False
+        if self.current_job_id:
+            job = Job.load(self.manager.redis_connection, self.current_job_id)
+            if job:
+                job.cancel()
+
+    def _wait_for_jobs(self):
+        while self.continue_working:
+            job_id = self.manager.queue.pop()
+            if job_id:
+                self._update_status('jobs_count')
+                logging.info("[%s] Processing %s", self.name, job_id)
+                self._fork_and_process(job_id)
+                if self.child_pid == 0:
+                    return
+            else:
+                time.sleep(self.sleep_time)
+
+    def _update_status(self, counter):
+        self.status['updated_at'] = time.time()
+        self.status[counter] += 1
+        self._save_status()
+
+    @property
+    def _key(self):
+        return 'worker:%s' % self.worker_id
+
+    def _save_status(self):
+        self.manager.redis_connection.hmset(self._key, self.status)
+
+    def _fork_and_process(self, job_id):
+        self.current_job_id = job_id
+        self.child_pid = os.fork()
+        if self.child_pid == 0:
+            self.set_title("processing %s" % job_id)
+            self._process(job_id)
+        else:
+            logging.info("[%s] Waiting for pid: %d", self.name, self.child_pid)
+
+            try:
+                _, status = os.waitpid(self.child_pid, 0)
+            except OSError:
+                logging.info("[%s] OSError while waiting for child to finish", self.name)
+                # setting status to >0, so the job cleanup is triggered
+                status = 1
+
+            self._update_status('done_jobs_count')
+
+            job = Job.load(self.manager.redis_connection, job_id)
+            if status > 0 and not job.is_finished():
+                self._update_status('cancelled_jobs_count')
+                logging.info("[%s] process interrupted and job %s hasn't finished; registering interruption in job",
+                             self.name, job_id)
+                job.done(None, "Interrupted/Cancelled while running.")
+
+            job.expire(24 * 3600)
+
+            logging.info("[%s] Finished Processing %s (pid: %d status: %d)",
+                         self.name, job_id, self.child_pid, status)
+
+            self.child_pid = None
+            self.current_job_id = None
+
+    def _process(self, job_id):
+        redis_connection = redis.StrictRedis(**self.redis_connection_params)
+        job = Job.load(redis_connection, job_id)
+        if job.is_finished():
+            logging.warning("[%s][%s] tried to process finished job.", self.name, job)
+            return
+
+        pid = os.getpid()
+        job.processing(pid)
+
+        logging.info("[%s][%s] running query...", self.name, job.id)
+        start_time = time.time()
+        self.set_title("running query %s" % job_id)
+
+        logging.info("[%s][%s] Loading query runner (%s, %s)...", self.name, job.id,
+                     job.data_source_name, job.data_source_type)
+
+        query_runner = get_query_runner(job.data_source_type, job.data_source_options)
+
+        if getattr(query_runner, 'annotate_query', True):
+            annotated_query = "/* Pid: %s, Job Id: %s, Query hash: %s, Priority: %s */ %s" % \
+                              (pid, job.id, job.query_hash, job.priority, job.query)
+        else:
+            annotated_query = job.query
+
+        # TODO: here's the part that needs to be forked, not all of the worker process...
+        with self.statsd_client.timer('worker_{}.query_runner.{}.{}.run_time'.format(self.worker_id,
+                                                                                     job.data_source_type,
+                                                                                     job.data_source_name)):
+            data, error = query_runner(annotated_query)
+
+        run_time = time.time() - start_time
+        logging.info("[%s][%s] query finished... data length=%s, error=%s",
+                     self.name, job.id, data and len(data), error)
+
+        # TODO: it is possible that storing the data will fail, and we will need to retry
+        # while we already marked the job as done
+        query_result_id = None
+        if not error:
+            self.set_title("storing results %s" % job_id)
+            query_result_id = self.manager.store_query_result(job.data_source_id,
+                                                              job.query, data, run_time,
+                                                              datetime.datetime.utcnow())
+
+        self.set_title("marking job as done %s" % job_id)
+        job.done(query_result_id, error)

redash/events.py

@@ -0,0 +1,23 @@
+import logging
+import json
+
+logger = logging.getLogger("redash.events")
+logger.propagate = False
+
+
+def setup_logging(log_path, console_output=False):
+    if log_path:
+        fh = logging.FileHandler(log_path)
+        formatter = logging.Formatter('%(message)s')
+        fh.setFormatter(formatter)
+        logger.addHandler(fh)
+
+    if console_output:
+        handler = logging.StreamHandler()
+        formatter = logging.Formatter('[%(name)s] %(message)s')
+        handler.setFormatter(formatter)
+        logger.addHandler(handler)
+
+
+def record_event(event):
+    logger.info(json.dumps(event))

redash/import_export.py

@@ -0,0 +1,158 @@
+import contextlib
+import json
+from redash import models
+from flask.ext.script import Manager
+
+
+class Importer(object):
+    def __init__(self, object_mapping=None, data_source=None):
+        if object_mapping is None:
+            object_mapping = {}
+        self.object_mapping = object_mapping
+        self.data_source = data_source
+
+    def import_query_result(self, query_result):
+        query_result = self._get_or_create(models.QueryResult, query_result['id'],
+                                           data_source=self.data_source,
+                                           data=json.dumps(query_result['data']),
+                                           query_hash=query_result['query_hash'],
+                                           retrieved_at=query_result['retrieved_at'],
+                                           query=query_result['query'],
+                                           runtime=query_result['runtime'])
+
+        return query_result
+
+
+    def import_query(self, user, query):
+        query_result = self.import_query_result(query['latest_query_data'])
+
+        new_query = self._get_or_create(models.Query, query['id'], name=query['name'],
+                                        user=user,
+                                        ttl=-1,
+                                        query=query['query'],
+                                        query_hash=query['query_hash'],
+                                        description=query['description'],
+                                        latest_query_data=query_result,
+                                        data_source=self.data_source)
+
+        return new_query
+
+
+    def import_visualization(self, user, visualization):
+        query = self.import_query(user, visualization['query'])
+
+        new_visualization = self._get_or_create(models.Visualization, visualization['id'],
+                                                name=visualization['name'],
+                                                description=visualization['description'],
+                                                type=visualization['type'],
+                                                options=json.dumps(visualization['options']),
+                                                query=query)
+        return new_visualization
+
+    def import_widget(self, dashboard, widget):
+        visualization = self.import_visualization(dashboard.user, widget['visualization'])
+
+        new_widget = self._get_or_create(models.Widget, widget['id'],
+                                         dashboard=dashboard,
+                                         width=widget['width'],
+                                         options=json.dumps(widget['options']),
+                                         visualization=visualization)
+
+        return new_widget
+
+    def import_dashboard(self, user, dashboard):
+        """
+        Imports dashboard along with widgets, visualizations and queries from another re:dash.
+
+        user - the user to associate all objects with.
+        dashboard - dashboard to import (can be result of loading a json output).
+        """
+
+        new_dashboard = self._get_or_create(models.Dashboard, dashboard['id'],
+                                            name=dashboard['name'],
+                                            slug=dashboard['slug'],
+                                            layout='[]',
+                                            user=user)
+
+        layout = []
+
+        for widgets in dashboard['widgets']:
+            row = []
+            for widget in widgets:
+                widget_id = self.import_widget(new_dashboard, widget).id
+                row.append(widget_id)
+
+            layout.append(row)
+
+        new_dashboard.layout = json.dumps(layout)
+        new_dashboard.save()
+
+        return new_dashboard
+
+    def _get_or_create(self, object_type, external_id, **properties):
+        internal_id = self._get_mapping(object_type, external_id)
+        if internal_id:
+            update = object_type.update(**properties).where(object_type.id == internal_id)
+            update.execute()
+            obj = object_type.get_by_id(internal_id)
+        else:
+            obj = object_type.create(**properties)
+            self._update_mapping(object_type, external_id, obj.id)
+
+        return obj
+
+    def _get_mapping(self, object_type, external_id):
+        self.object_mapping.setdefault(object_type.__name__, {})
+        return self.object_mapping[object_type.__name__].get(str(external_id), None)
+
+    def _update_mapping(self, object_type, external_id, internal_id):
+        self.object_mapping.setdefault(object_type.__name__, {})
+        self.object_mapping[object_type.__name__][str(external_id)] = internal_id
+
+import_manager = Manager(help="import utilities")
+export_manager = Manager(help="export utilities")
+
+
+@contextlib.contextmanager
+def importer_with_mapping_file(mapping_filename):
+    with open(mapping_filename) as f:
+        mapping = json.loads(f.read())
+
+    importer = Importer(object_mapping=mapping, data_source=get_data_source())
+    yield importer
+
+    with open(mapping_filename, 'w') as f:
+        f.write(json.dumps(importer.object_mapping, indent=2))
+
+
+def get_data_source():
+    try:
+        data_source = models.DataSource.get(models.DataSource.name=="Import")
+    except models.DataSource.DoesNotExist:
+        data_source = models.DataSource.create(name="Import", type="import", options='{}')
+
+    return data_source
+
+@import_manager.command
+def query(mapping_filename, query_filename, user_id):
+    user = models.User.get_by_id(user_id)
+    with open(query_filename) as f:
+        query = json.loads(f.read())
+
+    with importer_with_mapping_file(mapping_filename) as importer:
+        imported_query = importer.import_query(user, query)
+
+        print "New query id: {}".format(imported_query.id)
+
+
+@import_manager.command
+def dashboard(mapping_filename, dashboard_filename, user_id):
+    user = models.User.get_by_id(user_id)
+    with open(dashboard_filename) as f:
+        dashboard = json.loads(f.read())
+
+    with importer_with_mapping_file(mapping_filename) as importer:
+        importer.import_dashboard(user, dashboard)
+
+
+

redash/models.py

@@ -0,0 +1,368 @@
+import json
+import hashlib
+import time
+import datetime
+from flask.ext.peewee.utils import slugify
+from flask.ext.login import UserMixin, AnonymousUserMixin
+from passlib.apps import custom_app_context as pwd_context
+import peewee
+from playhouse.postgres_ext import ArrayField
+from redash import db, utils
+
+
+class BaseModel(db.Model):
+    @classmethod
+    def get_by_id(cls, model_id):
+        return cls.get(cls.id == model_id)
+
+
+class AnonymousUser(AnonymousUserMixin):
+    @property
+    def permissions(self):
+        return []
+
+
+class ApiUser(UserMixin):
+    def __init__(self, api_key):
+        self.id = api_key
+
+    @property
+    def permissions(self):
+        return ['view_query']
+
+
+class User(BaseModel, UserMixin):
+    DEFAULT_PERMISSIONS = ['create_dashboard', 'create_query', 'edit_dashboard', 'edit_query',
+                           'view_query', 'view_source', 'execute_query']
+
+    id = peewee.PrimaryKeyField()
+    name = peewee.CharField(max_length=320)
+    email = peewee.CharField(max_length=320, index=True, unique=True)
+    password_hash = peewee.CharField(max_length=128, null=True)
+    is_admin = peewee.BooleanField(default=False)
+    permissions = ArrayField(peewee.CharField, default=DEFAULT_PERMISSIONS)
+
+    class Meta:
+        db_table = 'users'
+
+    def to_dict(self):
+        return {
+            'id': self.id,
+            'name': self.name,
+            'email': self.email,
+            'is_admin': self.is_admin
+        }
+
+    def __unicode__(self):
+        return '%r, %r' % (self.name, self.email)
+
+    def hash_password(self, password):
+        self.password_hash = pwd_context.encrypt(password)
+
+    def verify_password(self, password):
+        return self.password_hash and pwd_context.verify(password, self.password_hash)
+
+
+class ActivityLog(BaseModel):
+    QUERY_EXECUTION = 1
+    
+    id = peewee.PrimaryKeyField()
+    user = peewee.ForeignKeyField(User)
+    type = peewee.IntegerField()
+    activity = peewee.TextField()
+    created_at = peewee.DateTimeField(default=datetime.datetime.now)
+
+    class Meta:
+        db_table = 'activity_log'
+
+    def to_dict(self):
+        return {
+            'id': self.id,
+            'user': self.user.to_dict(),
+            'type': self.type,
+            'activity': self.activity,
+            'created_at': self.created_at
+        }
+
+    def __unicode__(self):
+        return unicode(self.id)
+
+
+class DataSource(BaseModel):
+    id = peewee.PrimaryKeyField()
+    name = peewee.CharField()
+    type = peewee.CharField()
+    options = peewee.TextField()
+    created_at = peewee.DateTimeField(default=datetime.datetime.now)
+
+    class Meta:
+        db_table = 'data_sources'
+
+    def to_dict(self):
+        return {
+            'id': self.id,
+            'name': self.name,
+            'type': self.type
+        }
+
+
+class QueryResult(BaseModel):
+    id = peewee.PrimaryKeyField()
+    data_source = peewee.ForeignKeyField(DataSource)
+    query_hash = peewee.CharField(max_length=32, index=True)
+    query = peewee.TextField()
+    data = peewee.TextField()
+    runtime = peewee.FloatField()
+    retrieved_at = peewee.DateTimeField()
+
+    class Meta:
+        db_table = 'query_results'
+
+    def to_dict(self):
+        return {
+            'id': self.id,
+            'query_hash': self.query_hash,
+            'query': self.query,
+            'data': json.loads(self.data),
+            'data_source_id': self._data.get('data_source', None),
+            'runtime': self.runtime,
+            'retrieved_at': self.retrieved_at
+        }
+
+    @classmethod
+    def get_latest(cls, data_source, query, ttl=0):
+        query_hash = utils.gen_query_hash(query)
+
+        query = cls.select().where(cls.query_hash == query_hash, cls.data_source == data_source,
+                                   peewee.SQL("retrieved_at + interval '%s second' >= now() at time zone 'utc'", ttl)).order_by(cls.retrieved_at.desc())
+
+        return query.first()
+
+    def __unicode__(self):
+        return u"%d | %s | %s" % (self.id, self.query_hash, self.retrieved_at)
+
+
+class Query(BaseModel):
+    id = peewee.PrimaryKeyField()
+    data_source = peewee.ForeignKeyField(DataSource)
+    latest_query_data = peewee.ForeignKeyField(QueryResult, null=True)
+    name = peewee.CharField(max_length=255)
+    description = peewee.CharField(max_length=4096, null=True)
+    query = peewee.TextField()
+    query_hash = peewee.CharField(max_length=32)
+    api_key = peewee.CharField(max_length=40)
+    ttl = peewee.IntegerField()
+    user_email = peewee.CharField(max_length=360, null=True)
+    user = peewee.ForeignKeyField(User)
+    created_at = peewee.DateTimeField(default=datetime.datetime.now)
+
+    class Meta:
+        db_table = 'queries'
+
+    def create_default_visualizations(self):
+        table_visualization = Visualization(query=self, name="Table",
+                                            description='',
+                                            type="TABLE", options="{}")
+        table_visualization.save()
+
+    def to_dict(self, with_result=True, with_stats=False, with_visualizations=False, with_user=True):
+        d = {
+            'id': self.id,
+            'latest_query_data_id': self._data.get('latest_query_data', None),
+            'name': self.name,
+            'description': self.description,
+            'query': self.query,
+            'query_hash': self.query_hash,
+            'ttl': self.ttl,
+            'api_key': self.api_key,
+            'created_at': self.created_at,
+            'data_source_id': self._data.get('data_source', None)
+        }
+
+        if with_user:
+            d['user'] = self.user.to_dict()
+        else:
+            d['user_id'] = self._data['user']
+
+        if with_stats:
+            d['avg_runtime'] = self.avg_runtime
+            d['min_runtime'] = self.min_runtime
+            d['max_runtime'] = self.max_runtime
+            d['last_retrieved_at'] = self.last_retrieved_at
+            d['times_retrieved'] = self.times_retrieved
+
+        if with_visualizations:
+            d['visualizations'] = [vis.to_dict(with_query=False)
+                                   for vis in self.visualizations]
+
+        if with_result and self.latest_query_data:
+            d['latest_query_data'] = self.latest_query_data.to_dict()
+
+        return d
+
+    @classmethod
+    def all_queries(cls):
+        q = Query.select(Query, User,
+                     peewee.fn.Count(QueryResult.id).alias('times_retrieved'),
+                     peewee.fn.Avg(QueryResult.runtime).alias('avg_runtime'),
+                     peewee.fn.Min(QueryResult.runtime).alias('min_runtime'),
+                     peewee.fn.Max(QueryResult.runtime).alias('max_runtime'),
+                     peewee.fn.Max(QueryResult.retrieved_at).alias('last_retrieved_at'))\
+            .join(QueryResult, join_type=peewee.JOIN_LEFT_OUTER)\
+            .switch(Query).join(User)\
+            .group_by(Query.id, User.id)
+
+        return q
+
+    @classmethod
+    def update_instance(cls, query_id, **kwargs):
+        if 'query' in kwargs:
+            kwargs['query_hash'] = utils.gen_query_hash(kwargs['query'])
+
+        update = cls.update(**kwargs).where(cls.id == query_id)
+        return update.execute()
+
+    def save(self, *args, **kwargs):
+        self.query_hash = utils.gen_query_hash(self.query)
+        self._set_api_key()
+        super(Query, self).save(*args, **kwargs)
+
+    def _set_api_key(self):
+        if not self.api_key:
+            self.api_key = hashlib.sha1(
+                u''.join((str(time.time()), self.query, str(self._data['user']), self.name)).encode('utf-8')).hexdigest()
+
+    def __unicode__(self):
+        return unicode(self.id)
+
+
+class Dashboard(BaseModel):
+    id = peewee.PrimaryKeyField()
+    slug = peewee.CharField(max_length=140, index=True)
+    name = peewee.CharField(max_length=100)
+    user_email = peewee.CharField(max_length=360, null=True)
+    user = peewee.ForeignKeyField(User)
+    layout = peewee.TextField()
+    is_archived = peewee.BooleanField(default=False, index=True)
+    created_at = peewee.DateTimeField(default=datetime.datetime.now)
+
+    class Meta:
+        db_table = 'dashboards'
+
+    def to_dict(self, with_widgets=False):
+        layout = json.loads(self.layout)
+
+        if with_widgets:
+            widgets = Widget.select(Widget, Visualization, Query, QueryResult, User)\
+                .where(Widget.dashboard == self.id)\
+                .join(Visualization)\
+                .join(Query)\
+                .join(User)\
+                .switch(Query)\
+                .join(QueryResult)
+            widgets = {w.id: w.to_dict() for w in widgets}
+            widgets_layout = map(lambda row: map(lambda widget_id: widgets.get(widget_id, None), row), layout)
+        else:
+            widgets_layout = None
+
+        return {
+            'id': self.id,
+            'slug': self.slug,
+            'name': self.name,
+            'user_id': self._data['user'],
+            'layout': layout,
+            'widgets': widgets_layout
+        }
+
+    @classmethod
+    def get_by_slug(cls, slug):
+        return cls.get(cls.slug == slug)
+
+    def save(self, *args, **kwargs):
+        if not self.slug:
+            self.slug = slugify(self.name)
+
+            tries = 1
+            while self.select().where(Dashboard.slug == self.slug).first() is not None:
+                self.slug = slugify(self.name) + "_{0}".format(tries)
+                tries += 1
+
+        super(Dashboard, self).save(*args, **kwargs)
+
+    def __unicode__(self):
+        return u"%s=%s" % (self.id, self.name)
+
+
+class Visualization(BaseModel):
+    id = peewee.PrimaryKeyField()
+    type = peewee.CharField(max_length=100)
+    query = peewee.ForeignKeyField(Query, related_name='visualizations')
+    name = peewee.CharField(max_length=255)
+    description = peewee.CharField(max_length=4096, null=True)
+    options = peewee.TextField()
+
+    class Meta:
+        db_table = 'visualizations'
+
+    def to_dict(self, with_query=True):
+        d = {
+            'id': self.id,
+            'type': self.type,
+            'name': self.name,
+            'description': self.description,
+            'options': json.loads(self.options),
+        }
+
+        if with_query:
+            d['query'] = self.query.to_dict()
+
+        return d
+
+    def __unicode__(self):
+        return u"%s %s" % (self.id, self.type)
+
+
+class Widget(BaseModel):
+    id = peewee.PrimaryKeyField()
+    visualization = peewee.ForeignKeyField(Visualization, related_name='widgets')
+
+    width = peewee.IntegerField()
+    options = peewee.TextField()
+    dashboard = peewee.ForeignKeyField(Dashboard, related_name='widgets', index=True)
+    created_at = peewee.DateTimeField(default=datetime.datetime.now)
+
+    # unused; kept for backward compatability:
+    type = peewee.CharField(max_length=100, null=True)
+    query_id = peewee.IntegerField(null=True)
+
+    class Meta:
+        db_table = 'widgets'
+
+    def to_dict(self):
+        return {
+            'id': self.id,
+            'width': self.width,
+            'options': json.loads(self.options),
+            'visualization': self.visualization.to_dict(),
+            'dashboard_id': self._data['dashboard']
+        }
+
+    def __unicode__(self):
+        return u"%s" % self.id
+
+all_models = (DataSource, User, QueryResult, Query, Dashboard, Visualization, Widget, ActivityLog)
+
+
+def create_db(create_tables, drop_tables):
+    db.connect_db()
+
+    for model in all_models:
+        if drop_tables and model.table_exists():
+            # TODO: submit PR to peewee to allow passing cascade option to drop_table.
+            db.database.execute_sql('DROP TABLE %s CASCADE' % model._meta.db_table)
+            #model.drop_table()
+
+        if create_tables and not model.table_exists():
+            model.create_table()
+
+    db.close_db(None)