Fix build status image in readme

Update psycopg2 to 2.5.2.

.gitignore

@@ -4,10 +4,17 @@
 .coverage
 rd_ui/dist
 .DS_Store
+celerybeat-schedule*
+.#*
+\#*#
+*~
 
 # Vagrant related
 .vagrant
 Berksfile.lock
 redash/dump.rdb
 .env
-.ruby-version
+.ruby-version
+venv
+
+dump.rdb

Makefile

@@ -1,19 +1,22 @@
 NAME=redash
 VERSION=`python ./manage.py version`
 FULL_VERSION=$(VERSION)+b$(CIRCLE_BUILD_NUM)
-FILENAME=$(CIRCLE_ARTIFACTS)/$(NAME).$(FULL_VERSION).tar.gz
+# VERSION gets evaluated every time it's referenced, therefore we need to use VERSION here instead of FULL_VERSION.
+FILENAME=$(CIRCLE_ARTIFACTS)/$(NAME).$(VERSION).tar.gz
 
 deps:
 	cd rd_ui && npm install
-	cd rd_ui && npm install grunt-cli bower
+	cd rd_ui && npm install -g bower grunt-cli
 	cd rd_ui && bower install
 	cd rd_ui && grunt build
 
 pack:
+	sed -ri "s/^__version__ = '([0-9.]*)'/__version__ = '$(FULL_VERSION)'/" redash/__init__.py
 	tar -zcv -f $(FILENAME) --exclude=".git*" --exclude="*.pyc" --exclude="*.pyo" --exclude="venv" --exclude="rd_ui/node_modules" --exclude="rd_ui/dist/bower_components" --exclude="rd_ui/app" *
 
 upload:
-	python bin/upload_version.py $(FULL_VERSION) $(FILENAME)
+	python bin/upload_version.py $(VERSION) $(FILENAME)
 
 test:
 	nosetests --with-coverage --cover-package=redash tests/*.py
+	cd rd_ui && grunt test

Procfile.dev

@@ -1,2 +1,2 @@
 web: ./manage.py runserver -p $PORT
-worker: ./manage.py runworkers
+worker: ./bin/run celery worker --app=redash.worker --beat -Qqueries,celery,scheduled_queries

Procfile.heroku

@@ -1,2 +1,2 @@
 web: ./manage.py runserver -p $PORT --host 0.0.0.0 -d -r
-worker: ./manage.py runworkers
+worker: ./bin/run celery worker --app=redash.worker --beat -Qqueries,celery,scheduled_queries

README.md

@@ -1,18 +1,24 @@
-# [_re:dash_](https://github.com/everythingme/redash)
-![Build Status](https://circleci.com/gh/EverythingMe/redash.png?circle-token=8a695aa5ec2cbfa89b48c275aea298318016f040 "Build Status")
+<p align="center">
+  <img title="re:dash" src='https://raw.githubusercontent.com/EverythingMe/redash/screenshots/redash_logo.png' />
+
+</p>
+<p align="center">
+    <img title="Build Status" src='https://circleci.com/gh/EverythingMe/redash.png?circle-token=8a695aa5ec2cbfa89b48c275aea298318016f040'/>
+</p>
 
 **_re:dash_** is our take on freeing the data within our company in a way that will better fit our culture and usage patterns.
 
-Prior to **_re:dash_**, we tried to use tranditional BI suites and discovered a set of bloated, technically challenged and slow tools/flows. What we were looking for was a more hacker'ish way to look at data, so we built one.
+Prior to **_re:dash_**, we tried to use traditional BI suites and discovered a set of bloated, technically challenged and slow tools/flows. What we were looking for was a more hacker'ish way to look at data, so we built one.
 
 **_re:dash_** was built to allow fast and easy access to billions of records, that we process and collect using Amazon Redshift ("petabyte scale data warehouse" that "speaks" PostgreSQL).
+Today **_re:dash_** has support for querying multiple databases, including: Redshift, Google BigQuery, PostgreSQL, MySQL, Graphite and custom scripts.
 
 **_re:dash_** consists of two parts:
 
-1. **Query Editor**: think of [JS Fiddle](http://jsfiddle.net) for SQL queries. It's your way to share data in the organization in an open way, by sharing both the dataset and the query that generated it. This way everyone can peer review not only the resulting dataset but also the process that generated it. Also it's possible to fork it and generate new datasets and reach new insights. 
-2. **Dashboards/Visualizations**: once you have a dataset, you can create different visualizations out of it, and then combine several visualizations into a single dashboard. Currently it supports bar charts, pivot table and cohorts.
+1. **Query Editor**: think of [JS Fiddle](http://jsfiddle.net) for SQL queries. It's your way to share data in the organization in an open way, by sharing both the dataset and the query that generated it. This way everyone can peer review not only the resulting dataset but also the process that generated it. Also it's possible to fork it and generate new datasets and reach new insights.
+2. **Dashboards/Visualizations**: once you have a dataset, you can create different visualizations out of it, and then combine several visualizations into a single dashboard. Currently it supports charts, pivot table and cohorts.
 
-This is the first release, which is more than usable but still has its rough edges and way to go to fulfill its full potential. The Query Editor part is quite solid, but the visualizations need more work to enrich them and to make them more user friendly.
+**_re:dash_** is a work in progress and has its rough edges and way to go to fulfill its full potential. The Query Editor part is quite solid, but the visualizations need more work to enrich them and to make them more user friendly.
 
 ## Demo
 
@@ -22,51 +28,20 @@ You can try out the demo instance: http://rd-demo.herokuapp.com/ (login with any
 
 Due to Heroku dev plan limits, it has a small database of flights (see schema [here](http://rd-demo.herokuapp.com/dashboard/schema)). Also due to another Heroku limitation, it is running with the regular user, hence you can DELETE or INSERT data/tables. Please be nice and don't do this.
 
+## Getting Started
+
+* [Setting re:dash on your own server (Ubuntu)](https://github.com/EverythingMe/redash/wiki/Setting-re:dash-on-your-own-server-(for-version-0.4))
+
+
 ## Getting help
 
 * [Google Group (mailing list)](https://groups.google.com/forum/#!forum/redash-users): the best place to get updates about new releases or ask general questions.
-* #redash IRC channel on [Freenode](http://www.freenode.net/).
-
-## Technology
-
-* Python
-* [AngularJS](http://angularjs.org/)
-* [PostgreSQL](http://www.postgresql.org/) / [AWS Redshift](http://aws.amazon.com/redshift/)
-* [Redis](http://redis.io)
-
-PostgreSQL is used both as the operatinal database for the system, but also as the data store that is being queried. To be exact, we built this system to use on top of Amazon's Redshift, which supports the PG driver. But it's quite simple to add support for other datastores, and we do plan to do so.
-
-This is our first large scale AngularJS project, and we learned a lot during the development of it. There are still things we need to iron out, and comments on the way we use AngularJS are more than welcome (and pull requests just as well).
-
-### HighCharts
-
-HighCharts is really great, but it's not free for commercial use. Please refer to their [licensing options](http://shop.highsoft.com/highcharts.html), to see what applies for your use. 
-
-It's very likely that in the future we will switch to [D3.js](http://d3js.org/) instead.
-
-## Getting Started
-
-* [Setting up re:dash on Heroku in 5 minutes](https://github.com/EverythingMe/redash/wiki/Setting-up-re:dash-on-Heroku-in-5-minutes)
-* [Setting re:dash on your own server (Ubuntu)](https://github.com/EverythingMe/redash/wiki/Setting-re:dash-on-your-own-server-(Ubuntu))
-
-**Need help setting re:dash or one of the dependencies up?** Ping @arikfr on the IRC #redash channel or send a message to the [mailing list](https://groups.google.com/forum/#!forum/redash-users), and he will gladly help.
+* Find us [on gitter](https://gitter.im/EverythingMe/redash#) (chat).
+* Contact Arik, the maintainer directly: arik@everything.me.
 
 ## Roadmap
 
-Below you can see the "big" features of the next 3 releases (for full list, click on the link):
-
-### [v0.3](https://github.com/EverythingMe/redash/issues?milestone=2&state=open)
-
-- Dashboard filters: ability to filter/slice the data you see in a single dashboard using filters (date or selectors).
-- Multiple databases support (including other database type than PostgreSQL).
-- Scheduled reports by email.
-- Comments on queries.
-
-### [v0.4](https://github.com/EverythingMe/redash/issues?milestone=3&state=open)
-
-- Query versioning. 
-- More "realtime" UI (using websockets).
-- More visualizations.
+TBD.
 
 ## Reporting Bugs and Contributing Code
 

bin/latest_release.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+import sys
+import requests
+
+if __name__ == '__main__':
+    response = requests.get('https://api.github.com/repos/EverythingMe/redash/releases')
+
+    if response.status_code != 200:
+        exit("Failed getting releases (status code: %s)." % response.status_code)
+
+    sorted_releases = sorted(response.json(), key=lambda release: release['id'], reverse=True)
+
+    latest_release = sorted_releases[0]
+    asset_url = latest_release['assets'][0]['url']
+    filename = latest_release['assets'][0]['name']
+
+    wget_command = 'wget --header="Accept: application/octet-stream" %s -O %s' % (asset_url, filename)
+
+    if '--url-only' in sys.argv:
+        print asset_url
+    elif '--wget' in sys.argv:
+        print wget_command
+    else:
+        print "Latest release: %s" % latest_release['tag_name']
+        print latest_release['body']
+
+        print "\nTarball URL: %s" % asset_url
+        print 'wget: %s' % (wget_command)
+
+

bin/run

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+# Ideally I would use stdin with source, but in older bash versions this
+# wasn't supported properly.
+TEMP_ENV_FILE=`mktemp /tmp/redash_env.XXXXXX`
+sed 's/^REDASH/export REDASH/' .env > $TEMP_ENV_FILE
+source $TEMP_ENV_FILE
+rm $TEMP_ENV_FILE
+
+exec "$@"

bin/test_multithreading.py

@@ -0,0 +1,63 @@
+"""
+Script to test concurrency (multithreading/multiprocess) issues with the workers. Use with caution.
+"""
+import json
+import atfork
+atfork.monkeypatch_os_fork_functions()
+import atfork.stdlib_fixer
+atfork.stdlib_fixer.fix_logging_module()
+
+import time
+from redash.data import worker
+from redash import models, data_manager, redis_connection
+
+if __name__ == '__main__':
+    models.create_db(True, False)
+
+    print "Creating data source..."
+    data_source = models.DataSource.create(name="Concurrency", type="pg", options="dbname=postgres")
+
+    print "Clear jobs/hashes:"
+    redis_connection.delete("jobs")
+    query_hashes = redis_connection.keys("query_hash_*")
+    if query_hashes:
+        redis_connection.delete(*query_hashes)
+
+    starting_query_results_count = models.QueryResult.select().count()
+    jobs_count = 5000
+    workers_count = 10
+
+    print "Creating jobs..."
+    for i in xrange(jobs_count):
+        query = "SELECT {}".format(i)
+        print "Inserting: {}".format(query)
+        data_manager.add_job(query=query, priority=worker.Job.LOW_PRIORITY,
+                             data_source=data_source)
+
+    print "Starting workers..."
+    workers = data_manager.start_workers(workers_count)
+
+    print "Waiting for jobs to be done..."
+    keep_waiting = True
+    while keep_waiting:
+        results_count = models.QueryResult.select().count() - starting_query_results_count
+        print "QueryResults: {}".format(results_count)
+        time.sleep(5)
+        if results_count == jobs_count:
+            print "Yay done..."
+            keep_waiting = False
+
+    data_manager.stop_workers()
+
+    qr_count = 0
+    for qr in models.QueryResult.select():
+        number = int(qr.query.split()[1])
+        data_number = json.loads(qr.data)['rows'][0].values()[0]
+
+        if number != data_number:
+            print "Oops? {} != {} ({})".format(number, data_number, qr.id)
+        qr_count += 1
+
+    print "Verified {} query results.".format(qr_count)
+
+    print "Done."

bin/upload_version.py

@@ -3,30 +3,44 @@ import os
 import sys
 import json
 import requests
+import subprocess
+
+
+def capture_output(command):
+    proc = subprocess.Popen(command, stdout=subprocess.PIPE)
+    return proc.stdout.read()
+
 
 if __name__ == '__main__':
-  version = sys.argv[1]
-  filepath = sys.argv[2]
-  filename = filepath.split('/')[-1]
-  github_token = os.environ['GITHUB_TOKEN']
-  auth = (github_token, 'x-oauth-basic')
-  commit_sha = os.environ['CIRCLE_SHA1']
+    version = sys.argv[1]
+    filepath = sys.argv[2]
+    filename = filepath.split('/')[-1]
+    github_token = os.environ['GITHUB_TOKEN']
+    auth = (github_token, 'x-oauth-basic')
+    commit_sha = os.environ['CIRCLE_SHA1']
 
-  params = json.dumps({
-    'tag_name': 'v{0}'.format(version),
-    'name': 're:dash v{0}'.format(version),
-    'target_commitish': commit_sha,
-    'prerelease': True
-  })
+    commit_body = capture_output(["git", "log", "--format=%b", "-n", "1", commit_sha])
+    file_md5_checksum = capture_output(["md5sum", filepath]).split()[0]
+    file_sha256_checksum = capture_output(["sha256sum", filepath]).split()[0]
+    version_body = "%s\n\nMD5: %s\nSHA256: %s" % (commit_body, file_md5_checksum, file_sha256_checksum)
 
-  response = requests.post('https://api.github.com/repos/everythingme/redash/releases',
-                            data=params,
-                            auth=auth)
+    params = json.dumps({
+        'tag_name': 'v{0}'.format(version),
+        'name': 're:dash v{0}'.format(version),
+        'body': version_body,
+        'target_commitish': commit_sha,
+        'prerelease': True
+    })
 
-  upload_url = response.json()['upload_url']
-  upload_url = upload_url.replace('{?name}', '')
+    response = requests.post('https://api.github.com/repos/everythingme/redash/releases',
+                             data=params,
+                             auth=auth)
 
-  with open(filepath) as file_content:
-    headers = {'Content-Type': 'application/gzip'}
-    response = requests.post(upload_url, file_content, params={'name': filename}, auth=auth, headers=headers, verify=False)
+    upload_url = response.json()['upload_url']
+    upload_url = upload_url.replace('{?name}', '')
+
+    with open(filepath) as file_content:
+        headers = {'Content-Type': 'application/gzip'}
+        response = requests.post(upload_url, file_content, params={'name': filename}, auth=auth,
+                                 headers=headers, verify=False)
 

circle.yml

@@ -1,7 +1,7 @@
 machine:
   node:
     version:
-      0.10.22
+      0.10.24
   python:
     version:
       2.7.3
@@ -17,9 +17,12 @@ test:
   override:
     - make test
   post:
-    - make pack 
+    - make pack
 deployment:
   github:
     branch: master
     commands:
       - make upload
+notify:
+  webhooks:
+    - url: https://webhooks.gitter.im/e/895d09c3165a0913ac2f

manage.py

@@ -2,20 +2,17 @@
 """
 CLI to manage redash.
 """
-import atfork
-atfork.monkeypatch_os_fork_functions()
-import atfork.stdlib_fixer
-atfork.stdlib_fixer.fix_logging_module()
-
-import logging
-import time
-from redash import settings, app, db, models, data_manager, __version__
-from redash.import_export import import_manager
+import datetime
 from flask.ext.script import Manager, prompt_pass
 
+from redash import settings, models, __version__
+from redash.wsgi import app
+from redash.import_export import import_manager
+
 manager = Manager(app)
 database_manager = Manager(help="Manages the database (create/drop tables).")
 users_manager = Manager(help="Users management commands.")
+data_sources_manager = Manager(help="Data sources management commands.")
 
 @manager.command
 def version():
@@ -25,39 +22,74 @@ def version():
 
 @manager.command
 def runworkers():
-    """Starts the re:dash query executors/workers."""
+    """Prints deprecation warning."""
+    print "** This command is deprecated. Please use Celery's CLI to control the workers. **"
 
-    try:
-        old_workers = data_manager.redis_connection.smembers('workers')
-        data_manager.redis_connection.delete('workers')
-
-        logging.info("Cleaning old workers: %s", old_workers)
-
-        data_manager.start_workers(settings.WORKERS_COUNT, settings.CONNECTION_ADAPTER, settings.CONNECTION_STRING)
-        logging.info("Workers started.")
-
-        while True:
-            try:
-                data_manager.refresh_queries()
-                data_manager.report_status()
-            except Exception as e:
-                logging.error("Something went wrong with refreshing queries...")
-                logging.exception(e)
-            time.sleep(60)
-    except KeyboardInterrupt:
-        logging.warning("Exiting; waiting for threads")
-        data_manager.stop_workers()
 
 @manager.shell
 def make_shell_context():
+    from redash.models import db
     return dict(app=app, db=db, models=models)
 
+@manager.command
+def check_settings():
+    from types import ModuleType
+
+    for name in dir(settings):
+        item = getattr(settings, name)
+        if not callable(item) and not name.startswith("__") and not isinstance(item, ModuleType):
+            print "{} = {}".format(name, item)
+
+@manager.command
+def import_events(events_file):
+    import json
+    from collections import Counter
+
+    count = Counter()
+
+    with open(events_file) as f:
+        for line in f:
+            try:
+                event = json.loads(line)
+
+                user = event.pop('user_id')
+                action = event.pop('action')
+                object_type = event.pop('object_type')
+                object_id = event.pop('object_id', None)
+
+                if object_id == 'dashboard' and object_type == 'dashboard':
+                    count['bad dashboard id'] += 1
+                    continue
+
+                created_at = datetime.datetime.utcfromtimestamp(event.pop('timestamp'))
+                additional_properties = json.dumps(event)
+
+                models.Event.create(user=user, action=action, object_type=object_type, object_id=object_id,
+                                    additional_properties=additional_properties, created_at=created_at)
+
+                count['imported'] += 1
+
+            except Exception as ex:
+                print "Failed importing line:"
+                print line
+                print ex.message
+                count[ex.message] += 1
+                count['failed'] += 1
+
+                models.db.close_db(None)
+
+    for k, v in count.iteritems():
+        print k
+        print v
+
+
 @database_manager.command
 def create_tables():
     """Creates the database tables."""
-    from redash.models import create_db
+    from redash.models import create_db, init_db
 
     create_db(True, False)
+    init_db()
 
 @database_manager.command
 def drop_tables():
@@ -69,16 +101,24 @@ def drop_tables():
 
 @users_manager.option('email', help="User's email")
 @users_manager.option('name', help="User's full name")
-@users_manager.option('--admin', dest='is_admin', default=False, help="set user as admin")
-@users_manager.option('--google', dest='google_auth', default=False, help="user uses Google Auth to login")
-def create(email, name, is_admin=False, google_auth=False):
+@users_manager.option('--admin', dest='is_admin', action="store_true", default=False, help="set user as admin")
+@users_manager.option('--google', dest='google_auth', action="store_true", default=False, help="user uses Google Auth to login")
+@users_manager.option('--password', dest='password', default=None, help="Password for users who don't use Google Auth (leave blank for prompt).")
+@users_manager.option('--groups', dest='groups', default=models.User.DEFAULT_GROUPS, help="Comma seperated list of groups (leave blank for default).")
+def create(email, name, groups, is_admin=False, google_auth=False, password=None):
     print "Creating user (%s, %s)..." % (email, name)
     print "Admin: %r" % is_admin
     print "Login with Google Auth: %r\n" % google_auth
+    if isinstance(groups, basestring):
+        groups= groups.split(',')
+        groups.remove('') # in case it was empty string
 
-    user = models.User(email=email, name=name, is_admin=is_admin)
+    if is_admin:
+        groups += ['admin']
+
+    user = models.User(email=email, name=name, groups=groups)
     if not google_auth:
-        password = prompt_pass("Password")
+        password = password or prompt_pass("Password")
         user.hash_password(password)
 
     try:
@@ -92,13 +132,38 @@ def delete(email):
     deleted_count = models.User.delete().where(models.User.email == email).execute()
     print "Deleted %d users." % deleted_count
 
+@data_sources_manager.command
+def import_from_settings(name=None):
+    """Import data source from settings (env variables)."""
+    name = name or "Default"
+    data_source = models.DataSource.create(name=name,
+                                           type=settings.CONNECTION_ADAPTER,
+                                           options=settings.CONNECTION_STRING)
+
+    print "Imported data source from settings (id={}).".format(data_source.id)
+
+
+@data_sources_manager.command
+def list():
+    """List currently configured data sources"""
+    for ds in models.DataSource.select():
+        print "Name: {}\nType: {}\nOptions: {}".format(ds.name, ds.type, ds.options)
+
+@data_sources_manager.command
+def new(name, type, options):
+    """Create new data source"""
+    # TODO: validate it's a valid type and in the future, validate the options.
+    print "Creating {} data source ({}) with options:\n{}".format(type, name, options)
+    data_source = models.DataSource.create(name=name,
+                                           type=type,
+                                           options=options)
+    print "Id: {}".format(data_source.id)
+
+
 manager.add_command("database", database_manager)
 manager.add_command("users", users_manager)
 manager.add_command("import", import_manager)
+manager.add_command("ds", data_sources_manager)
 
 if __name__ == '__main__':
-    channel = logging.StreamHandler()
-    logging.getLogger().addHandler(channel)
-    logging.getLogger().setLevel(settings.LOG_LEVEL)
-
     manager.run()

migrations/add_global_filters_to_dashboard.py

@@ -0,0 +1,12 @@
+from playhouse.migrate import Migrator
+from redash import models
+from redash.models import db
+
+
+if __name__ == '__main__':
+    db.connect_db()
+    migrator = Migrator(db.database)
+    with db.database.transaction():
+        migrator.add_column(models.Dashboard, models.Dashboard.dashboard_filters_enabled, 'dashboard_filters_enabled')
+
+    db.close_db(None)

migrations/add_permissions_to_user.py

@@ -0,0 +1,13 @@
+from playhouse.migrate import Migrator
+from redash import db
+from redash import models
+
+
+if __name__ == '__main__':
+    db.connect_db()
+    migrator = Migrator(db.database)
+    with db.database.transaction():
+        migrator.add_column(models.User, models.User.permissions, 'permissions')
+        models.User.update(permissions=['admin'] + models.User.DEFAULT_PERMISSIONS).where(models.User.is_admin == True).execute()
+
+    db.close_db(None)

migrations/add_queue_name_to_data_source.py

@@ -0,0 +1,13 @@
+from playhouse.migrate import Migrator
+from redash.models import db
+from redash import models
+
+
+if __name__ == '__main__':
+    db.connect_db()
+    migrator = Migrator(db.database)
+    with db.database.transaction():
+        migrator.add_column(models.DataSource, models.DataSource.queue_name, 'queue_name')
+        migrator.add_column(models.DataSource, models.DataSource.scheduled_queue_name, 'scheduled_queue_name')
+
+    db.close_db(None)

migrations/add_text_to_widgets.py

@@ -0,0 +1,13 @@
+from playhouse.migrate import Migrator
+from redash.models import db
+from redash import models
+
+
+if __name__ == '__main__':
+    db.connect_db()
+    migrator = Migrator(db.database)
+    with db.database.transaction():
+        migrator.add_column(models.Widget, models.Widget.text, 'text')
+        migrator.set_nullable(models.Widget, models.Widget.visualization, True)
+
+    db.close_db(None)

migrations/add_view_query_permission.py

@@ -0,0 +1,13 @@
+import peewee
+from redash import db
+from redash import models
+
+
+if __name__ == '__main__':
+    db.connect_db()
+
+    previous_default_permissions = models.User.DEFAULT_PERMISSIONS[:]
+    previous_default_permissions.remove('view_query')
+    models.User.update(permissions=peewee.fn.array_append(models.User.permissions, 'view_query')).where(peewee.SQL("'view_source' = any(permissions)")).execute()
+
+    db.close_db(None)

migrations/create_data_sources.py

@@ -0,0 +1,48 @@
+import logging
+import peewee
+from playhouse.migrate import Migrator
+from redash import db
+from redash import models
+from redash import settings
+
+if __name__ == '__main__':
+    db.connect_db()
+
+    if not models.DataSource.table_exists():
+        print "Creating data_sources table..."
+        models.DataSource.create_table()
+
+        default_data_source = models.DataSource.create(name="Default",
+                                                       type=settings.CONNECTION_ADAPTER,
+                                                       options=settings.CONNECTION_STRING)
+    else:
+        default_data_source = models.DataSource.select().first()
+
+    migrator = Migrator(db.database)
+    models.Query.data_source.null = True
+    models.QueryResult.data_source.null = True
+    try:
+        with db.database.transaction():
+            migrator.add_column(models.Query, models.Query.data_source, "data_source_id")
+    except peewee.ProgrammingError:
+        print "Failed to create data_source_id column -- assuming it already exists"
+
+    try:
+        with db.database.transaction():
+            migrator.add_column(models.QueryResult, models.QueryResult.data_source, "data_source_id")
+    except peewee.ProgrammingError:
+        print "Failed to create data_source_id column -- assuming it already exists"
+
+    print "Updating data source to existing one..."
+    models.Query.update(data_source=default_data_source.id).execute()
+    models.QueryResult.update(data_source=default_data_source.id).execute()
+
+    with db.database.transaction():
+        print "Setting data source to non nullable..."
+        migrator.set_nullable(models.Query, models.Query.data_source, False)
+
+    with db.database.transaction():
+        print "Setting data source to non nullable..."
+        migrator.set_nullable(models.QueryResult, models.QueryResult.data_source, False)
+
+    db.close_db(None)

migrations/create_events.py

@@ -0,0 +1,12 @@
+from redash.models import db
+from redash import models
+
+
+if __name__ == '__main__':
+    db.connect_db()
+
+    if not models.Event.table_exists():
+        print "Creating events table..."
+        models.Event.create_table()
+
+    db.close_db(None)

migrations/permissions_migration.py

@@ -0,0 +1,29 @@
+import peewee
+from playhouse.migrate import Migrator
+from redash import models
+from redash.models import db
+
+
+if __name__ == '__main__':
+    db.connect_db()
+    migrator = Migrator(db.database)
+    
+    if not models.Group.table_exists():
+        print "Creating groups table..."
+        models.Group.create_table()
+    
+    with db.database.transaction():
+        models.Group.insert(name='admin', permissions=['admin'], tables=['*']).execute()
+        models.Group.insert(name='api', permissions=['view_query'], tables=['*']).execute()
+        models.Group.insert(name='default', permissions=models.Group.DEFAULT_PERMISSIONS, tables=['*']).execute()
+
+        migrator.add_column(models.User, models.User.groups, 'groups')
+        
+        models.User.update(groups=['admin', 'default']).where(peewee.SQL("is_admin = true")).execute()
+        models.User.update(groups=['admin', 'default']).where(peewee.SQL("'admin' = any(permissions)")).execute()
+        models.User.update(groups=['default']).where(peewee.SQL("is_admin = false")).execute()
+
+        migrator.drop_column(models.User, 'permissions')
+        migrator.drop_column(models.User, 'is_admin')
+
+    db.close_db(None)

rd_ui/.travis.yml

@@ -1,6 +1,5 @@
 language: node_js
 node_js:
-  - '0.8'
   - '0.10'
 before_script:
   - 'npm install -g bower grunt-cli'

rd_ui/Gruntfile.js

@@ -1,10 +1,5 @@
-// Generated on 2013-08-25 using generator-angular 0.4.0
+// Generated on 2014-07-30 using generator-angular 0.9.2
 'use strict';
-var LIVERELOAD_PORT = 35729;
-var lrSnippet = require('connect-livereload')({ port: LIVERELOAD_PORT });
-var mountFolder = function (connect, dir) {
-  return connect.static(require('path').resolve(dir));
-};
 
 // # Globbing
 // for performance reasons we're only matching one level down:
@@ -13,48 +8,148 @@ var mountFolder = function (connect, dir) {
 // 'test/spec/**/*.js'
 
 module.exports = function (grunt) {
+
+  // Load grunt tasks automatically
   require('load-grunt-tasks')(grunt);
+
+  // Time how long tasks take. Can help when optimizing build times
   require('time-grunt')(grunt);
 
-  // configurable paths
-  var yeomanConfig = {
-    app: 'app',
+  // Configurable paths for the application
+  var appConfig = {
+    app: require('./bower.json').appPath || 'app',
     dist: 'dist'
   };
 
-  try {
-    yeomanConfig.app = require('./bower.json').appPath || yeomanConfig.app;
-  } catch (e) {}
-
+  // Define the configuration for all the tasks
   grunt.initConfig({
-    yeoman: yeomanConfig,
+
+    // Project settings
+    yeoman: appConfig,
+
+    // Watches files for changes and runs tasks based on the changed files
     watch: {
-      coffee: {
-        files: ['<%= yeoman.app %>/scripts/{,*/}*.coffee'],
-        tasks: ['coffee:dist']
+      bower: {
+        files: ['bower.json'],
+        tasks: ['wiredep']
       },
-      coffeeTest: {
-        files: ['test/spec/{,*/}*.coffee'],
-        tasks: ['coffee:test']
+      js: {
+        files: ['<%= yeoman.app %>/scripts/{,*/}*.js'],
+        tasks: ['newer:jshint:all'],
+        options: {
+          livereload: '<%= connect.options.livereload %>'
+        }
+      },
+      jsTest: {
+        files: ['test/spec/{,*/}*.js'],
+        tasks: ['newer:jshint:test', 'karma']
       },
       styles: {
         files: ['<%= yeoman.app %>/styles/{,*/}*.css'],
-        tasks: ['copy:styles', 'autoprefixer']
+        tasks: ['newer:copy:styles', 'autoprefixer']
+      },
+      gruntfile: {
+        files: ['Gruntfile.js']
       },
       livereload: {
         options: {
-          livereload: LIVERELOAD_PORT
+          livereload: '<%= connect.options.livereload %>'
         },
         files: [
           '<%= yeoman.app %>/{,*/}*.html',
           '.tmp/styles/{,*/}*.css',
-          '{.tmp,<%= yeoman.app %>}/scripts/{,*/}*.js',
           '<%= yeoman.app %>/images/{,*/}*.{png,jpg,jpeg,gif,webp,svg}'
         ]
       }
     },
+
+    // The actual grunt server settings
+    connect: {
+      options: {
+        port: 9000,
+        // Change this to '0.0.0.0' to access the server from outside.
+        hostname: 'localhost',
+        livereload: 35729
+      },
+      livereload: {
+        options: {
+          open: true,
+          middleware: function (connect) {
+            return [
+              connect.static('.tmp'),
+              connect().use(
+                '/bower_components',
+                connect.static('./bower_components')
+              ),
+              connect.static(appConfig.app)
+            ];
+          }
+        }
+      },
+      test: {
+        options: {
+          port: 9001,
+          middleware: function (connect) {
+            return [
+              connect.static('.tmp'),
+              connect.static('test'),
+              connect().use(
+                '/bower_components',
+                connect.static('./bower_components')
+              ),
+              connect.static(appConfig.app)
+            ];
+          }
+        }
+      },
+      dist: {
+        options: {
+          open: true,
+          base: '<%= yeoman.dist %>'
+        }
+      }
+    },
+
+    // Make sure code styles are up to par and there are no obvious mistakes
+    jshint: {
+      options: {
+        jshintrc: '.jshintrc',
+        reporter: require('jshint-stylish')
+      },
+      all: {
+        src: [
+          'Gruntfile.js',
+          '<%= yeoman.app %>/scripts/{,*/}*.js'
+        ]
+      },
+      test: {
+        options: {
+          jshintrc: 'test/.jshintrc'
+        },
+        src: ['test/spec/{,*/}*.js']
+      }
+    },
+
+    // Empties folders to start fresh
+    clean: {
+      dist: {
+        files: [{
+          dot: true,
+          src: [
+            '.tmp',
+            '<%= yeoman.dist %>/{,*/}*',
+            '!<%= yeoman.dist %>/.git*'
+          ]
+        }]
+      },
+      server: '.tmp'
+    },
+
+    // Add vendor prefixed styles
     autoprefixer: {
-      options: ['last 1 version'],
+      options: {
+        browsers: ['last 1 version']
+      },
       dist: {
         files: [{
           expand: true,
@@ -64,134 +159,95 @@ module.exports = function (grunt) {
         }]
       }
     },
-    connect: {
+
+    // Automatically inject Bower components into the app
+    wiredep: {
       options: {
-        port: 9000,
-        // Change this to '0.0.0.0' to access the server from outside.
-        hostname: 'localhost'
+        cwd: '<%= yeoman.app %>'
       },
-      livereload: {
-        options: {
-          middleware: function (connect) {
-            return [
-              lrSnippet,
-              mountFolder(connect, '.tmp'),
-              mountFolder(connect, yeomanConfig.app)
-            ];
-          }
-        }
-      },
-      test: {
-        options: {
-          middleware: function (connect) {
-            return [
-              mountFolder(connect, '.tmp'),
-              mountFolder(connect, 'test')
-            ];
-          }
-        }
-      },
-      dist: {
-        options: {
-          middleware: function (connect) {
-            return [
-              mountFolder(connect, yeomanConfig.dist)
-            ];
-          }
-        }
+      app: {
+        src: ['<%= yeoman.app %>/index.html'],
+        ignorePath:  /\.\.\//
       }
     },
-    open: {
-      server: {
-        url: 'http://localhost:<%= connect.options.port %>'
-      }
-    },
-    clean: {
-      dist: {
-        files: [{
-          dot: true,
-          src: [
-            '.tmp',
-            '<%= yeoman.dist %>/*',
-            '!<%= yeoman.dist %>/.git*'
-          ]
-        }]
-      },
-      server: '.tmp'
-    },
-    jshint: {
-      options: {
-        jshintrc: '.jshintrc'
-      },
-      all: [
-        'Gruntfile.js',
-        '<%= yeoman.app %>/scripts/{,*/}*.js'
-      ]
-    },
-    coffee: {
-      options: {
-        sourceMap: true,
-        sourceRoot: ''
-      },
-      dist: {
-        files: [{
-          expand: true,
-          cwd: '<%= yeoman.app %>/scripts',
-          src: '{,*/}*.coffee',
-          dest: '.tmp/scripts',
-          ext: '.js'
-        }]
-      },
-      test: {
-        files: [{
-          expand: true,
-          cwd: 'test/spec',
-          src: '{,*/}*.coffee',
-          dest: '.tmp/spec',
-          ext: '.js'
-        }]
-      }
-    },
-    // not used since Uglify task does concat,
-    // but still available if needed
-    /*concat: {
-      dist: {}
-    },*/
-    rev: {
-      dist: {
-        files: {
-          src: [
-            '<%= yeoman.dist %>/scripts/{,*/}*.js',
-            '<%= yeoman.dist %>/styles/{,*/}*.css',
-            '<%= yeoman.dist %>/images/{,*/}*.{png,jpg,jpeg,gif,webp,svg}',
-            '<%= yeoman.dist %>/styles/fonts/*'
-          ]
-        }
+
+    // Renames files for browser caching purposes
+    filerev: {
+      dist: {
+        src: [
+          '<%= yeoman.dist %>/scripts/{,*/}*.js',
+          '<%= yeoman.dist %>/styles/{,*/}*.css',
+          '<%= yeoman.dist %>/images/{,*/}*.{png,jpg,jpeg,gif,webp,svg}',
+          '<%= yeoman.dist %>/styles/fonts/*'
+        ]
       }
     },
+
+    // Reads HTML for usemin blocks to enable smart builds that automatically
+    // concat, minify and revision files. Creates configurations in memory so
+    // additional tasks can operate on them
     useminPrepare: {
       html: ['<%= yeoman.app %>/index.html', '<%= yeoman.app %>/login.html'],
       options: {
-        dest: '<%= yeoman.dist %>'
+        dest: '<%= yeoman.dist %>',
+        flow: {
+          html: {
+            steps: {
+              js: ['concat', 'uglifyjs'],
+              css: ['cssmin']
+            },
+            post: {}
+          }
+        }
       }
     },
+
+    // Performs rewrites based on filerev and the useminPrepare configuration
     usemin: {
       html: ['<%= yeoman.dist %>/{,*/}*.html'],
       css: ['<%= yeoman.dist %>/styles/{,*/}*.css'],
       options: {
-        dirs: ['<%= yeoman.dist %>']
+        assetsDirs: ['<%= yeoman.dist %>','<%= yeoman.dist %>/images']
       }
     },
+
+    // The following *-min tasks will produce minified files in the dist folder
+    // By default, your `index.html`'s <!-- Usemin block --> will take care of
+    // minification. These next options are pre-configured if you do not wish
+    // to use the Usemin blocks.
+    // cssmin: {
+    //   dist: {
+    //     files: {
+    //       '<%= yeoman.dist %>/styles/main.css': [
+    //         '.tmp/styles/{,*/}*.css'
+    //       ]
+    //     }
+    //   }
+    // },
+    // uglify: {
+    //   dist: {
+    //     files: {
+    //       '<%= yeoman.dist %>/scripts/scripts.js': [
+    //         '<%= yeoman.dist %>/scripts/scripts.js'
+    //       ]
+    //     }
+    //   }
+    // },
+    // concat: {
+    //   dist: {}
+    // },
+
     imagemin: {
       dist: {
         files: [{
           expand: true,
           cwd: '<%= yeoman.app %>/images',
-          src: '{,*/}*.{png,jpg,jpeg}',
+          src: '{,*/}*.{png,jpg,jpeg,gif}',
           dest: '<%= yeoman.dist %>/images'
         }]
       }
     },
+
     svgmin: {
       dist: {
         files: [{
@@ -202,41 +258,47 @@ module.exports = function (grunt) {
         }]
       }
     },
-    cssmin: {
-      // By default, your `index.html` <!-- Usemin Block --> will take care of
-      // minification. This option is pre-configured if you do not wish to use
-      // Usemin blocks.
-      // dist: {
-      //   files: {
-      //     '<%= yeoman.dist %>/styles/main.css': [
-      //       '.tmp/styles/{,*/}*.css',
-      //       '<%= yeoman.app %>/styles/{,*/}*.css'
-      //     ]
-      //   }
-      // }
-    },
+
     htmlmin: {
       dist: {
         options: {
-          /*removeCommentsFromCDATA: true,
-          // https://github.com/yeoman/grunt-usemin/issues/44
-          //collapseWhitespace: true,
+          collapseWhitespace: true,
+          conservativeCollapse: true,
           collapseBooleanAttributes: true,
-          removeAttributeQuotes: true,
-          removeRedundantAttributes: true,
-          useShortDoctype: true,
-          removeEmptyAttributes: true,
-          removeOptionalTags: true*/
+          removeCommentsFromCDATA: true,
+          removeOptionalTags: true
         },
         files: [{
           expand: true,
-          cwd: '<%= yeoman.app %>',
-          src: ['*.html', 'views/**/*.html'],
+          cwd: '<%= yeoman.dist %>',
+          src: ['*.html', 'views/{,*/}*.html'],
           dest: '<%= yeoman.dist %>'
         }]
       }
     },
-    // Put files not handled in other tasks here
+
+    // ngmin tries to make the code safe for minification automatically by
+    // using the Angular long form for dependency injection. It doesn't work on
+    // things like resolve or inject so those have to be done manually.
+    ngmin: {
+      dist: {
+        files: [{
+          expand: true,
+          cwd: '.tmp/concat/scripts',
+          src: '*.js',
+          dest: '.tmp/concat/scripts'
+        }]
+      }
+    },
+
+    // Replace Google CDN references
+    cdnify: {
+      dist: {
+        html: ['<%= yeoman.dist %>/*.html']
+      }
+    },
+
+    // Copies remaining files to places other tasks can use
     copy: {
       dist: {
         files: [{
@@ -247,17 +309,21 @@ module.exports = function (grunt) {
           src: [
             '*.{ico,png,txt}',
             '.htaccess',
-            'bower_components/**/*',
-            'images/{,*/}*.{gif,webp}',
+            '*.html',
+            'views/{,*/}*.html',
+            'images/{,*/}*.{webp}',
             'fonts/*'
           ]
         }, {
           expand: true,
           cwd: '.tmp/images',
           dest: '<%= yeoman.dist %>/images',
-          src: [
-            'generated/*'
-          ]
+          src: ['generated/*']
+        }, {
+          expand: true,
+          cwd: 'bower_components/bootstrap/dist',
+          src: 'fonts/*',
+          dest: '<%= yeoman.dist %>'
         }]
       },
       styles: {
@@ -267,70 +333,52 @@ module.exports = function (grunt) {
         src: '{,*/}*.css'
       }
     },
+
+    // Run some tasks in parallel to speed up the build process
     concurrent: {
       server: [
-        'coffee:dist',
         'copy:styles'
       ],
       test: [
-        'coffee',
         'copy:styles'
       ],
       dist: [
-        'coffee',
         'copy:styles',
         'imagemin',
-        'svgmin',
-        'htmlmin'
+        'svgmin'
       ]
     },
+
+    // Test settings
     karma: {
       unit: {
-        configFile: 'karma.conf.js',
+        configFile: 'test/karma.conf.js',
         singleRun: true
       }
-    },
-    cdnify: {
-      dist: {
-        html: ['<%= yeoman.dist %>/*.html']
-      }
-    },
-    ngmin: {
-      dist: {
-        files: [{
-          expand: true,
-          cwd: '<%= yeoman.dist %>/scripts',
-          src: '*.js',
-          dest: '<%= yeoman.dist %>/scripts'
-        }]
-      }
-    },
-    uglify: {
-      dist: {
-        files: {
-          '<%= yeoman.dist %>/scripts/scripts.js': [
-            '<%= yeoman.dist %>/scripts/scripts.js'
-          ]
-        }
-      }
     }
   });
 
-  grunt.registerTask('server', function (target) {
+
+  grunt.registerTask('serve', 'Compile then start a connect web server', function (target) {
     if (target === 'dist') {
-      return grunt.task.run(['build', 'open', 'connect:dist:keepalive']);
+      return grunt.task.run(['build', 'connect:dist:keepalive']);
     }
 
     grunt.task.run([
       'clean:server',
+      'wiredep',
       'concurrent:server',
       'autoprefixer',
       'connect:livereload',
-      'open',
       'watch'
     ]);
   });
 
+  grunt.registerTask('server', 'DEPRECATED TASK. Use the "serve" task instead', function (target) {
+    grunt.log.warn('The `server` task has been deprecated. Use `grunt serve` to start a server.');
+    grunt.task.run(['serve:' + target]);
+  });
+
   grunt.registerTask('test', [
     'clean:server',
     'concurrent:test',
@@ -341,21 +389,23 @@ module.exports = function (grunt) {
 
   grunt.registerTask('build', [
     'clean:dist',
+    'wiredep',
     'useminPrepare',
     'concurrent:dist',
     'autoprefixer',
     'concat',
+    'ngmin',
     'copy:dist',
     'cdnify',
-    'ngmin',
     'cssmin',
     'uglify',
-    'rev',
-    'usemin'
+    'filerev',
+    'usemin',
+    'htmlmin'
   ]);
 
   grunt.registerTask('default', [
-    'jshint',
+    'newer:jshint',
     'test',
     'build'
   ]);

rd_ui/app/index.html

@@ -4,7 +4,7 @@
 <!--[if IE 8]>         <html class="no-js lt-ie9" ng-app="redash" ng-controller='MainCtrl'> <![endif]-->
 <!--[if gt IE 8]><!--> <html class="no-js" ng-app="redash" ng-controller='MainCtrl'> <!--<![endif]-->
 <head>
-    <title ng-bind="'re:dash | ' + pageTitle"></title>
+    <title ng-bind="'{{name}} | ' + pageTitle"></title>
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
 
@@ -12,8 +12,10 @@
     <link rel="stylesheet" href="/bower_components/bootstrap/dist/css/bootstrap.css">
     <link rel="stylesheet" href="/bower_components/codemirror/lib/codemirror.css">
     <link rel="stylesheet" href="/bower_components/gridster/dist/jquery.gridster.css">
-    <link rel="stylesheet" href="/bower_components/pivottable/examples/pivot.css">
+    <link rel="stylesheet" href="/bower_components/pivottable/dist/pivot.css">
     <link rel="stylesheet" href="/bower_components/cornelius/src/cornelius.css">
+    <link rel="stylesheet" href="/bower_components/select2/select2.css">
+    <link rel="stylesheet" href="/bower_components/pace/themes/pace-theme-minimal.css">
     <link rel="stylesheet" href="/styles/redash.css">
     <!-- endbuild -->
 </head>
@@ -29,13 +31,13 @@
             <span class="icon-bar"></span>
             <span class="icon-bar"></span>
         </button>
-        <a class="navbar-brand" href="/"><strong>re:dash</strong></a>
+        <a class="navbar-brand" href="/"><strong>{{name}}</strong></a>
     </div>
     {% raw %}
     <div class="collapse navbar-collapse navbar-ex1-collapse">
         <ul class="nav navbar-nav">
             <li class="active" ng-show="pageTitle"><a class="page-title" ng-bind="pageTitle"></a></li>
-            <li class="dropdown">
+            <li class="dropdown" ng-show="groupedDashboards.length > 0 || otherDashboards.length > 0 || currentUser.hasPermission('create_dashboard')">
                 <a href="#" class="dropdown-toggle" data-toggle="dropdown"><span class="glyphicon glyphicon-th-large"></span> <b class="caret"></b></a>
                 <ul class="dropdown-menu">
                     <span ng-repeat="(name, group) in groupedDashboards">
@@ -51,21 +53,24 @@
                     <li ng-repeat="dashboard in otherDashboards">
                         <a role="menu-item" ng-href="/dashboard/{{dashboard.slug}}" ng-bind="dashboard.name"></a>
                     </li>
-                    <li class="divider"></li>
-                    <li><a data-toggle="modal" href="#new_dashboard_dialog">New Dashboard</a></li>
+                    <li class="divider" ng-show="currentUser.hasPermission('create_dashboard') && (groupedDashboards.length > 0 || otherDashboards.length > 0)"></li>
+                    <li><a data-toggle="modal" href="#new_dashboard_dialog" ng-show="currentUser.hasPermission('create_dashboard')">New Dashboard</a></li>
                 </ul>
             </li>
-            <li class="dropdown">
+            <li class="dropdown" ng-show="currentUser.hasPermission('view_query')">
                 <a href="#" class="dropdown-toggle" data-toggle="dropdown">Queries <b class="caret"></b></a>
                 <ul class="dropdown-menu">
-                    <li><a href="/queries/new">New Query</a></li>
+                    <li ng-show="currentUser.hasPermission('create_query')"><a href="/queries/new">New Query</a></li>
                     <li><a href="/queries">Queries</a></li>
                 </ul>
             </li>
         </ul>
         <ul class="nav navbar-nav navbar-right">
-            <p class="navbar-text avatar">
-                <img ng-src="{{currentUser.gravatar_url}}" class="img-circle" alt="{{currentUser.name}}" width="40" height="40"/>
+            <p class="navbar-text avatar" ng-show="currentUser.id" ng-cloak>
+                <img ng-src="{{currentUser.gravatar_url}}" class="img-circle" alt="{{currentUser.name}}"/>
+                <a target="_self" href="/logout" id="logout" title="Logout">
+                    <span class="glyphicon glyphicon-log-out"></span>
+                </a>
             </p>
         </ul>
     </div>
@@ -99,35 +104,46 @@
 <script src="/bower_components/highcharts/modules/exporting.js"></script>
 <script src="/bower_components/gridster/dist/jquery.gridster.js"></script>
 <script src="/bower_components/angular-growl/build/angular-growl.js"></script>
-<script src="/bower_components/pivottable/examples/pivot.js"></script>
+<script src="/bower_components/pivottable/dist/pivot.js"></script>
 <script src="/bower_components/cornelius/src/cornelius.js"></script>
 <script src="/bower_components/mousetrap/mousetrap.js"></script>
 <script src="/bower_components/mousetrap/plugins/global-bind/mousetrap-global-bind.js"></script>
-
+<script src="/bower_components/select2/select2.js"></script>
+<script src="/bower_components/angular-ui-select2/src/select2.js"></script>
+<script src="/bower_components/underscore.string/lib/underscore.string.js"></script>
+<script src="/bower_components/marked/lib/marked.js"></script>
 <script src="/scripts/ng_highchart.js"></script>
 <script src="/scripts/ng_smart_table.js"></script>
 <script src="/scripts/ui-bootstrap-tpls-0.5.0.min.js"></script>
-
+<script src="/bower_components/bucky/bucky.js"></script>
+<script src="/bower_components/pace/pace.js"></script>
 <!-- endbuild -->
 
 <!-- build:js({.tmp,app}) /scripts/scripts.js -->
 <script src="/scripts/app.js"></script>
 <script src="/scripts/services/services.js"></script>
+<script src="/scripts/services/resources.js"></script>
 <script src="/scripts/services/notifications.js"></script>
 <script src="/scripts/services/dashboards.js"></script>
 <script src="/scripts/controllers/controllers.js"></script>
+<script src="/scripts/controllers/dashboard.js"></script>
 <script src="/scripts/controllers/admin_controllers.js"></script>
 <script src="/scripts/controllers/query_view.js"></script>
+<script src="/scripts/controllers/query_source.js"></script>
 <script src="/scripts/visualizations/base.js"></script>
 <script src="/scripts/visualizations/chart.js"></script>
 <script src="/scripts/visualizations/cohort.js"></script>
 <script src="/scripts/visualizations/table.js"></script>
 <script src="/scripts/visualizations/pivot.js"></script>
-<script src="/scripts/directives.js"></script>
+<script src="/scripts/directives/directives.js"></script>
+<script src="/scripts/directives/query_directives.js"></script>
+<script src="/scripts/directives/dashboard_directives.js"></script>
 <script src="/scripts/filters.js"></script>
 <!-- endbuild -->
 
 <script>
+    // TODO: move currentUser & features to be an Angular service
+    var featureFlags = {{ features|safe }};
     var currentUser = {{ user|safe }};
 
     currentUser.canEdit = function(object) {
@@ -135,8 +151,12 @@
         return user_id && (user_id == currentUser.id);
     };
 
+    currentUser.hasPermission = function(permission) {
+        return this.permissions.indexOf(permission) != -1;
+    }
+
     {{ analytics|safe }}
 </script>
 
 </body>
-</html>
+</html>

rd_ui/app/login.html

@@ -4,7 +4,7 @@
 <!--[if IE 8]>         <html class="no-js lt-ie9"> <![endif]-->
 <!--[if gt IE 8]><!--> <html class="no-js"> <!--<![endif]-->
 <head>
-    <title>re:dash Login</title>
+    <title>{{name}} Login</title>
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
 
@@ -26,7 +26,7 @@
             <span class="icon-bar"></span>
             <span class="icon-bar"></span>
         </button>
-        <a class="navbar-brand" href="/"><strong>re:dash</strong></a>
+        <a class="navbar-brand" href="/"><strong>{{name}}</strong></a>
     </div>
     </div>
 </nav>

rd_ui/app/scripts/app.js

@@ -8,91 +8,86 @@ angular.module('redash', [
     'redash.visualization',
     'ui.codemirror',
     'highchart',
+    'ui.select2',
     'angular-growl',
     'angularMoment',
     'ui.bootstrap',
     'smartTable.table',
     'ngResource',
     'ngRoute'
-]).config(['$routeProvider', '$locationProvider', '$compileProvider', 'growlProvider',
-    function($routeProvider, $locationProvider, $compileProvider, growlProvider) {
+  ]).config(['$routeProvider', '$locationProvider', '$compileProvider', 'growlProvider',
+    function ($routeProvider, $locationProvider, $compileProvider, growlProvider) {
+      if (featureFlags.clientSideMetrics) {
+        Bucky.setOptions({
+          host: '/api/metrics'
+        });
 
-        function getQuery(Query, $q, $route) {
-            var defer = $q.defer();
+        Bucky.requests.monitor('ajax_requsts');
+        Bucky.requests.transforms.enable('dashboards', /dashboard\/[\w-]+/ig, '/dashboard');
+      }
 
-            Query.get({
-                'id': $route.current.params.queryId
-            }, function(query) {
-                defer.resolve(query);
-            });
+      function getQuery(Query, $route) {
+        var query = Query.get({'id': $route.current.params.queryId });
+        return query.$promise;
+      };
 
-            return defer.promise;
+      $compileProvider.aHrefSanitizationWhitelist(/^\s*(https?|http|data):/);
+      $locationProvider.html5Mode(true);
+      growlProvider.globalTimeToLive(2000);
+
+      $routeProvider.when('/dashboard/:dashboardSlug', {
+        templateUrl: '/views/dashboard.html',
+        controller: 'DashboardCtrl',
+        reloadOnSearch: false
+      });
+      $routeProvider.when('/queries', {
+        templateUrl: '/views/queries.html',
+        controller: 'QueriesCtrl',
+        reloadOnSearch: false
+      });
+      $routeProvider.when('/queries/new', {
+        templateUrl: '/views/query.html',
+        controller: 'QuerySourceCtrl',
+        reloadOnSearch: false,
+        resolve: {
+          'query': ['Query', function newQuery(Query) {
+            return Query.newQuery();
+          }]
         }
+      });
+      $routeProvider.when('/queries/:queryId', {
+        templateUrl: '/views/query.html',
+        controller: 'QueryViewCtrl',
+        reloadOnSearch: false,
+        resolve: {
+          'query': ['Query', '$route', getQuery]
+        }
+      });
+      $routeProvider.when('/queries/:queryId/source', {
+        templateUrl: '/views/query.html',
+        controller: 'QuerySourceCtrl',
+        reloadOnSearch: false,
+        resolve: {
+          'query': ['Query', '$route', getQuery]
+        }
+      });
+      $routeProvider.when('/admin/status', {
+        templateUrl: '/views/admin_status.html',
+        controller: 'AdminStatusCtrl'
+      });
+      $routeProvider.when('/admin/workers', {
+        templateUrl: '/views/admin_workers.html',
+        controller: 'AdminWorkersCtrl'
+      });
 
-        $compileProvider.aHrefSanitizationWhitelist(/^\s*(https?|http|data):/);
-        $locationProvider.html5Mode(true);
-        growlProvider.globalTimeToLive(2000);
+      $routeProvider.when('/', {
+        templateUrl: '/views/index.html',
+        controller: 'IndexCtrl'
+      });
+      $routeProvider.otherwise({
+        redirectTo: '/'
+      });
 
-        $routeProvider.when('/dashboard/:dashboardSlug', {
-            templateUrl: '/views/dashboard.html',
-            controller: 'DashboardCtrl'
-        });
-        $routeProvider.when('/queries', {
-            templateUrl: '/views/queries.html',
-            controller: 'QueriesCtrl',
-            reloadOnSearch: false
-        });
-        $routeProvider.when('/queries/new', {
-            templateUrl: '/views/queryview.html',
-            controller: 'QueryViewCtrl',
-            reloadOnSearch: false,
-            resolve: {
-                'viewSource': function isViewSource() {
-                    return true;
-                }
-            }
-        });
-        // TODO
-        // we should have 2 controllers: queryViewCtrl and queryEditCtrl
-        $routeProvider.when('/queries/:queryId', {
-            templateUrl: '/views/queryview.html',
-            controller: 'QueryViewCtrl',
-            reloadOnSearch: false,
-            resolve: {
-                'query': getQuery
-            }
-        });
-        $routeProvider.when('/queries/:queryId/fiddle', {
-            templateUrl: '/views/queryfiddle.html',
-            controller: 'QueryFiddleCtrl',
-            reloadOnSearch: false
-        });
-        $routeProvider.when('/queries/:queryId/source', {
-            templateUrl: '/views/queryview.html',
-            controller: 'QueryViewCtrl',
-            reloadOnSearch: false,
-            resolve: {
-                'query': getQuery,
-                'viewSource': function isViewSource() {
-                    return true;
-                }
-            }
-        });
-        $routeProvider.when('/admin/status', {
-            templateUrl: '/views/admin_status.html',
-            controller: 'AdminStatusCtrl'
-        });
-        $routeProvider.when('/', {
-            templateUrl: '/views/index.html',
-            controller: 'IndexCtrl'
-        });
-        $routeProvider.otherwise({
-            redirectTo: '/'
-        });
 
-        Highcharts.setOptions({
-            colors: ["#4572A7", "#AA4643", "#89A54E", "#80699B", "#3D96AE",
-                        "#DB843D", "#92A8CD", "#A47D7C", "#B5CA92"]
-        });
     }
-]);
+  ]);

rd_ui/app/scripts/controllers/admin_controllers.js

@@ -1,23 +1,31 @@
 (function () {
-    var AdminStatusCtrl = function ($scope, $http, $timeout) {
-        $scope.$parent.pageTitle = "System Status";
+  var AdminStatusCtrl = function ($scope, Events, $http, $timeout) {
+    Events.record(currentUser, "view", "page", "admin/status");
+    $scope.$parent.pageTitle = "System Status";
 
-        var refresh = function () {
-            $scope.refresh_time = moment().add('minutes', 1);
-            $http.get('/status.json').success(function (data) {
-                $scope.workers = data.workers;
-                delete data.workers;
-                $scope.manager = data.manager;
-                delete data.manager;
-                $scope.status = data;
-            });
+    var refresh = function () {
+      $scope.refresh_time = moment().add('minutes', 1);
+      $http.get('/status.json').success(function (data) {
+        $scope.workers = data.workers;
+        delete data.workers;
+        $scope.manager = data.manager;
+        delete data.manager;
+        $scope.status = data;
+      });
 
-            $timeout(refresh, 59 * 1000);
-        };
+      $timeout(refresh, 59 * 1000);
+    };
 
-        refresh();
-    }
+    $scope.flowerUrl = featureFlags.flowerUrl;
 
-    angular.module('redash.admin_controllers', [])
-        .controller('AdminStatusCtrl', ['$scope', '$http', '$timeout', AdminStatusCtrl])
+    refresh();
+  }
+
+  var AdminWorkersCtrl = function ($scope, $sce) {
+    $scope.flowerUrl = $sce.trustAsResourceUrl(featureFlags.flowerUrl);
+  };
+
+  angular.module('redash.admin_controllers', [])
+         .controller('AdminStatusCtrl', ['$scope', 'Events', '$http', '$timeout', AdminStatusCtrl])
+         .controller('AdminWorkersCtrl', ['$scope', '$sce', AdminWorkersCtrl])
 })();

rd_ui/app/scripts/controllers/controllers.js

@@ -1,446 +1,169 @@
 (function () {
-    var DashboardCtrl = function ($scope, $routeParams, $http, $timeout, Dashboard) {
-        $scope.refreshEnabled = false;
-        $scope.refreshRate = 60;
-        $scope.dashboard = Dashboard.get({slug: $routeParams.dashboardSlug}, function(dashboard) {
-            $scope.$parent.pageTitle = dashboard.name;
-        });
-
-        var autoRefresh = function() {
-            if ($scope.refreshEnabled) {
-                $timeout(function() {
-                    Dashboard.get({slug: $routeParams.dashboardSlug}, function(dashboard) {
-                        var newWidgets = _.groupBy(_.flatten(dashboard.widgets), 'id');
-
-                        _.each($scope.dashboard.widgets, function(row) {
-                            _.each(row, function(widget, i) {
-                                var newWidget = newWidgets[widget.id];
-                                if (newWidget && newWidget[0].visualization.query.latest_query_data_id != widget.visualization.query.latest_query_data_id ) {
-                                    row[i] = newWidget[0];
-                                }
-                            });
-                        });
-
-                        autoRefresh();
-                    });
-
-                }, $scope.refreshRate);
-            };
-        }
-
-        $scope.triggerRefresh = function(){
-            $scope.refreshEnabled = !$scope.refreshEnabled;
-
-            if ($scope.refreshEnabled) {
-                var refreshRate = _.min(_.flatten($scope.dashboard.widgets), function(widget) {
-                    return widget.visualization.query.ttl;
-                }).visualization.query.ttl;
-
-                $scope.refreshRate = _.max([120, refreshRate * 2])*1000;
-
-                autoRefresh();
-            }
-        };
-    };
-
-    var WidgetCtrl = function ($scope, $http, $location, Query) {
-        $scope.deleteWidget = function() {
-            if (!confirm('Are you sure you want to remove "' + $scope.widget.visualization.name + '" from the dashboard?')) {
-                return;
-            }
-
-            $http.delete('/api/widgets/' + $scope.widget.id).success(function() {
-                $scope.dashboard.widgets = _.map($scope.dashboard.widgets, function(row) {
-                    return _.filter(row, function(widget) {
-                        return widget.id != $scope.widget.id;
-                    })
-                });
-            });
-        };
-
-        $scope.open = function(query, visualization) {
-            $location.path('/queries/' + query.id);
-            $location.hash(visualization.id);
-        }
-
-        $scope.query = new Query($scope.widget.visualization.query);
-        $scope.queryResult = $scope.query.getQueryResult();
-
-        $scope.updateTime = (new Date($scope.queryResult.getUpdatedAt())).toISOString();
-        $scope.nextUpdateTime = moment(new Date(($scope.query.updated_at + $scope.query.ttl + $scope.query.runtime + 300) * 1000)).fromNow();
-
-        $scope.updateTime = '';
+  var QueriesCtrl = function ($scope, $http, $location, $filter, Query) {
+    $scope.$parent.pageTitle = "All Queries";
+    $scope.gridConfig = {
+      isPaginationEnabled: true,
+      itemsByPage: 50,
+      maxSize: 8,
+      isGlobalSearchActivated: true
     }
 
-    var QueryFiddleCtrl = function ($scope, $window, $location, $routeParams, $http, $location, growl, notifications, Query, Visualization) {
-        var DEFAULT_TAB = 'table';
-        var pristineHash = null;
-        var leavingPageText = "You will lose your changes if you leave";
+    $scope.allQueries = [];
+    $scope.queries = [];
 
-        $scope.dirty = undefined;
-        $scope.newVisualization = undefined;
-
-        $window.onbeforeunload = function(){
-            if (currentUser.canEdit($scope.query) && $scope.dirty) {
-                return leavingPageText;
-            }
-        }
-
-        Mousetrap.bindGlobal("meta+s", function(e) {
-            e.preventDefault();
-
-            if (currentUser.canEdit($scope.query)) {
-                $scope.saveQuery();
-            }
-        });
-
-        $scope.$on('$locationChangeStart', function(event, next, current) {
-            if (next.split("#")[0] == current.split("#")[0]) {
-                return;
-            }
-
-            if (!currentUser.canEdit($scope.query)) {
-                return;
-            }
-
-            if($scope.dirty &&
-                !confirm(leavingPageText + "\n\nAre you sure you want to leave this page?")) {
-                event.preventDefault();
-            } else {
-                Mousetrap.unbind("meta+s");
-            }
-        });
-
-        $scope.$parent.pageTitle = "Query Fiddle";
-
-        $scope.$watch(function() {return $location.hash()}, function(hash) {
-            $scope.selectedTab = hash || DEFAULT_TAB;
-        });
-
-        $scope.lockButton = function (lock) {
-            $scope.queryExecuting = lock;
-        };
-
-        $scope.formatQuery = function() {
-            $scope.editorOptions.readOnly = 'nocursor';
-
-            $http.post('/api/queries/format', {'query': $scope.query.query}).success(function(response) {
-                $scope.query.query = response;
-                $scope.editorOptions.readOnly = false;
-            })
-        }
-
-        $scope.saveQuery = function (duplicate, oldId) {
-            if (!oldId) {
-                oldId = $scope.query.id;
-            }
-            
-            delete $scope.query.latest_query_data;
-            $scope.query.$save(function (q) {
-                pristineHash = q.getHash();
-                $scope.dirty = false;
-
-                if (duplicate) {
-                    growl.addInfoMessage("Query duplicated.", {ttl: 2000});
-                } else{
-                    growl.addSuccessMessage("Query saved.", {ttl: 2000});
-                }
-
-                if (oldId != q.id) {
-                    if (oldId == undefined) {
-                        $location.path($location.path().replace('new', q.id)).replace();
-                    } else {
-                        // TODO: replace this with a safer method
-                        $location.path($location.path().replace(oldId, q.id)).replace();
-
-                        // Reset visualizations tab to table after duplicating a query:
-                        $location.hash('table');
-                    }
-                }
-            }, function(httpResponse) {
-                growl.addErrorMessage("Query could not be saved");
-            });
-        };
-
-        $scope.duplicateQuery = function () {
-            var oldId = $scope.query.id;
-            $scope.query.id = null;
-            $scope.query.ttl = -1;
-
-            $scope.saveQuery(true, oldId);
-        };
-
-        // Query Editor:
-        $scope.editorOptions = {
-            mode: 'text/x-sql',
-            lineWrapping: true,
-            lineNumbers: true,
-            readOnly: false,
-            matchBrackets: true,
-            autoCloseBrackets: true
-        };
-
-        $scope.refreshOptions = [
-            {value: -1, name: 'No Refresh'},
-            {value: 60, name: 'Every minute'},
-        ]
-
-        _.each(_.range(1, 13), function(i) {
-            $scope.refreshOptions.push({value: i*3600, name: 'Every ' + i + 'h'});
-        })
-
-        $scope.refreshOptions.push({value: 24*3600, name: 'Every 24h'});
-        $scope.refreshOptions.push({value: 7*24*3600, name: 'Once a week'});
-
-        $scope.$watch('queryResult && queryResult.getError()', function (newError, oldError) {
-            if (newError == undefined) {
-                return;
-            }
-
-            if (oldError == undefined && newError != undefined) {
-                $scope.lockButton(false);
-            }
-        });
-
-        $scope.$watch('queryResult && queryResult.getData()', function (data, oldData) {
-            if (!data) {
-                return;
-            }
-
-            if ($scope.queryResult.getId() == null) {
-                $scope.dataUri = "";
-            } else {
-                $scope.dataUri = '/api/queries/' + $scope.query.id + '/results/' + $scope.queryResult.getId() + '.csv';
-                $scope.dataFilename = $scope.query.name.replace(" ", "_") + moment($scope.queryResult.getUpdatedAt()).format("_YYYY_MM_DD") + ".csv";
-            }
-        });
-
-        $scope.$watch("queryResult && queryResult.getStatus()", function (status) {
-            if (!status) {
-                return;
-            }
-
-            if (status == "done") {
-                if ($scope.query.id && $scope.query.latest_query_data_id != $scope.queryResult.getId() &&
-                    $scope.query.query_hash == $scope.queryResult.query_result.query_hash) {
-                    Query.save({'id': $scope.query.id, 'latest_query_data_id': $scope.queryResult.getId()})
-                }
-                $scope.query.latest_query_data_id = $scope.queryResult.getId();
-
-                notifications.showNotification("re:dash", $scope.query.name + " updated.");
-
-                $scope.lockButton(false);
-            }
-        });
-
-        if ($routeParams.queryId != undefined) {
-            $scope.query = Query.get({id: $routeParams.queryId}, function(q) {
-                pristineHash = q.getHash();
-                $scope.dirty = false;
-                $scope.queryResult = $scope.query.getQueryResult();
-            });
-        } else {
-            $scope.query = new Query({query: "", name: "New Query", ttl: -1, user: currentUser});
-            $scope.lockButton(false);
-        }
-
-        $scope.$watch('query.name', function() {
-            $scope.$parent.pageTitle = $scope.query.name;
-        });
-
-        $scope.$watch(function() {
-            return $scope.query.getHash();
-        }, function(newHash) {
-            $scope.dirty = (newHash !== pristineHash);
-        });
-
-        $scope.executeQuery = function() {
-            $scope.queryResult = $scope.query.getQueryResult(0);
-            $scope.lockButton(true);
-            $scope.cancelling = false;
-        };
-
-        $scope.cancelExecution = function() {
-            $scope.cancelling = true;
-            $scope.queryResult.cancelExecution();
-        };
-
-        $scope.deleteVisualization = function($e, vis) {
-            $e.preventDefault();
-            if (confirm('Are you sure you want to delete ' + vis.name + ' ?')) {
-                Visualization.delete(vis);
-                if ($scope.selectedTab == vis.id) {
-                    $scope.selectedTab = DEFAULT_TAB;
-                }
-                $scope.query.visualizations =
-                    $scope.query.visualizations.filter(function(v) {
-                        return vis.id !== v.id;
-                    });
-            }
-        };
-        
-        unbind = $scope.$watch('selectedTab == "add"', function(newPanel) {
-            if (newPanel && $routeParams.queryId == undefined) {
-                unbind();
-                $scope.saveQuery();
-            }
-        });
+    var dateFormatter = function (value) {
+      if (!value) return "-";
+      return value.format("DD/MM/YY HH:mm");
     }
 
-    var QueriesCtrl = function($scope, $http, $location, $filter, Query) {
-        $scope.$parent.pageTitle = "All Queries";
-        $scope.gridConfig = {
-            isPaginationEnabled: true,
-            itemsByPage: 50,
-            maxSize: 8,
-            isGlobalSearchActivated: true
+    var filterQueries = function () {
+      $scope.queries = _.filter($scope.allQueries, function (query) {
+        if (!$scope.selectedTab) {
+          return false;
         }
 
-        $scope.allQueries = [];
-        $scope.queries = [];
-
-        var dateFormatter = function (value) {
-            if (!value) return "-";
-            return value.format("DD/MM/YY HH:mm");
+        if ($scope.selectedTab.key == 'my') {
+          return query.user.id == currentUser.id && query.name != 'New Query';
+        } else if ($scope.selectedTab.key == 'drafts') {
+          return query.user.id == currentUser.id && query.name == 'New Query';
         }
 
-        var filterQueries = function() {
-            $scope.queries = _.filter($scope.allQueries, function(query) {
-                if (!$scope.selectedTab) {
-                    return false;
-                }
-
-                if ($scope.selectedTab.key == 'my') {
-                    return query.user.id == currentUser.id && query.name != 'New Query';
-                } else if ($scope.selectedTab.key == 'drafts') {
-                    return query.user.id == currentUser.id && query.name == 'New Query';
-                }
-
-                return query.name != 'New Query';
-            });
-        }
-
-        Query.query(function(queries) {
-            $scope.allQueries = _.map(queries, function(query) {
-                query.created_at = moment(query.created_at);
-                query.last_retrieved_at = moment(query.last_retrieved_at);
-                return query;
-            });
-
-            filterQueries();
-        });
-
-        $scope.gridColumns = [
-            {
-                "label": "Name",
-                "map": "name",
-                "cellTemplateUrl": "/views/queries_query_name_cell.html"
-            },
-            {
-                'label': 'Created By',
-                'map': 'user.name'
-            },
-            {
-                'label': 'Created At',
-                'map': 'created_at',
-                'formatFunction': dateFormatter
-            },
-            {
-                'label': 'Runtime (avg)',
-                'map': 'avg_runtime',
-                'formatFunction': function(value) {
-                    return $filter('durationHumanize')(value);
-                }
-            },
-            {
-                'label': 'Runtime (min)',
-                'map': 'min_runtime',
-                'formatFunction': function(value) {
-                    return $filter('durationHumanize')(value);
-                }
-            },
-            {
-                'label': 'Runtime (max)',
-                'map': 'max_runtime',
-                'formatFunction': function(value) {
-                    return $filter('durationHumanize')(value);
-                }
-            },
-            {
-                'label': 'Last Executed At',
-                'map': 'last_retrieved_at',
-                'formatFunction': dateFormatter
-            },
-            {
-                'label': 'Times Executed',
-                'map': 'times_retrieved'
-            },
-            {
-                'label': 'Update Schedule',
-                'map': 'ttl',
-                'formatFunction': function(value) {
-                    return $filter('refreshRateHumanize')(value);
-                }
-            }
-        ]
-        $scope.tabs = [{"name": "My Queries", "key": "my"}, {"key": "all", "name": "All Queries"}, {"key": "drafts", "name": "Drafts"}];
-
-        $scope.$watch('selectedTab', function(tab) {
-            if (tab) {
-                $scope.$parent.pageTitle =  tab.name;
-            }
-
-            filterQueries();
-        });
+        return query.name != 'New Query';
+      });
     }
 
-    var MainCtrl = function ($scope, Dashboard, notifications) {
-        $scope.dashboards = [];
-        $scope.reloadDashboards = function() {
-            Dashboard.query(function (dashboards) {
-                $scope.dashboards = _.sortBy(dashboards, "name");
-                $scope.allDashboards = _.groupBy($scope.dashboards, function(d) {
-                    parts = d.name.split(":");
-                    if (parts.length == 1) {
-                        return "Other";
-                    }
-                    return parts[0];
-                });
-                $scope.otherDashboards = $scope.allDashboards['Other'] || [];
-                $scope.groupedDashboards = _.omit($scope.allDashboards, 'Other');
-            });
+    Query.query(function (queries) {
+      $scope.allQueries = _.map(queries, function (query) {
+        query.created_at = moment(query.created_at);
+        query.last_retrieved_at = moment(query.last_retrieved_at);
+        return query;
+      });
+
+      filterQueries();
+    });
+
+    $scope.gridColumns = [
+      {
+        "label": "Name",
+        "map": "name",
+        "cellTemplateUrl": "/views/queries_query_name_cell.html"
+      },
+      {
+        'label': 'Created By',
+        'map': 'user.name'
+      },
+      {
+        'label': 'Created At',
+        'map': 'created_at',
+        'formatFunction': dateFormatter
+      },
+      {
+        'label': 'Runtime (avg)',
+        'map': 'avg_runtime',
+        'formatFunction': function (value) {
+          return $filter('durationHumanize')(value);
         }
-
-        $scope.reloadDashboards();
-
-        $scope.currentUser = currentUser;
-        $scope.newDashboard = {
-            'name': null,
-            'layout': null
+      },
+      {
+        'label': 'Runtime (min)',
+        'map': 'min_runtime',
+        'formatFunction': function (value) {
+          return $filter('durationHumanize')(value);
         }
+      },
+      {
+        'label': 'Runtime (max)',
+        'map': 'max_runtime',
+        'formatFunction': function (value) {
+          return $filter('durationHumanize')(value);
+        }
+      },
+      {
+        'label': 'Last Executed At',
+        'map': 'last_retrieved_at',
+        'formatFunction': dateFormatter
+      },
+      {
+        'label': 'Times Executed',
+        'map': 'times_retrieved'
+      },
+      {
+        'label': 'Update Schedule',
+        'map': 'ttl',
+        'formatFunction': function (value) {
+          return $filter('refreshRateHumanize')(value);
+        }
+      }
+    ]
+    $scope.tabs = [
+      {"name": "My Queries", "key": "my"},
+      {"key": "all", "name": "All Queries"},
+      {"key": "drafts", "name": "Drafts"}
+    ];
 
-        $(window).click(function () {
-            notifications.getPermissions();
+    $scope.$watch('selectedTab', function (tab) {
+      if (tab) {
+        $scope.$parent.pageTitle = tab.name;
+      }
+
+      filterQueries();
+    });
+  }
+
+  var MainCtrl = function ($scope, Dashboard, notifications) {
+    if (featureFlags.clientSideMetrics) {
+      $scope.$on('$locationChangeSuccess', function(event, newLocation, oldLocation) {
+        // This will be called once per actual page load.
+        Bucky.sendPagePerformance();
+      });
+    }
+
+
+    $scope.dashboards = [];
+    $scope.reloadDashboards = function () {
+      Dashboard.query(function (dashboards) {
+        $scope.dashboards = _.sortBy(dashboards, "name");
+        $scope.allDashboards = _.groupBy($scope.dashboards, function (d) {
+          parts = d.name.split(":");
+          if (parts.length == 1) {
+            return "Other";
+          }
+          return parts[0];
         });
+        $scope.otherDashboards = $scope.allDashboards['Other'] || [];
+        $scope.groupedDashboards = _.omit($scope.allDashboards, 'Other');
+      });
     }
 
-    var IndexCtrl = function($scope, Dashboard) {
-        $scope.$parent.pageTitle = "Home";
+    $scope.reloadDashboards();
 
-        $scope.archiveDashboard = function(dashboard) {
-            if (confirm('Are you sure you want to delete "' + dashboard.name + '" dashboard?')) {
-                dashboard.$delete(function() {
-                    $scope.$parent.reloadDashboards();
-                });
-            }
-        }
+    $scope.currentUser = currentUser;
+    $scope.newDashboard = {
+      'name': null,
+      'layout': null
     }
 
-    angular.module('redash.controllers', [])
-        .controller('DashboardCtrl', ['$scope', '$routeParams', '$http', '$timeout', 'Dashboard', DashboardCtrl])
-        .controller('WidgetCtrl', ['$scope', '$http', '$location', 'Query', WidgetCtrl])
-        .controller('QueriesCtrl', ['$scope', '$http', '$location', '$filter', 'Query', QueriesCtrl])
-        .controller('QueryFiddleCtrl', ['$scope', '$window', '$location', '$routeParams', '$http', '$location', 'growl', 'notifications', 'Query', 'Visualization', QueryFiddleCtrl])
-        .controller('IndexCtrl', ['$scope', 'Dashboard', IndexCtrl])
-        .controller('MainCtrl', ['$scope', 'Dashboard', 'notifications', MainCtrl]);
+    $(window).click(function () {
+      notifications.getPermissions();
+    });
+  }
+
+  var IndexCtrl = function ($scope, Events, Dashboard) {
+    Events.record(currentUser, "view", "page", "homepage");
+    $scope.$parent.pageTitle = "Home";
+
+    $scope.archiveDashboard = function (dashboard) {
+      if (confirm('Are you sure you want to delete "' + dashboard.name + '" dashboard?')) {
+        Events.record(currentUser, "archive", "dashboard", dashboard.id);
+        dashboard.$delete(function () {
+          $scope.$parent.reloadDashboards();
+        });
+      }
+    }
+  }
+
+  angular.module('redash.controllers', [])
+    .controller('QueriesCtrl', ['$scope', '$http', '$location', '$filter', 'Query', QueriesCtrl])
+    .controller('IndexCtrl', ['$scope', 'Events', 'Dashboard', IndexCtrl])
+    .controller('MainCtrl', ['$scope', 'Dashboard', 'notifications', MainCtrl]);
 })();

rd_ui/app/scripts/controllers/dashboard.js

@@ -0,0 +1,143 @@
+(function() {
+  var DashboardCtrl = function($scope, Events, Widget, $routeParams, $http, $timeout, $q, Dashboard) {
+    $scope.refreshEnabled = false;
+    $scope.refreshRate = 60;
+
+    var loadDashboard = _.throttle(function() {
+      $scope.dashboard = Dashboard.get({ slug: $routeParams.dashboardSlug }, function (dashboard) {
+        Events.record(currentUser, "view", "dashboard", dashboard.id);
+
+        $scope.$parent.pageTitle = dashboard.name;
+
+        var promises = [];
+
+        $scope.dashboard.widgets = _.map($scope.dashboard.widgets, function (row) {
+          return _.map(row, function (widget) {
+            var w = new Widget(widget);
+
+            if (w.visualization && dashboard.dashboard_filters_enabled) {
+              promises.push(w.getQuery().getQueryResultPromise());
+            }
+
+            return w;
+          });
+        });
+
+        $q.all(promises).then(function(queryResults) {
+          var filters = {};
+          _.each(queryResults, function(queryResult) {
+            var queryFilters = queryResult.getFilters();
+            _.each(queryFilters, function (filter) {
+              if (!_.has(filters, filter.name)) {
+                // TODO: first object should be a copy, otherwise one of the chart filters behaves different than the others.
+                filters[filter.name] = filter;
+                filters[filter.name].originFilters = [];
+
+                $scope.$watch(function () { return filter.current }, function (value) {
+                  _.each(filter.originFilters, function (originFilter) {
+                    originFilter.current = value;
+                  });
+                });
+              };
+
+              // TODO: merge values.
+              filters[filter.name].originFilters.push(filter);
+            });
+          });
+
+          if (dashboard.dashboard_filters_enabled) {
+            $scope.filters = _.values(filters);
+          }
+        });
+
+
+      }, function () {
+        // error...
+        // try again. we wrap loadDashboard with throttle so it doesn't happen too often.\
+        // we might want to consider exponential backoff and also move this as a general solution in $http/$resource for
+        // all AJAX calls.
+        loadDashboard();
+      });
+    }, 1000);
+
+    loadDashboard();
+
+    var autoRefresh = function() {
+      if ($scope.refreshEnabled) {
+        $timeout(function() {
+          Dashboard.get({
+            slug: $routeParams.dashboardSlug
+          }, function(dashboard) {
+            var newWidgets = _.groupBy(_.flatten(dashboard.widgets), 'id');
+
+            _.each($scope.dashboard.widgets, function(row) {
+              _.each(row, function(widget, i) {
+                var newWidget = newWidgets[widget.id];
+                if (newWidget && newWidget[0].visualization.query.latest_query_data_id != widget.visualization.query.latest_query_data_id) {
+                  row[i] = newWidget[0];
+                }
+              });
+            });
+
+            autoRefresh();
+          });
+
+        }, $scope.refreshRate);
+      };
+    }
+
+    $scope.triggerRefresh = function() {
+      $scope.refreshEnabled = !$scope.refreshEnabled;
+
+      Events.record(currentUser, "autorefresh", "dashboard", dashboard.id, {'enable': $scope.refreshEnabled});
+
+      if ($scope.refreshEnabled) {
+        var refreshRate = _.min(_.flatten($scope.dashboard.widgets), function(widget) {
+          return widget.visualization.query.ttl;
+        }).visualization.query.ttl;
+
+        $scope.refreshRate = _.max([120, refreshRate * 2]) * 1000;
+
+        autoRefresh();
+      }
+    };
+  };
+
+  var WidgetCtrl = function($scope, Events, Query) {
+    $scope.deleteWidget = function() {
+      if (!confirm('Are you sure you want to remove "' + $scope.widget.getName() + '" from the dashboard?')) {
+        return;
+      }
+
+      Events.record(currentUser, "delete", "widget", $scope.widget.id);
+
+      $scope.widget.$delete(function() {
+        $scope.dashboard.widgets = _.map($scope.dashboard.widgets, function(row) {
+          return _.filter(row, function(widget) {
+            return widget.id != undefined;
+          })
+        });
+      });
+    };
+
+    Events.record(currentUser, "view", "widget", $scope.widget.id);
+
+    if ($scope.widget.visualization) {
+      Events.record(currentUser, "view", "query", $scope.widget.visualization.query.id);
+      Events.record(currentUser, "view", "visualization", $scope.widget.visualization.id);
+
+      $scope.query = $scope.widget.getQuery();
+      $scope.queryResult = $scope.query.getQueryResult();
+      $scope.nextUpdateTime = moment(new Date(($scope.query.updated_at + $scope.query.ttl + $scope.query.runtime + 300) * 1000)).fromNow();
+
+      $scope.type = 'visualization';
+    } else {
+      $scope.type = 'textbox';
+    }
+  };
+
+  angular.module('redash.controllers')
+    .controller('DashboardCtrl', ['$scope', 'Events', 'Widget', '$routeParams', '$http', '$timeout', '$q', 'Dashboard', DashboardCtrl])
+    .controller('WidgetCtrl', ['$scope', 'Events', 'Query', WidgetCtrl])
+
+})();

rd_ui/app/scripts/controllers/query_source.js

@@ -0,0 +1,120 @@
+(function() {
+  'use strict';
+
+  function QuerySourceCtrl(Events, growl, $controller, $scope, $location, Query, Visualization, KeyboardShortcuts) {
+    // extends QueryViewCtrl
+    $controller('QueryViewCtrl', {$scope: $scope});
+    // TODO:
+    // This doesn't get inherited. Setting it on this didn't work either (which is weird).
+    // Obviously it shouldn't be repeated, but we got bigger fish to fry.
+    var DEFAULT_TAB = 'table';
+
+    Events.record(currentUser, 'view_source', 'query', $scope.query.id);
+
+    var isNewQuery = !$scope.query.id,
+        queryText = $scope.query.query,
+        // ref to QueryViewCtrl.saveQuery
+        saveQuery = $scope.saveQuery,
+        shortcuts = {
+          'meta+s': function () {
+            if ($scope.canEdit) {
+              $scope.saveQuery();
+            }
+          },
+          'meta+enter': function () {
+            $scope.executeQuery();
+          }
+        };
+
+    $scope.sourceMode = true;
+    $scope.canEdit = currentUser.canEdit($scope.query);
+    $scope.isDirty = false;
+
+    $scope.newVisualization = undefined;
+
+    // @override
+    Object.defineProperty($scope, 'showDataset', {
+      get: function() {
+        return $scope.queryResult && $scope.queryResult.getStatus() == 'done';
+      }
+    });
+
+
+    KeyboardShortcuts.bind(shortcuts);
+
+    // @override
+    $scope.saveQuery = function(options, data) {
+      var savePromise = saveQuery(options, data);
+
+      savePromise.then(function(savedQuery) {
+        queryText = savedQuery.query;
+        $scope.isDirty = $scope.query.query !== queryText;
+
+        if (isNewQuery) {
+          // redirect to new created query (keep hash)
+          $location.path(savedQuery.getSourceLink()).replace();
+        }
+      });
+
+      return savePromise;
+    };
+
+    $scope.duplicateQuery = function() {
+      Events.record(currentUser, 'fork', 'query', $scope.query.id);
+      $scope.query.id = null;
+      $scope.query.ttl = -1;
+
+      $scope.saveQuery({
+        successMessage: 'Query forked',
+        errorMessage: 'Query could not be forked'
+      }).then(function redirect(savedQuery) {
+        // redirect to forked query (clear hash)
+        $location.url(savedQuery.getSourceLink()).replace()
+      });
+    };
+
+    $scope.deleteVisualization = function($e, vis) {
+      $e.preventDefault();
+      if (confirm('Are you sure you want to delete ' + vis.name + ' ?')) {
+        Events.record(currentUser, 'delete', 'visualization', vis.id);
+
+        Visualization.delete(vis, function() {
+          if ($scope.selectedTab == vis.id) {
+            $scope.selectedTab = DEFAULT_TAB;
+            $location.hash($scope.selectedTab);
+          }
+          $scope.query.visualizations =
+            $scope.query.visualizations.filter(function (v) {
+              return vis.id !== v.id;
+            });
+        }, function () {
+          growl.addErrorMessage("Error deleting visualization. Maybe it's used in a dashboard?");
+        });
+
+      }
+    };
+
+    $scope.$watch('query.query', function(newQueryText) {
+      $scope.isDirty = (newQueryText !== queryText);
+    });
+
+    $scope.$on('$destroy', function destroy() {
+      KeyboardShortcuts.unbind(shortcuts);
+    });
+
+    if (isNewQuery) {
+      // save new query when creating a visualization
+      var unbind = $scope.$watch('selectedTab == "add"', function(triggerSave) {
+        if (triggerSave) {
+          unbind();
+          $scope.saveQuery();
+        }
+      });
+    }
+  }
+
+  angular.module('redash.controllers').controller('QuerySourceCtrl', [
+    'Events', 'growl', '$controller', '$scope', '$location', 'Query',
+    'Visualization', 'KeyboardShortcuts', QuerySourceCtrl
+    ]);
+})();

rd_ui/app/scripts/controllers/query_view.js

@@ -1,282 +1,144 @@
-(function () {
-    'use strict';
+(function() {
+  'use strict';
 
-    var QueryViewCtrl = function ($scope, $window, $route, $http, $location, growl, notifications, Query, Visualization) {
-        var DEFAULT_TAB = 'table';
-        var pristineHash = "";
-        var leavingPageText = "You will lose your changes if you leave";
-        var route = $route.current;
+  function QueryViewCtrl($scope, Events, $route, $location, notifications, growl, Query, DataSource) {
+    var DEFAULT_TAB = 'table';
 
-        $scope.dirty = undefined;
-        $scope.isNewQuery = false;
-        $scope.isOwner = false;
-        $scope.canEdit = false;
-        $scope.canFork = false;
+    $scope.query = $route.current.locals.query;
+    Events.record(currentUser, 'view', 'query', $scope.query.id);
+    $scope.queryResult = $scope.query.getQueryResult();
+    $scope.queryExecuting = false;
 
-        $scope.isSourceVisible = route.locals.viewSource;
+    $scope.isQueryOwner = currentUser.id === $scope.query.user.id;
+    $scope.canViewSource = currentUser.hasPermission('view_source');
 
-        $scope.queryExecuting = false;
+    $scope.dataSources = DataSource.get(function(dataSources) {
+      $scope.query.data_source_id = $scope.query.data_source_id || dataSources[0].id;
+    });
 
-        $scope.newVisualization = undefined;
+    // in view mode, latest dataset is always visible
+    // source mode changes this behavior
+    $scope.showDataset = true;
 
-        updateSourceHref();
-
-        $window.onbeforeunload = function () {
-            if ($scope.canEdit && $scope.dirty) {
-                return leavingPageText;
-            }
-        }
-
-        function updateSourceHref() {
-            if ($scope.query && $scope.query.id) {
-                $scope.sourceHref = $scope.isSourceVisible ?
-                    $location.url().replace('source', '') : getQuerySourceUrl($scope.query.id);
-            }
-        };
-
-        function getQuerySourceUrl(queryId) {
-            return '/queries/' + queryId + '/source#' + $location.hash();
-        };
-
-        Mousetrap.bindGlobal("meta+s", function (e) {
-            e.preventDefault();
-
-            if ($scope.canEdit) {
-                $scope.saveQuery();
-            }
-        });
-
-        $scope.$on('$locationChangeStart', function (event, next, current) {
-            if (next.split("#")[0] == current.split("#")[0]) {
-                return;
-            }
-
-            if (!$scope.canEdit) {
-                return;
-            }
-
-            if ($scope.dirty && !confirm(leavingPageText + "\n\nAre you sure you want to leave this page?")) {
-                event.preventDefault();
-            } else {
-                Mousetrap.unbind("meta+s");
-            }
-        });
-
-        $scope.$watch(function () {
-            return $location.hash()
-        }, function (hash) {
-            $scope.selectedTab = hash || DEFAULT_TAB;
-            updateSourceHref();
-        });
-
-        $scope.lockButton = function (lock) {
-            $scope.queryExecuting = lock;
-        };
-
-        $scope.formatQuery = function () {
-            $scope.editorOptions.readOnly = 'nocursor';
-
-            $http.post('/api/queries/format', {
-                'query': $scope.query.query
-            }).success(function (response) {
-                $scope.query.query = response;
-                $scope.editorOptions.readOnly = false;
-            })
-        }
-
-        $scope.saveQuery = function (duplicate, oldId) {
-            if (!oldId) {
-                oldId = $scope.query.id;
-            }
-
-            delete $scope.query.latest_query_data;
-            $scope.query.$save(function (q) {
-                pristineHash = q.getHash();
-                $scope.dirty = false;
-
-                if (duplicate) {
-                    growl.addSuccessMessage("Query forked");
-                } else {
-                    growl.addSuccessMessage("Query saved");
-                }
-
-                if (oldId != q.id) {
-                    $location.url(getQuerySourceUrl(q.id)).replace();
-                }
-            }, function (httpResponse) {
-                growl.addErrorMessage("Query could not be saved");
-            });
-        };
-
-        $scope.duplicateQuery = function () {
-            var oldId = $scope.query.id;
-            $scope.query.id = null;
-            $scope.query.ttl = -1;
-
-            $scope.saveQuery(true, oldId);
-        };
-
-        // Query Editor:
-        $scope.editorOptions = {
-            mode: 'text/x-sql',
-            lineWrapping: true,
-            lineNumbers: true,
-            readOnly: false,
-            matchBrackets: true,
-            autoCloseBrackets: true
-        };
-
-        $scope.refreshOptions = [
-            {
-                value: -1,
-                name: 'No Refresh'
-            },
-            {
-                value: 60,
-                name: 'Every minute'
-            },
-        ]
-
-        _.each(_.range(1, 13), function (i) {
-            $scope.refreshOptions.push({
-                value: i * 3600,
-                name: 'Every ' + i + 'h'
-            });
-        })
-
-        $scope.refreshOptions.push({
-            value: 24 * 3600,
-            name: 'Every 24h'
-        });
-        $scope.refreshOptions.push({
-            value: 7 * 24 * 3600,
-            name: 'Once a week'
-        });
-
-        $scope.$watch('queryResult && queryResult.getError()', function (newError, oldError) {
-            if (newError == undefined) {
-                return;
-            }
-
-            if (oldError == undefined && newError != undefined) {
-                $scope.lockButton(false);
-            }
-        });
-
-        $scope.$watch('queryResult && queryResult.getData()', function (data, oldData) {
-            if (!data) {
-                return;
-            }
-
-            if ($scope.queryResult.getId() == null) {
-                $scope.dataUri = "";
-            } else {
-                $scope.dataUri = '/api/queries/' + $scope.query.id + '/results/' + $scope.queryResult.getId() + '.csv';
-                $scope.dataFilename = $scope.query.name.replace(" ", "_") + moment($scope.queryResult.getUpdatedAt()).format("_YYYY_MM_DD") + ".csv";
-            }
-        });
-
-        $scope.$watch("queryResult && queryResult.getStatus()", function (status) {
-            if (!status) {
-                return;
-            }
-
-            if (status == "done") {
-                if ($scope.query.id && $scope.query.latest_query_data_id != $scope.queryResult.getId() &&
-                    $scope.query.query_hash == $scope.queryResult.query_result.query_hash) {
-                    Query.save({
-                        'id': $scope.query.id,
-                        'latest_query_data_id': $scope.queryResult.getId()
-                    })
-                }
-                $scope.query.latest_query_data_id = $scope.queryResult.getId();
-
-                notifications.showNotification("re:dash", $scope.query.name + " updated.");
-
-                $scope.lockButton(false);
-            }
-        });
-
-        // view or source pages: controller is instantiated with a query
-        if (route.locals.query) {
-            $scope.query = route.locals.query;
-            pristineHash = $scope.query.getHash();
-            $scope.dirty = false;
-            $scope.queryResult = $scope.query.getQueryResult();
-
-            $scope.isOwner = currentUser.canEdit($scope.query);
-            $scope.canEdit = $scope.isSourceVisible && $scope.isOwner;
-            $scope.canFork = true;
-
-        } else {
-            // new query
-            $scope.query = new Query({
-                query: "",
-                name: "New Query",
-                ttl: -1,
-                user: currentUser
-            });
-            $scope.lockButton(false);
-            $scope.isOwner = $scope.canEdit = true;
-            $scope.isNewQuery = true;
-        }
-
-        $scope.$watch('query.name', function () {
-            $scope.$parent.pageTitle = $scope.query.name;
-        });
-
-        $scope.$watch(function () {
-            return $scope.query.getHash();
-        }, function (newHash) {
-            $scope.dirty = (newHash !== pristineHash);
-        });
-
-        $scope.executeQuery = function () {
-            $scope.queryResult = $scope.query.getQueryResult(0);
-            $scope.lockButton(true);
-            $scope.cancelling = false;
-        };
-
-        $scope.cancelExecution = function () {
-            $scope.cancelling = true;
-            $scope.queryResult.cancelExecution();
-        };
-
-        $scope.deleteVisualization = function ($e, vis) {
-            $e.preventDefault();
-            if (confirm('Are you sure you want to delete ' + vis.name + ' ?')) {
-                Visualization.delete(vis);
-                if ($scope.selectedTab == vis.id) {
-                    $scope.selectedTab = DEFAULT_TAB;
-                    $location.hash($scope.selectedTab);
-                }
-                $scope.query.visualizations =
-                    $scope.query.visualizations.filter(function (v) {
-                        return vis.id !== v.id;
-                    });
-            }
-        };
-
-        var unbind = $scope.$watch('selectedTab == "add"', function (newPanel) {
-            if (newPanel && route.params.queryId == undefined) {
-                unbind();
-                $scope.saveQuery();
-            }
-        });
+    $scope.lockButton = function(lock) {
+      $scope.queryExecuting = lock;
     };
 
-    angular.module('redash.controllers').controller('QueryViewCtrl',
-        [
-            '$scope',
-            '$window',
-            '$route',
-            '$http',
-            '$location',
-            'growl',
-            'notifications',
-            'Query',
-            'Visualization',
-            QueryViewCtrl
-        ]);
+    $scope.saveQuery = function(options, data) {
+      if (data) {
+        data.id = $scope.query.id;
+      } else {
+        data = _.clone($scope.query);
+      }
 
-})();
+      options = _.extend({}, {
+        successMessage: 'Query saved',
+        errorMessage: 'Query could not be saved'
+      }, options);
+
+      delete data.latest_query_data;
+      delete data.queryResult;
+
+      return Query.save(data, function() {
+        growl.addSuccessMessage(options.successMessage);
+      }, function(httpResponse) {
+        growl.addErrorMessage(options.errorMessage);
+      }).$promise;
+    }
+
+    $scope.saveDescription = function() {
+      Events.record(currentUser, 'edit_description', 'query', $scope.query.id);
+      $scope.saveQuery(undefined, {'description': $scope.query.description});
+    };
+
+    $scope.saveName = function() {
+      Events.record(currentUser, 'edit_name', 'query', $scope.query.id);
+      $scope.saveQuery(undefined, {'name': $scope.query.name});
+    };
+
+    $scope.executeQuery = function() {
+      $scope.queryResult = $scope.query.getQueryResult(0);
+      $scope.lockButton(true);
+      $scope.cancelling = false;
+      Events.record(currentUser, 'execute', 'query', $scope.query.id);
+    };
+
+    $scope.cancelExecution = function() {
+      $scope.cancelling = true;
+      $scope.queryResult.cancelExecution();
+      Events.record(currentUser, 'cancel_execute', 'query', $scope.query.id);
+    };
+
+    $scope.updateDataSource = function() {
+      Events.record(currentUser, 'update_data_source', 'query', $scope.query.id);
+
+      $scope.query.latest_query_data = null;
+      $scope.query.latest_query_data_id = null;
+
+      if ($scope.query.id) {
+        Query.save({
+          'id': $scope.query.id,
+          'data_source_id': $scope.query.data_source_id,
+          'latest_query_data_id': null
+        });
+      }
+
+      $scope.executeQuery();
+    };
+
+    $scope.setVisualizationTab = function (visualization) {
+      $scope.selectedTab = visualization.id;
+      $location.hash(visualization.id);
+    };
+
+    $scope.$watch('query.name', function() {
+      $scope.$parent.pageTitle = $scope.query.name;
+    });
+
+    $scope.$watch('queryResult && queryResult.getData()', function(data, oldData) {
+      if (!data) {
+        return;
+      }
+
+      $scope.filters = $scope.queryResult.getFilters();
+    });
+
+    $scope.$watch("queryResult && queryResult.getStatus()", function(status) {
+      if (!status) {
+        return;
+      }
+
+      if (status == 'done') {
+        if ($scope.query.id &&
+          $scope.query.latest_query_data_id != $scope.queryResult.getId() &&
+          $scope.query.query_hash == $scope.queryResult.query_result.query_hash) {
+          Query.save({
+            'id': $scope.query.id,
+            'latest_query_data_id': $scope.queryResult.getId()
+          })
+        }
+        $scope.query.latest_query_data_id = $scope.queryResult.getId();
+        $scope.query.queryResult = $scope.queryResult;
+
+        notifications.showNotification("re:dash", $scope.query.name + " updated.");
+      }
+
+      if (status === 'done' || status === 'failed') {
+        $scope.lockButton(false);
+      }
+    });
+
+    $scope.$watch(function() {
+      return $location.hash()
+    }, function(hash) {
+      if (hash == 'pivot') {
+        Events.record(currentUser, 'pivot', 'query', $scope.query && $scope.query.id);
+      }
+      $scope.selectedTab = hash || DEFAULT_TAB;
+    });
+  };
+
+  angular.module('redash.controllers')
+    .controller('QueryViewCtrl',
+      ['$scope', 'Events', '$route', '$location', 'notifications', 'growl', 'Query', 'DataSource', QueryViewCtrl]);
+})();

rd_ui/app/scripts/directives.js

@@ -1,347 +0,0 @@
-(function() {
-    'use strict';
-
-    var directives = angular.module('redash.directives', []);
-
-    directives.directive('rdTab', function() {
-        return {
-            restrict: 'E',
-            scope: {
-                'tabId': '@',
-                'name': '@'
-            },
-            transclude: true,
-            template: '<li class="rd-tab" ng-class="{active: tabId==selectedTab}"><a href="#{{tabId}}">{{name}}<span ng-transclude></span></a></li>',
-            replace: true,
-            link: function(scope) {
-                scope.$watch(function(){return scope.$parent.selectedTab}, function(tab) {
-                    scope.selectedTab = tab;
-                });
-            }
-        }
-    });
-
-    directives.directive('rdTabs', ['$location', function($location) {
-        return {
-            restrict: 'E',
-            scope: {
-                tabsCollection: '=',
-                selectedTab: '='
-            },
-            template: '<ul class="nav nav-tabs"><li ng-class="{active: tab==selectedTab}" ng-repeat="tab in tabsCollection"><a href="#{{tab.key}}">{{tab.name}}</a></li></ul>',
-            replace: true,
-            link: function($scope, element, attrs) {
-                $scope.selectTab = function(tabKey) {
-                    $scope.selectedTab  = _.find($scope.tabsCollection, function(tab) { return tab.key == tabKey; });
-                }
-
-                $scope.$watch(function() { return $location.hash()}, function(hash) {
-                    if (hash) {
-                        $scope.selectTab($location.hash());
-                    } else {
-                        $scope.selectTab($scope.tabsCollection[0].key);
-                    }
-                });
-            }
-        }
-    }]);
-
-    directives.directive('editDashboardForm', ['$http', '$location', '$timeout', 'Dashboard', function($http, $location, $timeout, Dashboard) {
-        return {
-            restrict: 'E',
-            scope: {
-                dashboard: '='
-            },
-            templateUrl: '/views/edit_dashboard.html',
-            replace: true,
-            link: function($scope, element, attrs) {
-                var gridster = element.find(".gridster ul").gridster({
-                    widget_margins: [5, 5],
-                    widget_base_dimensions: [260, 100],
-                    min_cols: 2,
-                    max_cols: 2,
-                    serialize_params: function($w, wgd) {
-                        return {
-                            col: wgd.col,
-                            row: wgd.row,
-                            id: $w.data('widget-id')
-                        }
-                    }
-                }).data('gridster');
-
-                var gsItemTemplate = '<li data-widget-id="{id}" class="widget panel panel-default gs-w">' +
-                                 '<div class="panel-heading">{name}' +
-                                 '</div></li>';
-
-                $scope.$watch('dashboard.widgets', function(widgets) {
-                    $timeout(function () {
-                        gridster.remove_all_widgets();
-
-                        if (widgets && widgets.length) {
-                            var layout = [];
-
-                            _.each(widgets, function(row, rowIndex) {
-                                _.each(row, function(widget, colIndex) {
-                                    layout.push({
-                                        id: widget.id,
-                                        col: colIndex+1,
-                                        row: rowIndex+1,
-                                        ySize: 1,
-                                        xSize: widget.width,
-                                        name: widget.visualization.query.name
-                                    });
-                                });
-                            });
-
-                            _.each(layout, function(item) {
-                                var el = gsItemTemplate.replace('{id}', item.id).replace('{name}', item.name);
-                                gridster.add_widget(el, item.xSize, item.ySize, item.col, item.row);
-
-                            });
-                        }
-                    });
-                }, true);
-
-                $scope.saveDashboard = function() {
-                    $scope.saveInProgress = true;
-                    // TODO: we should use the dashboard service here.
-                    if ($scope.dashboard.id) {
-                        var positions = $(element).find('.gridster ul').data('gridster').serialize();
-                        var layout = [];
-                        _.each(_.sortBy(positions, function (pos) {
-                            return pos.row * 10 + pos.col;
-                        }), function (pos) {
-                            var row = pos.row - 1;
-                            var col = pos.col - 1;
-                            layout[row] = layout[row] || [];
-                            if (col > 0 && layout[row][col - 1] == undefined) {
-                                layout[row][col - 1] = pos.id;
-                            } else {
-                                layout[row][col] = pos.id;
-                            }
-
-                        });
-                        $scope.dashboard.layout = layout;
-
-                        layout = JSON.stringify(layout);
-                        $http.post('/api/dashboards/' + $scope.dashboard.id, {'name': $scope.dashboard.name, 'layout': layout}).success(function(response) {
-                            $scope.dashboard = new Dashboard(response);
-                            $scope.saveInProgress = false;
-                            $(element).modal('hide');
-                        })
-                    } else {
-                        $http.post('/api/dashboards', {'name': $scope.dashboard.name}).success(function(response) {
-                            $(element).modal('hide');
-                            $location.path('/dashboard/' + response.slug).replace();
-                        })
-                    }
-                }
-
-            }
-        }
-    }]);
-
-    directives.directive('newWidgetForm', ['$http', 'Query', function($http, Query) {
-        return {
-            restrict: 'E',
-            scope: {
-                dashboard: '='
-            },
-            templateUrl: '/views/new_widget_form.html',
-            replace: true,
-            link: function($scope, element, attrs) {
-                $scope.widgetSizes = [{name: 'Regular', value: 1}, {name: 'Double', value: 2}];
-
-                var reset = function() {
-                    $scope.saveInProgress = false;
-                    $scope.widgetSize = 1;
-                    $scope.queryId = null;
-                    $scope.selectedVis = null;
-                    $scope.query = null;
-
-                }
-
-                reset();
-
-                $scope.loadVisualizations = function() {
-                    if (!$scope.queryId) {
-                        return;
-                    }
-
-                    Query.get({
-                        id: $scope.queryId
-                    }, function(query) {
-                        if (query) {
-                            $scope.query = query;
-                            if(query.visualizations.length) {
-                                $scope.selectedVis = query.visualizations[0];
-                            }
-                        }
-                    });
-                };
-
-                $scope.saveWidget = function() {
-                    $scope.saveInProgress = true;
-
-                    var widget = {
-                        'visualization_id': $scope.selectedVis.id,
-                        'dashboard_id': $scope.dashboard.id,
-                        'options': {},
-                        'width': $scope.widgetSize
-                    }
-
-                    $http.post('/api/widgets', widget).success(function(response) {
-                        // update dashboard layout
-                        $scope.dashboard.layout = response['layout'];
-                        if (response['new_row']) {
-                            $scope.dashboard.widgets.push([response['widget']]);
-                        } else {
-                            $scope.dashboard.widgets[$scope.dashboard.widgets.length-1].push(response['widget']);
-                        }
-
-                        // close the dialog
-                        $('#add_query_dialog').modal('hide');
-                        reset();
-                    })
-                }
-
-            }
-        }
-    }])
-
-    // From: http://jsfiddle.net/joshdmiller/NDFHg/
-    directives.directive('editInPlace', function () {
-        return {
-            restrict: 'E',
-            scope: {
-                value: '=',
-                ignoreBlanks: '=',
-                editable: '=',
-                done: '='
-            },
-            template: function(tElement, tAttrs) {
-                var elType = tAttrs.editor || 'input';
-                var placeholder = tAttrs.placeholder || 'Click to edit';
-                return '<span ng-click="editable && edit()" ng-bind="value" ng-class="{editable: editable}"></span>' +
-                       '<span ng-click="editable && edit()" ng-show="editable && !value" ng-class="{editable: editable}">' + placeholder + '</span>' +
-                       '<{elType} ng-model="value" class="rd-form-control"></{elType}>'.replace('{elType}', elType);
-            },
-            link: function ($scope, element, attrs) {
-                // Let's get a reference to the input element, as we'll want to reference it.
-                var inputElement = angular.element(element.children()[2]);
-
-                // This directive should have a set class so we can style it.
-                element.addClass('edit-in-place');
-
-                // Initially, we're not editing.
-                $scope.editing = false;
-
-                // ng-click handler to activate edit-in-place
-                $scope.edit = function () {
-                    $scope.oldValue = $scope.value;
-
-                    $scope.editing = true;
-
-                    // We control display through a class on the directive itself. See the CSS.
-                    element.addClass('active');
-
-                    // And we must focus the element.
-                    // `angular.element()` provides a chainable array, like jQuery so to access a native DOM function,
-                    // we have to reference the first element in the array.
-                    inputElement[0].focus();
-                };
-
-                function save() {
-                    if ($scope.editing) {
-                        if ($scope.ignoreBlanks && _.isEmpty($scope.value)) {
-                            $scope.value = $scope.oldValue;
-                        }
-                        $scope.editing = false;
-                        element.removeClass('active');
-
-                        if ($scope.value !== $scope.oldValue) {
-                            $scope.done && $scope.done();
-                        }
-                    }
-                }
-
-                $(inputElement).keydown(function(e) {
-                    // 'return' or 'enter' key pressed
-                    // allow 'shift' to break lines
-                    if (e.which === 13 && !e.shiftKey) {
-                        save();
-                    } else if (e.which === 27) {
-                        $scope.value = $scope.oldValue;
-                        $scope.$apply(function() {
-                            $(inputElement[0]).blur();
-                        });
-                    }
-                }).blur(function() {
-                    save();
-                });
-            }
-        };
-    });
-
-    // http://stackoverflow.com/a/17904092/1559840
-    directives.directive('jsonText', function() {
-        return {
-            restrict: 'A',
-            require: 'ngModel',
-            link: function(scope, element, attr, ngModel) {
-              function into(input) {
-                return JSON.parse(input);
-              }
-              function out(data) {
-                return JSON.stringify(data, undefined, 2);
-              }
-              ngModel.$parsers.push(into);
-              ngModel.$formatters.push(out);
-
-            }
-        };
-    });
-
-    directives.directive('rdTimer', ['$timeout', function ($timeout) {
-        return {
-            restrict: 'E',
-            scope: { timestamp: '=' },
-            template: '{{currentTime}}',
-            controller: ['$scope' ,function ($scope) {
-                $scope.currentTime = "00:00:00";
-                var currentTimeout = null;
-
-                var updateTime = function() {
-                    $scope.currentTime = moment(moment() - moment($scope.timestamp)).utc().format("HH:mm:ss")
-                    currentTimeout = $timeout(updateTime, 1000);
-                }
-
-                var cancelTimer = function() {
-                    if (currentTimeout) {
-                        $timeout.cancel(currentTimeout);
-                        currentTimeout = null;
-                    }
-                }
-
-                updateTime();
-
-                $scope.$on('$destroy', function () {
-                    cancelTimer();
-                });
-            }]
-        };
-    }]);
-
-    directives.directive('rdTimeAgo', function() {
-        return {
-            restrict: 'E',
-            scope: {
-                value: '='
-            },
-            template: '<span>' +
-                        '<span ng-show="value" am-time-ago="value"></span>' +
-                        '<span ng-hide="value">-</span>' +
-                      '</span>'
-        }
-    });
-})();

rd_ui/app/scripts/directives/dashboard_directives.js

@@ -0,0 +1,208 @@
+(function() {
+  'use strict'
+
+  var directives = angular.module('redash.directives');
+
+  directives.directive('editDashboardForm', ['Events', '$http', '$location', '$timeout', 'Dashboard',
+    function(Events, $http, $location, $timeout, Dashboard) {
+      return {
+        restrict: 'E',
+        scope: {
+          dashboard: '='
+        },
+        templateUrl: '/views/edit_dashboard.html',
+        replace: true,
+        link: function($scope, element, attrs) {
+          var gridster = element.find(".gridster ul").gridster({
+            widget_margins: [5, 5],
+            widget_base_dimensions: [260, 100],
+            min_cols: 2,
+            max_cols: 2,
+            serialize_params: function($w, wgd) {
+              return {
+                col: wgd.col,
+                row: wgd.row,
+                id: $w.data('widget-id')
+              }
+            }
+          }).data('gridster');
+
+          var gsItemTemplate = '<li data-widget-id="{id}" class="widget panel panel-default gs-w">' +
+            '<div class="panel-heading">{name}' +
+            '</div></li>';
+
+          $scope.$watch('dashboard.widgets && dashboard.widgets.length', function(widgets_length) {
+            $timeout(function() {
+              gridster.remove_all_widgets();
+
+              if ($scope.dashboard.widgets && $scope.dashboard.widgets.length) {
+                var layout = [];
+
+                _.each($scope.dashboard.widgets, function(row, rowIndex) {
+                  _.each(row, function(widget, colIndex) {
+                    layout.push({
+                      id: widget.id,
+                      col: colIndex + 1,
+                      row: rowIndex + 1,
+                      ySize: 1,
+                      xSize: widget.width,
+                      name: widget.getName()//visualization.query.name
+                    });
+                  });
+                });
+
+                _.each(layout, function(item) {
+                  var el = gsItemTemplate.replace('{id}', item.id).replace('{name}', item.name);
+                  gridster.add_widget(el, item.xSize, item.ySize, item.col, item.row);
+                });
+              }
+            });
+          });
+
+          $scope.saveDashboard = function() {
+            $scope.saveInProgress = true;
+            // TODO: we should use the dashboard service here.
+            if ($scope.dashboard.id) {
+              var positions = $(element).find('.gridster ul').data('gridster').serialize();
+              var layout = [];
+              _.each(_.sortBy(positions, function(pos) {
+                return pos.row * 10 + pos.col;
+              }), function(pos) {
+                var row = pos.row - 1;
+                var col = pos.col - 1;
+                layout[row] = layout[row] || [];
+                if (col > 0 && layout[row][col - 1] == undefined) {
+                  layout[row][col - 1] = pos.id;
+                } else {
+                  layout[row][col] = pos.id;
+                }
+
+              });
+              $scope.dashboard.layout = layout;
+
+              layout = JSON.stringify(layout);
+              $http.post('/api/dashboards/' + $scope.dashboard.id, {
+                'name': $scope.dashboard.name,
+                'layout': layout
+              }).success(function(response) {
+                $scope.dashboard = new Dashboard(response);
+                $scope.saveInProgress = false;
+                $(element).modal('hide');
+              });
+              Events.record(currentUser, 'edit', 'dashboard', $scope.dashboard.id);
+            } else {
+
+              $http.post('/api/dashboards', {
+                'name': $scope.dashboard.name
+              }).success(function(response) {
+                $(element).modal('hide');
+                $scope.dashboard = {
+                  'name': null,
+                  'layout': null
+                };
+                $scope.saveInProgress = false;
+                $location.path('/dashboard/' + response.slug).replace();
+              });
+              Events.record(currentUser, 'create', 'dashboard');
+            }
+          }
+
+        }
+      }
+    }
+  ]);
+
+  directives.directive('newWidgetForm', ['Query', 'Widget', 'growl',
+    function(Query, Widget, growl) {
+      return {
+        restrict: 'E',
+        scope: {
+          dashboard: '='
+        },
+        templateUrl: '/views/new_widget_form.html',
+        replace: true,
+        link: function($scope, element, attrs) {
+          $scope.widgetSizes = [{
+            name: 'Regular',
+            value: 1
+          }, {
+            name: 'Double',
+            value: 2
+          }];
+
+          $scope.type = 'visualization';
+
+          $scope.isVisualization = function () {
+            return $scope.type == 'visualization';
+          };
+
+          $scope.isTextBox = function () {
+            return $scope.type == 'textbox';
+          };
+
+          $scope.setType = function (type) {
+            $scope.type = type;
+          };
+
+          var reset = function() {
+            $scope.saveInProgress = false;
+            $scope.widgetSize = 1;
+            $scope.queryId = null;
+            $scope.selectedVis = null;
+            $scope.query = null;
+            $scope.text = "";
+          };
+
+          reset();
+
+          $scope.loadVisualizations = function () {
+            if (!$scope.queryId) {
+              return;
+            }
+
+            Query.get({ id: $scope.queryId }, function(query) {
+              if (query) {
+                $scope.query = query;
+                if (query.visualizations.length) {
+                  $scope.selectedVis = query.visualizations[0];
+                }
+              }
+            });
+          };
+
+          $scope.saveWidget = function() {
+            $scope.saveInProgress = true;
+
+            var widget = new Widget({
+              'visualization_id': $scope.selectedVis && $scope.selectedVis.id,
+              'dashboard_id': $scope.dashboard.id,
+              'options': {},
+              'width': $scope.widgetSize,
+              'text': $scope.text
+            });
+
+            widget.$save().then(function(response) {
+              // update dashboard layout
+              $scope.dashboard.layout = response['layout'];
+              var newWidget = new Widget(response['widget']);
+              if (response['new_row']) {
+                $scope.dashboard.widgets.push([newWidget]);
+              } else {
+                $scope.dashboard.widgets[$scope.dashboard.widgets.length - 1].push(newWidget);
+              }
+
+              // close the dialog
+              $('#add_query_dialog').modal('hide');
+              reset();
+            }).catch(function() {
+              growl.addErrorMessage("Widget can not be added");
+            }).finally(function() {
+              $scope.saveInProgress = false;
+            });
+          }
+
+        }
+      }
+    }
+  ])
+})();

rd_ui/app/scripts/directives/directives.js

@@ -0,0 +1,227 @@
+(function () {
+  'use strict';
+
+  var directives = angular.module('redash.directives', []);
+
+  directives.directive('alertUnsavedChanges', ['$window', function ($window) {
+    return {
+      restrict: 'E',
+      replace: true,
+      scope: {
+        'isDirty': '='
+      },
+      link: function ($scope) {
+        var
+
+          unloadMessage = "You will lose your changes if you leave",
+          confirmMessage = unloadMessage + "\n\nAre you sure you want to leave this page?",
+
+        // store original handler (if any)
+          _onbeforeunload = $window.onbeforeunload;
+
+        $window.onbeforeunload = function () {
+          return $scope.isDirty ? unloadMessage : null;
+        }
+
+        $scope.$on('$locationChangeStart', function (event, next, current) {
+          if (next.split("#")[0] == current.split("#")[0]) {
+            return;
+          }
+
+          if ($scope.isDirty && !confirm(confirmMessage)) {
+            event.preventDefault();
+          }
+        });
+
+        $scope.$on('$destroy', function () {
+          $window.onbeforeunload = _onbeforeunload;
+        });
+      }
+    }
+  }]);
+
+  directives.directive('rdTab', function () {
+    return {
+      restrict: 'E',
+      scope: {
+        'tabId': '@',
+        'name': '@'
+      },
+      transclude: true,
+      template: '<li class="rd-tab" ng-class="{active: tabId==selectedTab}"><a href="#{{tabId}}">{{name}}<span ng-transclude></span></a></li>',
+      replace: true,
+      link: function (scope) {
+        scope.$watch(function () {
+          return scope.$parent.selectedTab
+        }, function (tab) {
+          scope.selectedTab = tab;
+        });
+      }
+    }
+  });
+
+  directives.directive('rdTabs', ['$location', function ($location) {
+    return {
+      restrict: 'E',
+      scope: {
+        tabsCollection: '=',
+        selectedTab: '='
+      },
+      template: '<ul class="nav nav-tabs"><li ng-class="{active: tab==selectedTab}" ng-repeat="tab in tabsCollection"><a href="#{{tab.key}}">{{tab.name}}</a></li></ul>',
+      replace: true,
+      link: function ($scope, element, attrs) {
+        $scope.selectTab = function (tabKey) {
+          $scope.selectedTab = _.find($scope.tabsCollection, function (tab) {
+            return tab.key == tabKey;
+          });
+        }
+
+        $scope.$watch(function () {
+          return $location.hash()
+        }, function (hash) {
+          if (hash) {
+            $scope.selectTab($location.hash());
+          } else {
+            $scope.selectTab($scope.tabsCollection[0].key);
+          }
+        });
+      }
+    }
+  }]);
+
+  // From: http://jsfiddle.net/joshdmiller/NDFHg/
+  directives.directive('editInPlace', function () {
+    return {
+      restrict: 'E',
+      scope: {
+        value: '=',
+        ignoreBlanks: '=',
+        editable: '=',
+        done: '='
+      },
+      template: function (tElement, tAttrs) {
+        var elType = tAttrs.editor || 'input';
+        var placeholder = tAttrs.placeholder || 'Click to edit';
+        return '<span ng-click="editable && edit()" ng-bind="value" ng-class="{editable: editable}"></span>' +
+          '<span ng-click="editable && edit()" ng-show="editable && !value" ng-class="{editable: editable}">' + placeholder + '</span>' +
+          '<{elType} ng-model="value" class="rd-form-control"></{elType}>'.replace('{elType}', elType);
+      },
+      link: function ($scope, element, attrs) {
+        // Let's get a reference to the input element, as we'll want to reference it.
+        var inputElement = angular.element(element.children()[2]);
+
+        // This directive should have a set class so we can style it.
+        element.addClass('edit-in-place');
+
+        // Initially, we're not editing.
+        $scope.editing = false;
+
+        // ng-click handler to activate edit-in-place
+        $scope.edit = function () {
+          $scope.oldValue = $scope.value;
+
+          $scope.editing = true;
+
+          // We control display through a class on the directive itself. See the CSS.
+          element.addClass('active');
+
+          // And we must focus the element.
+          // `angular.element()` provides a chainable array, like jQuery so to access a native DOM function,
+          // we have to reference the first element in the array.
+          inputElement[0].focus();
+        };
+
+        function save() {
+          if ($scope.editing) {
+            if ($scope.ignoreBlanks && _.isEmpty($scope.value)) {
+              $scope.value = $scope.oldValue;
+            }
+            $scope.editing = false;
+            element.removeClass('active');
+
+            if ($scope.value !== $scope.oldValue) {
+              $scope.done && $scope.done();
+            }
+          }
+        }
+
+        $(inputElement).keydown(function (e) {
+          // 'return' or 'enter' key pressed
+          // allow 'shift' to break lines
+          if (e.which === 13 && !e.shiftKey) {
+            save();
+          } else if (e.which === 27) {
+            $scope.value = $scope.oldValue;
+            $scope.$apply(function () {
+              $(inputElement[0]).blur();
+            });
+          }
+        }).blur(function () {
+          save();
+        });
+      }
+    };
+  });
+
+  // http://stackoverflow.com/a/17904092/1559840
+  directives.directive('jsonText', function () {
+    return {
+      restrict: 'A',
+      require: 'ngModel',
+      link: function (scope, element, attr, ngModel) {
+        function into(input) {
+          return JSON.parse(input);
+        }
+
+        function out(data) {
+          return JSON.stringify(data, undefined, 2);
+        }
+
+        ngModel.$parsers.push(into);
+        ngModel.$formatters.push(out);
+
+        scope.$watch(attr.ngModel, function (newValue) {
+          element[0].value = out(newValue);
+        }, true);
+      }
+    };
+  });
+
+  directives.directive('rdTimer', [function () {
+    return {
+      restrict: 'E',
+      scope: { timestamp: '=' },
+      template: '{{currentTime}}',
+      controller: ['$scope' , function ($scope) {
+        $scope.currentTime = "00:00:00";
+
+        // We're using setInterval directly instead of $timeout, to avoid using $apply, to
+        // prevent the digest loop being run every second.
+        var currentTimer = setInterval(function () {
+          $scope.currentTime = moment(moment() - moment($scope.timestamp)).utc().format("HH:mm:ss");
+          $scope.$digest();
+        }, 1000);
+
+        $scope.$on('$destroy', function () {
+          if (currentTimer) {
+            clearInterval(currentTimer);
+            currentTimer = null;
+          }
+        });
+      }]
+    };
+  }]);
+
+  directives.directive('rdTimeAgo', function () {
+    return {
+      restrict: 'E',
+      scope: {
+        value: '='
+      },
+      template: '<span>' +
+        '<span ng-show="value" am-time-ago="value"></span>' +
+        '<span ng-hide="value">-</span>' +
+        '</span>'
+    }
+  });
+})();

rd_ui/app/scripts/directives/query_directives.js

@@ -0,0 +1,162 @@
+(function() {
+  'use strict'
+
+  function queryLink() {
+    return {
+      restrict: 'E',
+      scope: {
+        'query': '=',
+        'visualization': '=?'
+      },
+      template: '<a ng-href="{{link}}" class="query-link">{{query.name}}</a>',
+      link: function(scope, element) {
+        scope.link = '/queries/' + scope.query.id;
+        if (scope.visualization) {
+          if (scope.visualization.type === 'TABLE') {
+            // link to hard-coded table tab instead of the (hidden) visualization tab
+            scope.link += '#table';
+          } else {
+            scope.link += '#' + scope.visualization.id;
+          }
+        }
+        // element.find('a').attr('href', link);
+      }
+    }
+  }
+
+  function querySourceLink() {
+    return {
+      restrict: 'E',
+      template: '<span ng-show="query.id && canViewSource">\
+                    <a ng-show="!sourceMode"\
+                      ng-href="{{query.id}}/source#{{selectedTab}}">Show Source\
+                    </a>\
+                    <a ng-show="sourceMode"\
+                      ng-href="/queries/{{query.id}}#{{selectedTab}}">Hide Source\
+                    </a>\
+                </span>'
+    }
+  }
+
+  function queryResultCSVLink() {
+    return {
+      restrict: 'A',
+      link: function (scope, element) {
+        scope.$watch('queryResult && queryResult.getData()', function(data) {
+          if (!data) {
+            return;
+          }
+
+          if (scope.queryResult.getId() == null) {
+            element.attr('href', '');
+          } else {
+            element.attr('href', '/api/queries/' + scope.query.id + '/results/' + scope.queryResult.getId() + '.csv');
+            element.attr('download', scope.query.name.replace(" ", "_") + moment(scope.queryResult.getUpdatedAt()).format("_YYYY_MM_DD") + ".csv");
+          }
+        });
+      }
+    }
+  }
+
+  function queryEditor() {
+    return {
+      restrict: 'E',
+      scope: {
+        'query': '=',
+        'lock': '='
+      },
+      template: '<textarea\
+                  ui-codemirror="editorOptions"\
+                  ng-model="query.query">',
+      link: function($scope) {
+        $scope.editorOptions = {
+            mode: 'text/x-sql',
+            lineWrapping: true,
+            lineNumbers: true,
+            readOnly: false,
+            matchBrackets: true,
+            autoCloseBrackets: true
+        };
+
+        $scope.$watch('lock', function(locked) {
+          $scope.editorOptions.readOnly = locked ? 'nocursor' : false;
+        });
+      }
+    }
+  }
+
+  function queryFormatter($http) {
+    return {
+      restrict: 'E',
+      // don't create new scope to avoid ui-codemirror bug
+      // seehttps://github.com/angular-ui/ui-codemirror/pull/37
+      scope: false,
+      template: '<button type="button" class="btn btn-default btn-xs"\
+                   ng-click="formatQuery()">\
+                    <span class="glyphicon glyphicon-indent-left"></span>\
+                     Format SQL\
+                </button>',
+      link: function($scope) {
+        $scope.formatQuery = function formatQuery() {
+            $scope.queryExecuting = true;
+            $http.post('/api/queries/format', {
+                'query': $scope.query.query
+            }).success(function (response) {
+                $scope.query.query = response;
+            }).finally(function () {
+              $scope.queryExecuting = false;
+            });
+        };
+      }
+    }
+  }
+
+  function queryRefreshSelect() {
+    return {
+      restrict: 'E',
+      template: '<select\
+                  ng-disabled="!isQueryOwner"\
+                  ng-model="query.ttl"\
+                  ng-change="saveQuery()"\
+                  ng-options="c.value as c.name for c in refreshOptions">\
+                  </select>',
+      link: function($scope) {
+        $scope.refreshOptions = [
+            {
+                value: -1,
+                name: 'No Refresh'
+            },
+            {
+                value: 60,
+                name: 'Every minute'
+            },
+        ]
+
+        _.each(_.range(1, 13), function (i) {
+            $scope.refreshOptions.push({
+                value: i * 3600,
+                name: 'Every ' + i + 'h'
+            });
+        })
+
+        $scope.refreshOptions.push({
+            value: 24 * 3600,
+            name: 'Every 24h'
+        });
+        $scope.refreshOptions.push({
+            value: 7 * 24 * 3600,
+            name: 'Once a week'
+        });
+      }
+
+    }
+  }
+
+  angular.module('redash.directives')
+  .directive('queryLink', queryLink)
+  .directive('querySourceLink', querySourceLink)
+  .directive('queryResultLink', queryResultCSVLink)
+  .directive('queryEditor', queryEditor)
+  .directive('queryRefreshSelect', queryRefreshSelect)
+  .directive('queryFormatter', ['$http', queryFormatter]);
+})();

rd_ui/app/scripts/filters.js

@@ -1,61 +1,75 @@
 var durationHumanize = function (duration) {
-    var humanized = "";
-    if (duration == undefined) {
-        humanized = "-";
-    } else if (duration < 60) {
-        humanized = Math.round(duration) + "s";
-    } else if (duration > 3600*24) {
-        var days = Math.round(parseFloat(duration) / 60.0 / 60.0 / 24.0);
-        humanized = days + "days";
-    } else if (duration >= 3600) {
-        var hours = Math.round(parseFloat(duration) / 60.0 / 60.0);
-        humanized = hours + "h";
-    } else {
-        var minutes = Math.round(parseFloat(duration) / 60.0);
-        humanized = minutes + "m";
-    }
-    return humanized;
-}
+  var humanized = "";
+  if (duration == undefined) {
+    humanized = "-";
+  } else if (duration < 60) {
+    humanized = Math.round(duration) + "s";
+  } else if (duration > 3600 * 24) {
+    var days = Math.round(parseFloat(duration) / 60.0 / 60.0 / 24.0);
+    humanized = days + "days";
+  } else if (duration >= 3600) {
+    var hours = Math.round(parseFloat(duration) / 60.0 / 60.0);
+    humanized = hours + "h";
+  } else {
+    var minutes = Math.round(parseFloat(duration) / 60.0);
+    humanized = minutes + "m";
+  }
+  return humanized;
+};
+
+var urlPattern = /(^|[\s\n]|<br\/?>)((?:https?|ftp):\/\/[\-A-Z0-9+\u0026\u2019@#\/%?=()~_|!:,.;]*[\-A-Z0-9+\u0026@#\/%=~()_|])/gi;
 
 angular.module('redash.filters', []).
-    filter('durationHumanize', function () {
-        return durationHumanize;
-    })
+  filter('durationHumanize', function () {
+    return durationHumanize;
+  })
 
-    .filter('refreshRateHumanize', function () {
-        return function (ttl) {
-            if (ttl==-1) {
-                return "Never";
-            } else {
-                return "Every " + durationHumanize(ttl);
-            }
-        }
-    })
+  .filter('refreshRateHumanize', function () {
+    return function (ttl) {
+      if (ttl == -1) {
+        return "Never";
+      } else {
+        return "Every " + durationHumanize(ttl);
+      }
+    }
+  })
 
-    .filter('toHuman', function() {
-        return function(text) {
-            return text.replace(/_/g, ' ').replace(/(?:^|\s)\S/g, function (a) {
-                return a.toUpperCase();
-            });
-        }
-    })
+  .filter('toHuman', function () {
+    return function (text) {
+      return text.replace(/_/g, ' ').replace(/(?:^|\s)\S/g, function (a) {
+        return a.toUpperCase();
+      });
+    }
+  })
 
-    .filter('colWidth', function () {
-        return function (widgetWidth) {
-            if (widgetWidth == 1) {
-                return 6;
-            }
-            return 12;
-        }
-    })
-    
-    .filter('capitalize', function () {
-        return function (text) {
-            if (text) {
-                return text[0].toUpperCase() + text.slice(1).toLowerCase();
-            } else {
-                return null;
-            }
-            
-        }
-    });
+  .filter('colWidth', function () {
+    return function (widgetWidth) {
+      if (widgetWidth == 1) {
+        return 6;
+      }
+      return 12;
+    }
+  })
+
+  .filter('capitalize', function () {
+    return function (text) {
+      if (text) {
+        return _.str.capitalize(text);
+      } else {
+        return null;
+      }
+
+    }
+  })
+
+  .filter('linkify', function () {
+    return function (text) {
+      return text.replace(urlPattern, "$1<a href='$2' target='_blank'>$2</a>");
+    };
+  })
+
+  .filter('markdown', ['$sce', function ($sce) {
+    return function (text) {
+      return $sce.trustAsHtml(marked(text));
+    }
+  }]);

rd_ui/app/scripts/ng_highchart.js

@@ -1,278 +1,341 @@
 (function () {
-    'use strict';
+  'use strict';
 
-    var defaultOptions = {
+  Highcharts.setOptions({
+    colors: ["#4572A7", "#AA4643", "#89A54E", "#80699B", "#3D96AE",
+      "#DB843D", "#92A8CD", "#A47D7C", "#B5CA92"]
+  });
+
+  var defaultOptions = {
+    title: {
+      "text": null
+    },
+    xAxis: {
+      type: 'datetime'
+    },
+    yAxis: [
+      {
         title: {
-            "text": null
+          text: null
         },
-        xAxis: {
-            type: 'datetime'
+        // showEmpty: true // by default
+      },
+      {
+        title: {
+          text: null
         },
-        yAxis: {
-            title: {
-                text: null
+        opposite: true,
+        showEmpty: false
+      }
+    ],
+
+
+    tooltip: {
+      valueDecimals: 2,
+      formatter: function () {
+        if (!this.points) {
+          this.points = [this.point];
+        }
+        ;
+
+        if (moment.isMoment(this.x)) {
+          var s = '<b>' + moment(this.x).format("DD/MM/YY HH:mm") + '</b>',
+              pointsCount = this.points.length;
+
+          $.each(this.points, function (i, point) {
+            s += '<br/><span style="color:' + point.series.color + '">' + point.series.name + '</span>: ' +
+                Highcharts.numberFormat(point.y);
+
+            if (pointsCount > 1 && point.percentage) {
+              s += " (" + Highcharts.numberFormat(point.percentage) + "%)";
             }
+          });
+        } else {
+          var points = this.points;
+          var name = points[0].key || points[0].name;
+
+          var s = "<b>" + name + "</b>";
+
+          $.each(points, function (i, point) {
+            if (points.length > 1) {
+              s += '<br/><span style="color:' + point.series.color + '">' + point.series.name + '</span>: ' + Highcharts.numberFormat(point.y);
+            } else {
+              s += ": " + Highcharts.numberFormat(point.y);
+              if (point.percentage < 100) {
+                s += ' (' + Highcharts.numberFormat(point.percentage) + '%)';
+              }
+            }
+          });
+        }
+
+        return s;
+      },
+      shared: true
+    },
+    exporting: {
+      chartOptions: {
+        title: {
+          text: ''
+        }
+      },
+      buttons: {
+        contextButton: {
+          menuItems: [
+            {
+              text: 'Toggle % Stacking',
+              onclick: function () {
+                var newStacking = "normal";
+                if (this.series[0].options.stacking == "normal") {
+                  newStacking = "percent";
+                }
+
+                _.each(this.series, function (series) {
+                  series.update({stacking: newStacking}, true);
+                });
+              }
+            },
+            {
+              text: 'Select All',
+              onclick: function () {
+                _.each(this.series, function (s) {
+                  s.setVisible(true, false);
+                });
+                this.redraw();
+              }
+            },
+            {
+              text: 'Unselect All',
+              onclick: function () {
+                _.each(this.series, function (s) {
+                  s.setVisible(false, false);
+                });
+                this.redraw();
+              }
+            },
+            {
+              text: 'Show Total',
+              onclick: function () {
+                var data = {};
+                _.each(this.series, function (s) {
+                  s.setVisible(false, false);
+                  _.each(s.data, function (p) {
+                    data[p.x] = data[p.x] || {'x': p.x, 'y': 0};
+                    data[p.x].y = data[p.x].y + p.y;
+                  });
+                });
+
+                this.addSeries({
+                  data: _.values(data),
+                  type: 'line',
+                  name: 'Total'
+                }, false)
+
+                this.redraw();
+              }
+            }
+          ]
+        }
+      }
+    },
+    credits: {
+      enabled: false
+    },
+    plotOptions: {
+      area: {
+        marker: {
+          enabled: false,
+          symbol: 'circle',
+          radius: 2,
+          states: {
+            hover: {
+              enabled: true
+            }
+          }
+        }
+      },
+      column: {
+        stacking: "normal",
+        pointPadding: 0,
+        borderWidth: 1,
+        groupPadding: 0,
+        shadow: false
+      },
+      line: {
+        marker: {
+          radius: 1
+        },
+        lineWidth: 2,
+        states: {
+          hover: {
+            lineWidth: 2,
+            marker: {
+              radius: 3
+            }
+          }
+        }
+      },
+      pie: {
+        allowPointSelect: true,
+        cursor: 'pointer',
+        dataLabels: {
+          enabled: true,
+          color: '#000000',
+          connectorColor: '#000000',
+          format: '<b>{point.name}</b>: {point.y} ({point.percentage:.1f} %)'
+        }
+      },
+      scatter: {
+        marker: {
+          radius: 5,
+          states: {
+            hover: {
+              enabled: true,
+              lineColor: 'rgb(100,100,100)'
+            }
+          }
         },
         tooltip: {
-            valueDecimals: 2,
-            formatter: function () {
-                if (!this.points) {
-                    this.points = [this.point];
-                };
+          headerFormat: '<b>{series.name}</b><br>',
+          pointFormat: '{point.x}, {point.y}'
+        }
+      }
+    },
+    series: []
+  };
 
-                if (moment.isMoment(this.x)) {
-                    var s = '<b>' + moment(this.x).format("DD/MM/YY HH:mm") + '</b>',
-                        pointsCount = this.points.length;
+  angular.module('highchart', [])
+      .directive('chart', ['$timeout', function ($timeout) {
+        return {
+          restrict: 'E',
+          template: '<div></div>',
+          scope: {
+            options: "=options",
+            series: "=series"
+          },
+          transclude: true,
+          replace: true,
 
-                    $.each(this.points, function (i, point) {
-                        s += '<br/><span style="color:' + point.series.color + '">' + point.series.name + '</span>: ' +
-                            Highcharts.numberFormat(point.y);
-
-                        if (pointsCount > 1 && point.percentage) {
-                            s += " (" + Highcharts.numberFormat(point.percentage) + "%)";
-                        }
-                    });
-                } else {
-                    var points = this.points;
-                    var name = points[0].key || points[0].name;
-
-                    var s = "<b>" + name + "</b>";
-
-                    $.each(points, function (i, point) {
-                        if (points.length > 1) {
-                            s += '<br/><span style="color:' + point.series.color + '">' + point.series.name + '</span>: ' + Highcharts.numberFormat(point.y);
-                        } else {
-                            s += ": " + Highcharts.numberFormat(point.y);
-                            if (point.percentage < 100) {
-                                s += ' (' +Highcharts.numberFormat(point.percentage) +  '%)';
-                            }
-                        }
-                    });
-                }
-
-                return s;
-            },
-            shared: true
-        },
-        exporting: {
-            chartOptions: {
-                title: {
-                    text: ''
-                }
-            },
-            buttons: {
-                contextButton: {
-                    menuItems: [
-                        {
-                            text: 'Toggle % Stacking',
-                            onclick: function () {
-                                var newStacking = "normal";
-                                if (this.series[0].options.stacking == "normal") {
-                                    newStacking = "percent";
-                                }
-
-                                _.each(this.series, function (series) {
-                                    series.update({stacking: newStacking}, true);
-                                });
-                            }
-                        }
-                    ]
-                }
-            }
-        },
-        credits: {
-            enabled: false
-        },
-        plotOptions: {
-            area: {
-                marker: {
-                    enabled: false,
-                    symbol: 'circle',
-                    radius: 2,
-                    states: {
-                        hover: {
-                            enabled: true
-                        }
-                    }
-                }
-            },
-            column: {
-                stacking: "normal",
-                pointPadding: 0,
-                borderWidth: 1,
-                groupPadding: 0,
-                shadow: false
-            },
-            line: {
-                marker: {
-                    radius: 1
-                },
-                lineWidth: 2,
-                states: {
-                    hover: {
-                        lineWidth: 2,
-                        marker: {
-                            radius: 3
-                        }
-                    }
-                }
-            },
-            pie: {
-                allowPointSelect: true,
-                cursor: 'pointer',
-                dataLabels: {
-                    enabled: true,
-                    color: '#000000',
-                    connectorColor: '#000000',
-                    format: '<b>{point.name}</b>: {point.y} ({point.percentage:.1f} %)'
-                }
-            },
-            scatter: {
-                marker: {
-                    radius: 5,
-                    states: {
-                        hover: {
-                            enabled: true,
-                            lineColor: 'rgb(100,100,100)'
-                        }
-                    }
-                },
-                tooltip: {
-                    headerFormat: '<b>{series.name}</b><br>',
-                    pointFormat: '{point.x}, {point.y}'
-                }
-            }
-        },
-        series: []
-    };
-
-    angular.module('highchart', [])
-        .directive('chart', ['$timeout', function ($timeout) {
-            return {
-                restrict: 'E',
-                template: '<div></div>',
-                scope: {
-                    options: "=options",
-                    series: "=series"
-                },
-                transclude: true,
-                replace: true,
-
-                link: function (scope, element, attrs) {
-                    var chartsDefaults = {
-                        chart: {
-                            renderTo: element[0],
-                            type: attrs.type || null,
-                            height: attrs.height || null,
-                            width: attrs.width || null
-                        }
-                    };
-
-                    var chartOptions = $.extend(true, {}, defaultOptions, chartsDefaults);
-
-                    // $timeout makes sure that this function invoked after the DOM ready. When draw/init
-                    // invoked after the DOM is ready, we see first an empty HighCharts objects and later
-                    // they get filled up. Which gives the feeling that the charts loading faster (otherwise
-                    // we stare at an empty screen until the HighCharts object is ready).
-                    $timeout(function(){
-                        // Update when options change
-                        scope.$watch('options', function (newOptions) {
-                            initChart(newOptions);
-                        }, true);
-
-                        //Update when charts data changes
-                        scope.$watch(function () {
-                            // TODO: this might be an issue in case the series change, but they stay
-                            // with the same length
-                            return (scope.series && scope.series.length) || 0;
-                        }, function (length) {
-                            if (!length || length == 0) {
-                                scope.chart.showLoading();
-                            } else {
-                                drawChart();
-                            };
-                        }, true);
-                    });
-
-                    function initChart(options) {
-                        if (scope.chart) {
-                            scope.chart.destroy();
-                        };
-
-                        $.extend(true, chartOptions, options);
-
-                        scope.chart = new Highcharts.Chart(chartOptions);
-                        drawChart();
-                    }
-
-                    function drawChart() {
-                        while (scope.chart.series.length > 0) {
-                            scope.chart.series[0].remove(false);
-                        };
-
-                        if (scope.series.length > 0 && _.some(scope.series[0].data, function (p) {
-                            return (angular.isString(p.x) || angular.isDefined(p.name));
-                        })) {
-                            scope.chart.xAxis[0].update({type: 'category'});
-
-                            if (!angular.isDefined(scope.series[0].data[0].name)) {
-                                // We need to make sure that for each category, each series has a value.
-                                var categories = _.union.apply(this, _.map(scope.series, function (s) {
-                                    return _.pluck(s.data, 'x')
-                                }));
-
-                                _.each(scope.series, function (s) {
-                                    // TODO: move this logic to Query#getChartData
-                                    var yValues = _.groupBy(s.data, 'x');
-
-                                    var newData = _.map(categories, function (category) {
-                                        return {
-                                            name: category,
-                                            y: yValues[category] && yValues[category][0].y
-                                        }
-                                    });
-
-                                    if (categories.length == 1) {
-                                        newData = _.sortBy(newData, 'y').reverse();
-                                    };
-
-                                    s.data = newData;
-                                });
-                            }
-                        } else {
-                            scope.chart.xAxis[0].update({type: 'datetime'});
-                        }
-
-                        scope.chart.counters.color = 0;
-
-                        _.each(scope.series, function (s) {
-                            // here we override the series with the visualization config
-                            s = _.extend(s, chartOptions['series']);
-
-                            if (s.type == 'area') {
-                                _.each(s.data, function (p) {
-                                    // This is an insane hack: somewhere deep in HighChart's code,
-                                    // when you stack areas, it tries to convert the string representation
-                                    // of point's x into a number. With the default implementation of toString
-                                    // it fails....
-
-                                    if (moment.isMoment(p.x)) {
-                                        p.x.toString = function () {
-                                            return String(this.toDate().getTime());
-                                        };
-                                    }
-                                });
-                            };
-
-                            scope.chart.addSeries(s, false);
-                        });
-
-                        scope.chart.redraw();
-                        scope.chart.hideLoading();
-                    }
-
-                }
+          link: function (scope, element, attrs) {
+            var chartsDefaults = {
+              chart: {
+                renderTo: element[0],
+                type: attrs.type || null,
+                height: attrs.height || null,
+                width: attrs.width || null
+              }
             };
 
-        }]);
+            var chartOptions = $.extend(true, {}, defaultOptions, chartsDefaults);
+
+            // $timeout makes sure that this function invoked after the DOM ready. When draw/init
+            // invoked after the DOM is ready, we see first an empty HighCharts objects and later
+            // they get filled up. Which gives the feeling that the charts loading faster (otherwise
+            // we stare at an empty screen until the HighCharts object is ready).
+            $timeout(function () {
+              // Update when options change
+              scope.$watch('options', function (newOptions) {
+                initChart(newOptions);
+              }, true);
+
+              //Update when charts data changes
+              scope.$watchCollection('series', function (series) {
+                if (!series || series.length == 0) {
+                  scope.chart.showLoading();
+                } else {
+                  drawChart();
+                }
+                ;
+              });
+            });
+
+            function initChart(options) {
+              if (scope.chart) {
+                scope.chart.destroy();
+              }
+              ;
+
+              $.extend(true, chartOptions, options);
+
+              scope.chart = new Highcharts.Chart(chartOptions);
+              drawChart();
+            }
+
+            function drawChart() {
+              while (scope.chart.series.length > 0) {
+                scope.chart.series[0].remove(false);
+              };
+
+              if (!('xAxis' in chartOptions && 'type' in chartOptions['xAxis'])) {
+                if (scope.series.length > 0 && _.some(scope.series[0].data, function (p) {
+                  return (angular.isString(p.x) || angular.isDefined(p.name));
+                })) {
+                  chartOptions['xAxis'] = chartOptions['xAxis'] || {};
+                  chartOptions['xAxis']['type'] = 'category';
+                } else {
+                  chartOptions['xAxis'] = chartOptions['xAxis'] || {};
+                  chartOptions['xAxis']['type'] = 'datetime';
+                }
+              }
+
+              if (chartOptions['xAxis']['type'] == 'category' || chartOptions['series']['type']=='pie') {
+                if (!angular.isDefined(scope.series[0].data[0].name)) {
+                  // We need to make sure that for each category, each series has a value.
+                  var categories = _.union.apply(this, _.map(scope.series, function (s) {
+                    return _.pluck(s.data, 'x')
+                  }));
+
+                  _.each(scope.series, function (s) {
+                    // TODO: move this logic to Query#getChartData
+                    var yValues = _.groupBy(s.data, 'x');
+
+                    var newData = _.map(categories, function (category) {
+                      return {
+                        name: category,
+                        y: (yValues[category] && yValues[category][0].y) || 0
+                      }
+                    });
+
+                    if (categories.length == 1) {
+                      newData = _.sortBy(newData, 'y').reverse();
+                    }
+                    ;
+
+                    s.data = newData;
+                  });
+                }
+              }
+
+              scope.chart.counters.color = 0;
+
+              _.each(scope.series, function (s) {
+                // here we override the series with the visualization config
+                s = _.extend(s, chartOptions['series']);
+
+                if (s.type == 'area') {
+                  _.each(s.data, function (p) {
+                    // This is an insane hack: somewhere deep in HighChart's code,
+                    // when you stack areas, it tries to convert the string representation
+                    // of point's x into a number. With the default implementation of toString
+                    // it fails....
+
+                    if (moment.isMoment(p.x)) {
+                      p.x.toString = function () {
+                        return String(this.toDate().getTime());
+                      };
+                    }
+                  });
+                }
+                ;
+
+                scope.chart.addSeries(s, false);
+              });
+
+              scope.chart.redraw();
+              scope.chart.hideLoading();
+            }
+
+          }
+        };
+
+      }]);
 })();

rd_ui/app/scripts/ng_smart_table.js

@@ -217,7 +217,7 @@
                             element.html('<div editable-cell="" row="dataRow" column="column" type="column.type"></div>');
                             compile(element.contents())(scope);
                         } else {
-                            element.text(scope.formatedValue);
+                            element.html(scope.formatedValue);
                         }
                     }
 

rd_ui/app/scripts/services/dashboards.js

@@ -2,7 +2,7 @@
     var Dashboard = function($resource) {
         var resource = $resource('/api/dashboards/:slug', {slug: '@slug'});
         resource.prototype.canEdit = function() {
-            return currentUser.is_admin || currentUser.canEdit(this);
+            return currentUser.hasPermission('admin') || currentUser.canEdit(this);
         }
         return resource;
     }

rd_ui/app/scripts/services/notifications.js

@@ -1,5 +1,5 @@
 (function () {
-    var notifications = function () {
+    var notifications = function (Events) {
         var notificationService = {};
         var lastNotification = null;
 
@@ -40,6 +40,7 @@
             notification.onclick = function () {
                 window.focus();
                 this.cancel();
+                Events.record(currentUser, 'click', 'notification');
             };
 
             notification.show()
@@ -49,5 +50,5 @@
     }
 
     angular.module('redash.services')
-        .factory('notifications', notifications);
+        .factory('notifications', ['Events', notifications]);
 })();

rd_ui/app/scripts/services/resources.js

@@ -0,0 +1,454 @@
+(function () {
+  var QueryResult = function ($resource, $timeout, $q) {
+    var QueryResultResource = $resource('/api/query_results/:id', {id: '@id'}, {'post': {'method': 'POST'}});
+    var Job = $resource('/api/jobs/:id', {id: '@id'});
+
+    var updateFunction = function (props) {
+      angular.extend(this, props);
+      if ('query_result' in props) {
+        this.status = "done";
+        this.filters = undefined;
+        this.filterFreeze = undefined;
+
+        var columnTypes = {};
+
+        _.each(this.query_result.data.rows, function (row) {
+          _.each(row, function (v, k) {
+            if (angular.isNumber(v)) {
+              columnTypes[k] = 'float';
+            } else if (_.isString(v) && v.match(/^\d{4}-\d{2}-\d{2}T/)) {
+              row[k] = moment(v);
+              columnTypes[k] = 'datetime';
+            } else if (_.isString(v) && v.match(/^\d{4}-\d{2}-\d{2}/)) {
+              row[k] = moment(v);
+              columnTypes[k] = 'date';
+            }
+          }, this);
+        }, this);
+
+        _.each(this.query_result.data.columns, function(column) {
+          if (columnTypes[column.name]) {
+            column.type = columnTypes[column.name];
+          }
+        });
+
+        this.deferred.resolve(this);
+      } else if (this.job.status == 3) {
+        this.status = "processing";
+      } else {
+        this.status = undefined;
+      }
+    }
+
+    function QueryResult(props) {
+      this.deferred = $q.defer();
+      this.job = {};
+      this.query_result = {};
+      this.status = "waiting";
+      this.filters = undefined;
+      this.filterFreeze = undefined;
+
+      this.updatedAt = moment();
+
+      if (props) {
+        updateFunction.apply(this, [props]);
+      }
+    }
+
+    var statuses = {
+      1: "waiting",
+      2: "processing",
+      3: "done",
+      4: "failed"
+    }
+
+    QueryResult.prototype.update = updateFunction;
+
+    QueryResult.prototype.getId = function () {
+      var id = null;
+      if ('query_result' in this) {
+        id = this.query_result.id;
+      }
+      return id;
+    }
+
+    QueryResult.prototype.cancelExecution = function () {
+      Job.delete({id: this.job.id});
+    }
+
+    QueryResult.prototype.getStatus = function () {
+      return this.status || statuses[this.job.status];
+    }
+
+    QueryResult.prototype.getError = function () {
+      // TODO: move this logic to the server...
+      if (this.job.error == "None") {
+        return undefined;
+      }
+
+      return this.job.error;
+    }
+
+    QueryResult.prototype.getUpdatedAt = function () {
+      return this.query_result.retrieved_at || this.job.updated_at * 1000.0 || this.updatedAt;
+    }
+
+    QueryResult.prototype.getRuntime = function () {
+      return this.query_result.runtime;
+    }
+
+    QueryResult.prototype.getRawData = function () {
+      if (!this.query_result.data) {
+        return null;
+      }
+
+      var data = this.query_result.data.rows;
+
+      return data;
+    }
+
+    QueryResult.prototype.getData = function () {
+      if (!this.query_result.data) {
+        return null;
+      }
+
+      var filterValues = function (filters) {
+        if (!filters) {
+          return null;
+        }
+
+        return _.reduce(filters, function (str, filter) {
+          return str + filter.current;
+        }, "")
+      }
+
+      var filters = this.getFilters();
+      var filterFreeze = filterValues(filters);
+
+      if (this.filterFreeze != filterFreeze) {
+        this.filterFreeze = filterFreeze;
+
+        if (filters) {
+          this.filteredData = _.filter(this.query_result.data.rows, function (row) {
+            return _.reduce(filters, function (memo, filter) {
+              if (!_.isArray(filter.current)) {
+                filter.current = [filter.current];
+              };
+
+              return (memo && _.some(filter.current, function(v) {
+                // We compare with either the value or the String representation of the value,
+                // because Select2 casts true/false to "true"/"false".
+                return v == row[filter.name] || String(row[filter.name]) == v
+              }));
+            }, true);
+          });
+        } else {
+          this.filteredData = this.query_result.data.rows;
+        }
+      }
+
+      return this.filteredData;
+    }
+
+    QueryResult.prototype.getChartData = function (mapping) {
+      var series = {};
+
+      _.each(this.getData(), function (row) {
+        var point = {};
+        var seriesName = undefined;
+        var xValue = 0;
+        var yValues = {};
+
+        _.each(row, function (value, definition) {
+          var name = definition.split("::")[0];
+          var type = definition.split("::")[1];
+          if (mapping) {
+            type = mapping[definition];
+          }
+
+          if (type == 'unused') {
+            return;
+          }
+
+          if (type == 'x') {
+            xValue = value;
+            point[type] = value;
+          }
+          if (type == 'y') {
+            if (value == null) {
+              value = 0;
+            }
+            yValues[name] = value;
+            point[type] = value;
+          }
+
+          if (type == 'series') {
+            seriesName = String(value);
+          }
+
+          if (type == 'multi-filter') {
+            seriesName = String(value);
+          }
+        });
+
+        var addPointToSeries = function (seriesName, point) {
+          if (series[seriesName] == undefined) {
+            series[seriesName] = {
+              name: seriesName,
+              type: 'column',
+              data: []
+            }
+          }
+
+          series[seriesName]['data'].push(point);
+        }
+
+        if (seriesName === undefined) {
+          _.each(yValues, function (yValue, seriesName) {
+            addPointToSeries(seriesName, {'x': xValue, 'y': yValue});
+          });
+        } else {
+          addPointToSeries(seriesName, point);
+        }
+      });
+
+      _.each(series, function (series) {
+        series.data = _.sortBy(series.data, 'x');
+      });
+
+      return _.values(series);
+    };
+
+    QueryResult.prototype.getColumns = function () {
+      if (this.columns == undefined && this.query_result.data) {
+        this.columns = this.query_result.data.columns;
+      }
+
+      return this.columns;
+    }
+
+    QueryResult.prototype.getColumnNames = function () {
+      if (this.columnNames == undefined && this.query_result.data) {
+        this.columnNames = _.map(this.query_result.data.columns, function (v) {
+          return v.name;
+        });
+      }
+
+      return this.columnNames;
+    }
+
+    QueryResult.prototype.getColumnNameWithoutType = function (column) {
+      var parts = column.split('::');
+      if (parts[0] == "" && parts.length == 2) {
+        return parts[1];
+      }
+      return parts[0];
+    };
+
+    var charConversionMap = {
+      '__pct': /%/g,
+      '_': / /g,
+      '__qm': /\?/g,
+      '__brkt': /[\(\)\[\]]/g,
+      '__dash': /-/g,
+      '__amp': /&/g,
+      '__sl': /\//g,
+      '__fsl': /\\/g,
+    };
+
+    QueryResult.prototype.getColumnCleanName = function (column) {
+      var name = this.getColumnNameWithoutType(column);
+
+      if (name != '') {
+        _.each(charConversionMap, function(regex, replacement) {
+          name = name.replace(regex, replacement);
+        });
+      }
+
+      return name;
+    }
+
+    QueryResult.prototype.getColumnFriendlyName = function (column) {
+      return this.getColumnNameWithoutType(column).replace(/(?:^|\s)\S/g, function (a) {
+        return a.toUpperCase();
+      });
+    }
+
+    QueryResult.prototype.getColumnCleanNames = function () {
+      return _.map(this.getColumnNames(), function (col) {
+        return this.getColumnCleanName(col);
+      }, this);
+    }
+
+    QueryResult.prototype.getColumnFriendlyNames = function () {
+      return _.map(this.getColumnNames(), function (col) {
+        return this.getColumnFriendlyName(col);
+      }, this);
+    }
+
+    QueryResult.prototype.getFilters = function () {
+      if (!this.filters) {
+        this.prepareFilters();
+      }
+
+      return this.filters;
+    };
+
+    QueryResult.prototype.prepareFilters = function () {
+      var filters = [];
+      var filterTypes = ['filter', 'multi-filter'];
+      _.each(this.getColumnNames(), function (col) {
+        var type = col.split('::')[1]
+        if (_.contains(filterTypes, type)) {
+          // filter found
+          var filter = {
+            name: col,
+            friendlyName: this.getColumnFriendlyName(col),
+            values: [],
+            multiple: (type=='multi-filter')
+          }
+          filters.push(filter);
+        }
+      }, this);
+
+      _.each(this.getRawData(), function (row) {
+        _.each(filters, function (filter) {
+          filter.values.push(row[filter.name]);
+          if (filter.values.length == 1) {
+            filter.current = row[filter.name];
+          }
+        })
+      });
+
+      _.each(filters, function(filter) {
+        filter.values = _.uniq(filter.values);
+      });
+
+      this.filters = filters;
+    }
+
+    var refreshStatus = function (queryResult, query, ttl) {
+      Job.get({'id': queryResult.job.id}, function (response) {
+        queryResult.update(response);
+
+        if (queryResult.getStatus() == "processing" && queryResult.job.query_result_id && queryResult.job.query_result_id != "None") {
+          QueryResultResource.get({'id': queryResult.job.query_result_id}, function (response) {
+            queryResult.update(response);
+          });
+        } else if (queryResult.getStatus() != "failed") {
+          $timeout(function () {
+            refreshStatus(queryResult, query, ttl);
+          }, 3000);
+        }
+      })
+    }
+
+    QueryResult.getById = function (id) {
+      var queryResult = new QueryResult();
+
+      QueryResultResource.get({'id': id}, function (response) {
+        queryResult.update(response);
+      });
+
+      return queryResult;
+    };
+
+    QueryResult.prototype.toPromise = function() {
+      return this.deferred.promise;
+    }
+
+    QueryResult.get = function (data_source_id, query, ttl) {
+      var queryResult = new QueryResult();
+
+      QueryResultResource.post({'data_source_id': data_source_id, 'query': query, 'ttl': ttl}, function (response) {
+        queryResult.update(response);
+
+        if ('job' in response) {
+          refreshStatus(queryResult, query, ttl);
+        }
+      });
+
+      return queryResult;
+    }
+
+    return QueryResult;
+  };
+
+  var Query = function ($resource, QueryResult, DataSource) {
+    var Query = $resource('/api/queries/:id', {id: '@id'});
+
+    Query.newQuery = function () {
+      return new Query({
+        query: "",
+        name: "New Query",
+        ttl: -1,
+        user: currentUser
+      });
+    };
+
+    Query.prototype.getSourceLink = function () {
+      return '/queries/' + this.id + '/source';
+    };
+
+    Query.prototype.getQueryResult = function (ttl) {
+      if (ttl == undefined) {
+        ttl = this.ttl;
+      }
+
+      if (this.latest_query_data && ttl != 0) {
+        if (!this.queryResult) {
+          this.queryResult = new QueryResult({'query_result': this.latest_query_data});
+        }
+      } else if (this.latest_query_data_id && ttl != 0) {
+        if (!this.queryResult) {
+          this.queryResult = QueryResult.getById(this.latest_query_data_id);
+        }
+      } else if (this.data_source_id) {
+        this.queryResult = QueryResult.get(this.data_source_id, this.query, ttl);
+      }
+
+      return this.queryResult;
+    };
+
+    Query.prototype.getQueryResultPromise = function() {
+      return this.getQueryResult().toPromise();
+    }
+
+    return Query;
+  };
+
+
+
+  var DataSource = function ($resource) {
+    var DataSourceResource = $resource('/api/data_sources/:id', {id: '@id'}, {'get': {'method': 'GET', 'cache': true, 'isArray': true}});
+
+    return DataSourceResource;
+  }
+
+  var Widget = function ($resource, Query) {
+    var WidgetResource = $resource('/api/widgets/:id', {id: '@id'});
+
+    WidgetResource.prototype.getQuery = function () {
+      if (!this.query && this.visualization) {
+         this.query = new Query(this.visualization.query);
+      }
+
+      return this.query;
+    };
+
+    WidgetResource.prototype.getName = function () {
+      if (this.visualization) {
+        return this.visualization.query.name + ' (' + this.visualization.name + ')';
+      }
+      return _.str.truncate(this.text, 20);
+    };
+
+    return WidgetResource;
+  }
+
+  angular.module('redash.services')
+      .factory('QueryResult', ['$resource', '$timeout', '$q', QueryResult])
+      .factory('Query', ['$resource', 'QueryResult', 'DataSource', Query])
+      .factory('DataSource', ['$resource', DataSource])
+      .factory('Widget', ['$resource', 'Query', Widget]);
+})();

rd_ui/app/scripts/services/services.js

@@ -1,289 +1,52 @@
 (function () {
-    var QueryResult = function($resource, $timeout) {
-        var QueryResultResource = $resource('/api/query_results/:id', {id: '@id'}, {'post': {'method': 'POST'}});
-        var Job = $resource('/api/jobs/:id', {id: '@id'});
+  'use strict'
 
-        var updateFunction = function (props) {
-            angular.extend(this, props);
-            if ('query_result' in props) {
-                this.status = "done";
+  function KeyboardShortcuts() {
+    this.bind = function bind(keymap) {
+      _.forEach(keymap, function (fn, key) {
+        Mousetrap.bindGlobal(key, function (e) {
+          e.preventDefault();
+          fn();
+        });
+      });
 
-                _.each(this.query_result.data.rows, function (row) {
-                    _.each(row, function (v, k) {
-                        if (_.isString(v) && v.match(/^\d{4}-\d{2}-\d{2}/)) {
-                            row[k] = moment(v);
-                        }
-                    });
-                });
-            } else if (this.job.status == 3) {
-                this.status = "processing";
-            } else {
-                this.status = undefined;
-            }
-        }
+    }
 
-        function QueryResult(props) {
-            this.job = {};
-            this.query_result = {};
-            this.status = "waiting";
+    this.unbind = function unbind(keymap) {
+      _.forEach(keymap, function (fn, key) {
+        Mousetrap.unbind(key);
+      });
+    }
+  }
 
-            this.updatedAt = moment();
+  function Events($http) {
+    this.events = [];
 
-            if (props) {
-                updateFunction.apply(this, [props]);
-            }
-        }
+    this.post = _.debounce(function() {
+      var events = this.events;
+      this.events = [];
 
-        var statuses = {
-            1: "waiting",
-            2: "processing",
-            3: "done",
-            4: "failed"
-        }
+      $http.post('/api/events', events);
 
-        QueryResult.prototype.update = updateFunction;
+    }, 1000);
 
-        QueryResult.prototype.getId = function() {
-            var id = null;
-            if ('query_result' in this) {
-                id = this.query_result.id;
-            }
-            return id;
-        }
+    this.record = function (user, action, object_type, object_id, additional_properties) {
 
-        QueryResult.prototype.cancelExecution = function() {
-            Job.delete({id: this.job.id});
-        }
+      var event = {
+        "user_id": user.id,
+        "action": action,
+        "object_type": object_type,
+        "object_id": object_id,
+        "timestamp": Date.now()/1000.0
+      };
+      _.extend(event, additional_properties);
+      this.events.push(event);
 
-        QueryResult.prototype.getStatus = function() {
-            return this.status || statuses[this.job.status];
-        }
-
-        QueryResult.prototype.getError = function() {
-            // TODO: move this logic to the server...
-            if (this.job.error == "None") {
-                return undefined;
-            }
-
-            return this.job.error;
-        }
-
-        QueryResult.prototype.getUpdatedAt = function() {
-            return this.query_result.retrieved_at || this.job.updated_at*1000.0 || this.updatedAt;
-        }
-
-        QueryResult.prototype.getRuntime = function() {
-            return this.query_result.runtime;
-        }
-
-        QueryResult.prototype.getData = function() {
-            if (!this.query_result.data) {
-                return null;
-            }
-
-            var data = this.query_result.data.rows;
-            return data;
-        }
-
-        QueryResult.prototype.getChartData = function () {
-            var series = {};
-
-            _.each(this.getData(), function (row) {
-                var point = {};
-                var seriesName = undefined;
-                var xValue = 0;
-                var yValues = {};
-
-                _.each(row, function (value, definition) {
-                    var type = definition.split("::")[1];
-                    var name = definition.split("::")[0];
-
-                    if (type == 'x') {
-                        xValue = value;
-                        point[type] = value;
-                    }
-
-                    if (type == 'y') {
-                        yValues[name] = value;
-                        point[type] = value;
-                    }
-
-                    if (type == 'series') {
-                        seriesName = String(value);
-                    }
-                });
-
-                var addPointToSeries = function(seriesName, point) {
-                    if (series[seriesName] == undefined) {
-                        series[seriesName] = {
-                            name: seriesName,
-                            type: 'column',
-                            data: []
-                        }
-                    }
-
-                    series[seriesName]['data'].push(point);
-                }
-
-                if (seriesName === undefined) {
-                    _.each(yValues, function(yValue, seriesName) {
-                        addPointToSeries(seriesName, {'x': xValue, 'y': yValue});
-                    });
-                } else {
-                    addPointToSeries(seriesName, point);
-                }
-            });
-
-            _.each(series, function(series) {
-                series.data = _.sortBy(series.data, 'x');
-            });
-
-            return _.values(series);
-        };
-
-        QueryResult.prototype.getColumns = function () {
-            if (this.columns == undefined) {
-                this.columns = _.map(this.query_result.data.columns, function(v) {
-                    return v.name;
-                })
-            }
-
-            return this.columns;
-        }
-
-        QueryResult.prototype.getColumnCleanName = function (column) {
-            var parts = column.split('::');
-            var name = parts[1];
-            if (parts[0] != '') {
-                // TODO: it's probably time to generalize this.
-                // see also getColumnFriendlyName
-                name = parts[0].replace(/%/g, '__pct').replace(/ /g, '_').replace(/\?/g,'');
-            }
-            return name;
-        }
-
-        QueryResult.prototype.getColumnFriendlyName = function (column) {
-            return this.getColumnCleanName(column).replace('__pct', '%').replace(/_/g, ' ').replace(/(?:^|\s)\S/g, function (a) {
-                return a.toUpperCase();
-            });
-        }
-
-        QueryResult.prototype.getColumnCleanNames = function () {
-            return _.map(this.getColumns(), function (col) {
-                return this.getColumnCleanName(col);
-            }, this);
-        }
-
-        QueryResult.prototype.getColumnFriendlyNames = function () {
-            return _.map(this.getColumns(), function (col) {
-                return this.getColumnFriendlyName(col);
-            }, this);
-        }
-
-        QueryResult.prototype.getFilters = function () {
-            var filterNames = [];
-            _.each(this.getColumns(), function (col) {
-                if (col.split('::')[1] == 'filter') {
-                    filterNames.push(col);
-                }
-            });
-
-            var filterValues = [];
-            _.each(this.getData(), function (row) {
-                _.each(filterNames, function (filter, i) {
-                    if (filterValues[i] == undefined) {
-                        filterValues[i] = [];
-                    }
-                    filterValues[i].push(row[filter]);
-                })
-            });
-
-            var filters = _.map(filterNames, function (filter, i) {
-                var f = {
-                    name: filter,
-                    friendlyName: this.getColumnFriendlyName(filter),
-                    values: _.uniq(filterValues[i])
-                };
-
-                f.current = f.values[0];
-                return f;
-            }, this);
-
-            return filters;
-        };
-
-        var refreshStatus = function(queryResult, query, ttl) {
-            Job.get({'id': queryResult.job.id}, function(response) {
-                queryResult.update(response);
-
-                if (queryResult.getStatus() == "processing" && queryResult.job.query_result_id && queryResult.job.query_result_id != "None") {
-                    QueryResultResource.get({'id': queryResult.job.query_result_id}, function(response) {
-                        queryResult.update(response);
-                    });
-                } else if (queryResult.getStatus() != "failed") {
-                    $timeout(function () {
-                        refreshStatus(queryResult, query, ttl);
-                    }, 3000);
-                }
-            })
-        }
-
-        QueryResult.getById = function (id) {
-            var queryResult = new QueryResult();
-
-            QueryResultResource.get({'id': id}, function (response) {
-                queryResult.update(response);
-            });
-
-            return queryResult;
-        }
-
-        QueryResult.get = function (query, ttl) {
-            var queryResult = new QueryResult();
-
-            QueryResultResource.post({'query': query, 'ttl': ttl}, function (response) {
-                queryResult.update(response);
-
-                if ('job' in response) {
-                    refreshStatus(queryResult, query, ttl);
-                }
-            });
-
-            return queryResult;
-        }
-
-        return QueryResult;
+      this.post();
     };
+  }
 
-    var Query = function ($resource, QueryResult) {
-        var Query = $resource('/api/queries/:id', {id: '@id'});
-
-        Query.prototype.getQueryResult = function(ttl) {
-            if (ttl == undefined) {
-                ttl = this.ttl;
-            }
-
-
-            var queryResult = null;
-            if (this.latest_query_data && ttl != 0) {
-                queryResult = new QueryResult({'query_result': this.latest_query_data});
-            } else if (this.latest_query_data_id && ttl != 0) {
-                queryResult = QueryResult.getById(this.latest_query_data_id);
-            } else {
-                queryResult = QueryResult.get(this.query, ttl);
-            }
-
-            return queryResult;
-        };
-
-        Query.prototype.getHash = function() {
-            return this.query;
-        };
-
-        return Query;
-    };
-
-    angular.module('redash.services', [])
-        .factory('QueryResult', ['$resource', '$timeout', QueryResult])
-        .factory('Query', ['$resource', 'QueryResult', Query])
-})();
+  angular.module('redash.services', [])
+      .service('KeyboardShortcuts', [KeyboardShortcuts])
+      .service('Events', ['$http', Events])
+})();

rd_ui/app/scripts/visualizations/base.js

@@ -1,149 +1,213 @@
 (function () {
-    var VisualizationProvider = function() {
-        this.visualizations = {};
-        this.visualizationTypes = {};
-        var defaultConfig = {
-            defaultOptions: {},
-            skipTypes: false,
-            editorTemplate: null
+  var VisualizationProvider = function () {
+    this.visualizations = {};
+    this.visualizationTypes = {};
+    var defaultConfig = {
+      defaultOptions: {},
+      skipTypes: false,
+      editorTemplate: null
+    }
+
+    this.registerVisualization = function (config) {
+      var visualization = _.extend({}, defaultConfig, config);
+
+      // TODO: this is prone to errors; better refactor.
+      if (_.isEmpty(this.visualizations)) {
+        this.defaultVisualization = visualization;
+      }
+
+      this.visualizations[config.type] = visualization;
+
+      if (!config.skipTypes) {
+        this.visualizationTypes[config.name] = config.type;
+      }
+      ;
+    };
+
+    this.getSwitchTemplate = function (property) {
+      var pattern = /(<[a-zA-Z0-9-]*?)( |>)/
+
+      var mergedTemplates = _.reduce(this.visualizations, function (templates, visualization) {
+        if (visualization[property]) {
+          var ngSwitch = '$1 ng-switch-when="' + visualization.type + '" $2';
+          var template = visualization[property].replace(pattern, ngSwitch);
+
+          return templates + "\n" + template;
         }
 
-        this.registerVisualization = function(config) {
-            var visualization = _.extend({}, defaultConfig, config);
+        return templates;
+      }, "");
 
-            // TODO: this is prone to errors; better refactor.
-            if (_.isEmpty(this.visualizations)) {
-                this.defaultVisualization = visualization;
-            }
+      mergedTemplates = '<div ng-switch on="visualization.type">' + mergedTemplates + "</div>";
 
-            this.visualizations[config.type] = visualization;
+      return mergedTemplates;
+    }
 
-            if (!config.skipTypes) {
-                this.visualizationTypes[config.name] = config.type;
-            };
+    this.$get = ['$resource', function ($resource) {
+      var Visualization = $resource('/api/visualizations/:id', {id: '@id'});
+      Visualization.visualizations = this.visualizations;
+      Visualization.visualizationTypes = this.visualizationTypes;
+      Visualization.renderVisualizationsTemplate = this.getSwitchTemplate('renderTemplate');
+      Visualization.editorTemplate = this.getSwitchTemplate('editorTemplate');
+      Visualization.defaultVisualization = this.defaultVisualization;
+
+      return Visualization;
+    }];
+  };
+
+  var VisualizationRenderer = function ($location, Visualization) {
+    return {
+      restrict: 'E',
+      scope: {
+        visualization: '=',
+        queryResult: '='
+      },
+      // TODO: using switch here (and in the options editor) might introduce errors and bad
+      // performance wise. It's better to eventually show the correct template based on the
+      // visualization type and not make the browser render all of them.
+      template: '<filters></filters>\n' + Visualization.renderVisualizationsTemplate,
+      replace: false,
+      link: function (scope) {
+        scope.select2Options = {
+          width: '50%'
         };
 
-        this.getSwitchTemplate = function(property) {
-            var pattern = /(<[a-zA-Z0-9-]*?)( |>)/
-
-            var mergedTemplates = _.reduce(this.visualizations, function(templates, visualization) {
-                if (visualization[property]) {
-                    var ngSwitch = '$1 ng-switch-when="' + visualization.type + '" $2';
-                    var template = visualization[property].replace(pattern, ngSwitch);
-
-                    return templates + "\n" + template;
-                }
-
-                return templates;
-            }, "");
-
-            mergedTemplates = '<div ng-switch on="visualization.type">'+ mergedTemplates + "</div>";
-
-            return mergedTemplates;
+        function readURL() {
+          var searchFilters = angular.fromJson($location.search().filters);
+          if (searchFilters) {
+            _.forEach(scope.filters, function(filter) {
+              var value = searchFilters[filter.friendlyName];
+              if (value) {
+                filter.current = value;
+              }
+            });
+          }
         }
 
-        this.$get = ['$resource', function($resource) {
-            var Visualization = $resource('/api/visualizations/:id', {id: '@id'});
-            Visualization.visualizations = this.visualizations;
-            Visualization.visualizationTypes = this.visualizationTypes;
-            Visualization.renderVisualizationsTemplate = this.getSwitchTemplate('renderTemplate');
-            Visualization.editorTemplate = this.getSwitchTemplate('editorTemplate');
-            Visualization.defaultVisualization = this.defaultVisualization;
-
-            return Visualization;
-        }];
-    };
-
-    var VisualizationRenderer = function(Visualization) {
-        return {
-            restrict: 'E',
-            scope: {
-                visualization: '=',
-                queryResult: '='
-            },
-            // TODO: using switch here (and in the options editor) might introduce errors and bad
-            // performance wise. It's better to eventually show the correct template based on the
-            // visualization type and not make the browser render all of them.
-            template: Visualization.renderVisualizationsTemplate,
-            replace: false
-        }
-    };
-
-    var VisualizationOptionsEditor = function(Visualization) {
-        return {
-            restrict: 'E',
-            template: Visualization.editorTemplate,
-            replace: false
-        }
-    };
-
-    var EditVisualizationForm = function(Visualization, growl) {
-        return {
-            restrict: 'E',
-            templateUrl: '/views/visualizations/edit_visualization.html',
-            replace: true,
-            scope: {
-                query: '=',
-                queryResult: '=',
-                visualization: '=?'
-            },
-            link: function (scope, element, attrs) {
-                scope.visTypes = Visualization.visualizationTypes;
-
-                scope.newVisualization = function(q) {
-                    return {
-                        'query_id': q.id,
-                        'type': Visualization.defaultVisualization.type,
-                        'name': Visualization.defaultVisualization.name,
-                        'description': q.description || '',
-                        'options': Visualization.defaultVisualization.defaultOptions
-                    };
-                }
-
-                if (!scope.visualization) {
-                    // create new visualization
-                    // wait for query to load to populate with defaults
-                    var unwatch = scope.$watch('query', function (q) {
-                        if (q && q.id) {
-                            unwatch();
-
-                            scope.visualization = scope.newVisualization(q);
-                        }
-                    }, true);
-                }
-
-                scope.$watch('visualization.type', function (type, oldType) {
-                    // if not edited by user, set name to match type
-                    if (type && oldType != type && scope.visualization && !scope.visForm.name.$dirty) {
-                        // poor man's titlecase
-                        scope.visualization.name = scope.visualization.type[0] + scope.visualization.type.slice(1).toLowerCase();
-                    }
-                });
-
-                scope.submit = function () {
-                    Visualization.save(scope.visualization, function success(result) {
-                        growl.addSuccessMessage("Visualization saved");
-
-                        scope.visualization = scope.newVisualization(scope.query);
-
-                        var visIds = _.pluck(scope.query.visualizations, 'id');
-                        var index = visIds.indexOf(result.id);
-                        if (index > -1) {
-                            scope.query.visualizations[index] = result;
-                        } else {
-                            scope.query.visualizations.push(result);
-                        }
-                    }, function error() {
-                        growl.addErrorMessage("Visualization could not be saved");
-                    });
-                };
+        function updateURL(filters) {
+          var current = {};
+          _.each(filters, function(filter) {
+            if (filter.current) {
+              current[filter.friendlyName] = filter.current;
             }
-        }
-    };
+          });
 
-    angular.module('redash.visualization', [])
-        .provider('Visualization', VisualizationProvider)
-        .directive('visualizationRenderer', ['Visualization', VisualizationRenderer])
-        .directive('visualizationOptionsEditor', ['Visualization', VisualizationOptionsEditor])
-        .directive('editVisulatizationForm', ['Visualization', 'growl', EditVisualizationForm])
+          var newSearch = angular.extend($location.search(), {
+            filters: angular.toJson(current)
+          });
+          $location.search(newSearch);
+        }
+
+        scope.$watch('queryResult && queryResult.getFilters()', function (filters) {
+          if (filters) {
+            scope.filters = filters;
+
+            if (filters.length && false) {
+              readURL();
+
+              // start watching for changes and update URL
+              scope.$watch('filters', updateURL, true);
+            }
+          }
+        });
+      }
+    }
+  };
+
+  var VisualizationOptionsEditor = function (Visualization) {
+    return {
+      restrict: 'E',
+      template: Visualization.editorTemplate,
+      replace: false
+    }
+  };
+
+  var Filters = function () {
+    return {
+      restrict: 'E',
+      templateUrl: '/views/visualizations/filters.html'
+    }
+  }
+
+  var EditVisualizationForm = function (Events, Visualization, growl) {
+    return {
+      restrict: 'E',
+      templateUrl: '/views/visualizations/edit_visualization.html',
+      replace: true,
+      scope: {
+        query: '=',
+        queryResult: '=',
+        visualization: '=?',
+        openEditor: '=?',
+        onNewSuccess: '=?'
+      },
+      link: function (scope, element, attrs) {
+        scope.editRawOptions = currentUser.hasPermission('edit_raw_chart');
+        scope.visTypes = Visualization.visualizationTypes;
+
+        scope.newVisualization = function () {
+          return {
+            'type': Visualization.defaultVisualization.type,
+            'name': Visualization.defaultVisualization.name,
+            'description': '',
+            'options': Visualization.defaultVisualization.defaultOptions
+          };
+        }
+
+        if (!scope.visualization) {
+          var unwatch = scope.$watch('query.id', function (queryId) {
+            if (queryId) {
+              unwatch();
+
+              scope.visualization = scope.newVisualization();
+            }
+          });
+        }
+
+        scope.$watch('visualization.type', function (type, oldType) {
+          // if not edited by user, set name to match type
+          if (type && oldType != type && scope.visualization && !scope.visForm.name.$dirty) {
+            // poor man's titlecase
+            scope.visualization.name = scope.visualization.type[0] + scope.visualization.type.slice(1).toLowerCase();
+          }
+        });
+
+        scope.submit = function () {
+          if (scope.visualization.id) {
+            Events.record(currentUser, "update", "visualization", scope.visualization.id, {'type': scope.visualization.type});
+          } else {
+            Events.record(currentUser, "create", "visualization", null, {'type': scope.visualization.type});
+          }
+
+          scope.visualization.query_id = scope.query.id;
+
+          Visualization.save(scope.visualization, function success(result) {
+            growl.addSuccessMessage("Visualization saved");
+
+            scope.visualization = scope.newVisualization(scope.query);
+
+            var visIds = _.pluck(scope.query.visualizations, 'id');
+            var index = visIds.indexOf(result.id);
+            if (index > -1) {
+              scope.query.visualizations[index] = result;
+            } else {
+              // new visualization
+              scope.query.visualizations.push(result);
+              scope.onNewSuccess && scope.onNewSuccess(result);
+            }
+          }, function error() {
+            growl.addErrorMessage("Visualization could not be saved");
+          });
+        };
+      }
+    }
+  };
+
+
+  angular.module('redash.visualization', [])
+      .provider('Visualization', VisualizationProvider)
+      .directive('visualizationRenderer', ['$location', 'Visualization', VisualizationRenderer])
+      .directive('visualizationOptionsEditor', ['Visualization', VisualizationOptionsEditor])
+      .directive('filters', Filters)
+      .directive('editVisulatizationForm', ['Events', 'Visualization', 'growl', EditVisualizationForm])
 })();

rd_ui/app/scripts/visualizations/chart.js

@@ -1,106 +1,244 @@
 (function () {
-    var chartVisualization = angular.module('redash.visualization');
+  var chartVisualization = angular.module('redash.visualization');
 
-    chartVisualization.config(['VisualizationProvider', function(VisualizationProvider) {
-        var renderTemplate = '<chart-renderer options="visualization.options" query-result="queryResult"></chart-renderer>';
-        var editTemplate = '<chart-editor></chart-editor>';
-        var defaultOptions = {
-            'series': {
-                'type': 'column',
-                'stacking': null
-            }
+  chartVisualization.config(['VisualizationProvider', function (VisualizationProvider) {
+    var renderTemplate = '<chart-renderer options="visualization.options" query-result="queryResult"></chart-renderer>';
+    var editTemplate = '<chart-editor></chart-editor>';
+    var defaultOptions = {
+      'series': {
+//        'type': 'column',
+        'stacking': null
+      }
+    };
+
+    VisualizationProvider.registerVisualization({
+      type: 'CHART',
+      name: 'Chart',
+      renderTemplate: renderTemplate,
+      editorTemplate: editTemplate,
+      defaultOptions: defaultOptions
+    });
+  }]);
+
+  chartVisualization.directive('chartRenderer', function () {
+    return {
+      restrict: 'E',
+      scope: {
+        queryResult: '=',
+        options: '=?'
+      },
+      template: "<chart options='chartOptions' series='chartSeries' class='graph'></chart>",
+      replace: false,
+      controller: ['$scope', function ($scope) {
+        $scope.chartSeries = [];
+        $scope.chartOptions = {};
+
+        var reloadData = function(data) {
+          if (!data || ($scope.queryResult && $scope.queryResult.getData()) == null) {
+            $scope.chartSeries.splice(0, $scope.chartSeries.length);
+          } else {
+            $scope.chartSeries.splice(0, $scope.chartSeries.length);
+
+            _.each($scope.queryResult.getChartData($scope.options.columnMapping), function (s) {
+              var additional = {'stacking': 'normal'};
+              if ($scope.options.seriesOptions && $scope.options.seriesOptions[s.name]) {
+                additional = $scope.options.seriesOptions[s.name];
+                if (!additional.name || additional.name == "") {
+                  additional.name = s.name;
+                }
+              }
+              $scope.chartSeries.push(_.extend(s, additional));
+            });
+          };
         };
 
-        VisualizationProvider.registerVisualization({
-            type: 'CHART',
-            name: 'Chart',
-            renderTemplate: renderTemplate,
-            editorTemplate: editTemplate,
-            defaultOptions: defaultOptions
+        $scope.$watch('options', function (chartOptions) {
+          if (chartOptions) {
+            $scope.chartOptions = chartOptions;
+          }
         });
-    }]);
 
-    chartVisualization.directive('chartRenderer', function () {
-        return {
-            restrict: 'E',
-            scope: {
-                queryResult: '=',
-                options: '=?'
-            },
-            template: "<chart options='chartOptions' series='chartSeries' class='graph'></chart>",
-            replace: false,
-            controller: ['$scope', function ($scope) {
-                $scope.chartSeries = [];
-                $scope.chartOptions = {};
+        $scope.$watch('options.seriesOptions', function () {
+          reloadData(true);
+        }, true);
 
-                $scope.$watch('options', function(chartOptions) {
-                    if (chartOptions) {
-                        $scope.chartOptions = chartOptions;
-                    }
-                });
-                $scope.$watch('queryResult && queryResult.getData()', function (data) {
-                    if (!data || $scope.queryResult.getData() == null) {
-                        $scope.chartSeries.splice(0, $scope.chartSeries.length);
-                    } else {
-                        $scope.chartSeries.splice(0, $scope.chartSeries.length);
 
-                        _.each($scope.queryResult.getChartData(), function (s) {
-                            $scope.chartSeries.push(_.extend(s, {'stacking': 'normal'}));
-                        });
-                    }
-                });
-            }]
-        }
-    });
+        $scope.$watchCollection('options.columnMapping', function (chartOptions) {
+          reloadData(true);
+        });
 
-    chartVisualization.directive('chartEditor', function () {
-        return {
-            restrict: 'E',
-            templateUrl: '/views/visualizations/chart_editor.html',
-            link: function (scope, element, attrs) {
-                scope.seriesTypes = {
-                    'Line': 'line',
-                    'Column': 'column',
-                    'Area': 'area',
-                    'Scatter': 'scatter',
-                    'Pie': 'pie'
-                };
+        $scope.$watch('queryResult && queryResult.getData()', function (data) {
+          reloadData(data);
+        });
+      }]
+    };
+  });
 
-                scope.stackingOptions = {
-                    "None": "none",
-                    "Normal": "normal",
-                    "Percent": "percent"
-                };
+  chartVisualization.directive('chartEditor', function () {
+    return {
+      restrict: 'E',
+      templateUrl: '/views/visualizations/chart_editor.html',
+      link: function (scope, element, attrs) {
+        scope.seriesTypes = {
+          'Line': 'line',
+          'Column': 'column',
+          'Area': 'area',
+          'Scatter': 'scatter',
+          'Pie': 'pie'
+        };
 
-                scope.stacking = "none";
+        scope.globalSeriesType = 'column';
 
-                var chartOptionsUnwatch = null;
+        scope.stackingOptions = {
+          "None": "none",
+          "Normal": "normal",
+          "Percent": "percent"
+        };
 
-                scope.$watch('visualization', function (visualization) {
-                    if (visualization && visualization.type == 'CHART') {
-                        if (scope.visualization.options.series.stacking === null) {
-                            scope.stacking = "none";
-                        } else if (scope.visualization.options.series.stacking === undefined) {
-                            scope.stacking = "normal";
-                        } else {
-                            scope.stacking = scope.visualization.options.series.stacking;
-                        }
+        scope.xAxisOptions = {
+          "Date/Time": "datetime",
+          "Linear": "linear",
+          "Category": "category"
+        };
 
-                        chartOptionsUnwatch = scope.$watch("stacking", function (stacking) {
-                            if (stacking == "none") {
-                                scope.visualization.options.series.stacking = null;
-                            } else {
-                                scope.visualization.options.series.stacking = stacking;
-                            }
-                        });
-                    } else {
-                        if (chartOptionsUnwatch) {
-                            chartOptionsUnwatch();
-                            chartOptionsUnwatch = null;
-                        }
-                    }
-                });
+        scope.xAxisType = "datetime";
+        scope.stacking = "none";
+
+
+        scope.columnTypes = {
+          "X": "x",
+//          "X (Date time)": "x",
+//          "X (Linear)": "x-linear",
+//          "X (Category)": "x-category",
+          "Y": "y",
+          "Series": "series",
+          "Unused": "unused"
+        };
+
+        scope.series = [];
+
+        scope.columnTypeSelection = {};
+
+        var chartOptionsUnwatch = null,
+            columnsWatch = null;
+
+        scope.$watch('globalSeriesType', function(type, old) {
+          if (type && old && type !== old && scope.visualization.options.seriesOptions) {
+            _.each(scope.visualization.options.seriesOptions, function(sOptions) {
+              sOptions.type = type;
+            });
+          }
+        });
+        scope.$watch('visualization.type', function (visualizationType) {
+          if (visualizationType == 'CHART') {
+            if (scope.visualization.options.series.stacking === null) {
+              scope.stacking = "none";
+            } else if (scope.visualization.options.series.stacking === undefined) {
+              scope.stacking = "normal";
+            } else {
+              scope.stacking = scope.visualization.options.series.stacking;
             }
-        }
-    });
-}());
+
+            var refreshSeries = function() {
+              scope.series = _.map(scope.queryResult.getChartData(scope.visualization.options.columnMapping), function (s) { return s.name; });
+
+              // TODO: remove uneeded ones?
+              if (scope.visualization.options.seriesOptions == undefined) {
+                scope.visualization.options.seriesOptions = {
+                  type: scope.globalSeriesType
+                };
+              };
+
+              _.each(scope.series, function(s, i) {
+                if (scope.visualization.options.seriesOptions[s] == undefined) {
+                  scope.visualization.options.seriesOptions[s] = {'type': 'column', 'yAxis': 0};
+                }
+                scope.visualization.options.seriesOptions[s].zIndex = i;
+
+              });
+              scope.zIndexes = _.range(scope.series.length);
+              scope.yAxes = [[0, 'left'], [1, 'right']];
+            };
+
+            var initColumnMapping = function() {
+              scope.columns = scope.queryResult.getColumns();
+
+              if (scope.visualization.options.columnMapping == undefined) {
+                scope.visualization.options.columnMapping = {};
+              }
+
+              scope.columnTypeSelection = scope.visualization.options.columnMapping;
+
+              _.each(scope.columns, function(column) {
+                var definition = column.name.split("::"),
+                    definedColumns = _.keys(scope.visualization.options.columnMapping);
+
+                if (_.indexOf(definedColumns, column.name) != -1) {
+                  // Skip already defined columns.
+                  return;
+                };
+
+                if (definition.length == 1) {
+                  scope.columnTypeSelection[column.name] = scope.visualization.options.columnMapping[column.name] = 'unused';
+                } else if (definition == 'multi-filter') {
+                  scope.columnTypeSelection[column.name] = scope.visualization.options.columnMapping[column.name] = 'series';
+                } else if (_.indexOf(_.values(scope.columnTypes), definition[1]) != -1) {
+                  scope.columnTypeSelection[column.name] = scope.visualization.options.columnMapping[column.name] = definition[1];
+                } else {
+                  scope.columnTypeSelection[column.name] = scope.visualization.options.columnMapping[column.name] = 'unused';
+                }
+              });
+            };
+
+            columnsWatch = scope.$watch('queryResult.getId()', function(id) {
+              if (!id) {
+                return;
+              }
+
+              initColumnMapping();
+              refreshSeries();
+            });
+
+            scope.$watchCollection('columnTypeSelection', function(selections) {
+              _.each(scope.columnTypeSelection, function(type, name) {
+                scope.visualization.options.columnMapping[name] = type;
+              });
+
+              refreshSeries();
+            });
+
+            chartOptionsUnwatch = scope.$watch("stacking", function (stacking) {
+              if (stacking == "none") {
+                scope.visualization.options.series.stacking = null;
+              } else {
+                scope.visualization.options.series.stacking = stacking;
+              }
+            });
+
+            scope.xAxisType = (scope.visualization.options.xAxis && scope.visualization.options.xAxis.type) || scope.xAxisType;
+
+            xAxisUnwatch = scope.$watch("xAxisType", function (xAxisType) {
+              scope.visualization.options.xAxis = scope.visualization.options.xAxis || {};
+              scope.visualization.options.xAxis.type = xAxisType;
+            });
+          } else {
+            if (chartOptionsUnwatch) {
+              chartOptionsUnwatch();
+              chartOptionsUnwatch = null;
+            }
+
+            if (columnsWatch) {
+              columnWatch();
+              columnWatch = null;
+            }
+
+            if (xAxisUnwatch) {
+              xAxisUnwatch();
+              xAxisUnwatch = null;
+            }
+          }
+        });
+      }
+    }
+  });
+}());

rd_ui/app/scripts/visualizations/cohort.js

@@ -28,9 +28,13 @@
                     } else {
                         var sortedData = _.sortBy($scope.queryResult.getData(), "date");
                         var grouped = _.groupBy(sortedData, "date");
+                        var maxColumns = _.reduce(grouped, function(memo, data){ 
+                            return (data.length > memo)? data.length : memo;
+                        }, 0);
                         var data = _.map(grouped, function(values, date) {
                            var row = [values[0].total];
                             _.each(values, function(value) { row.push(value.value); });
+                            _.each(_.range(values.length, maxColumns), function() { row.push(null); });
                             return row;
                         });
 

rd_ui/app/scripts/visualizations/table.js

@@ -1,98 +1,109 @@
 (function () {
-    var tableVisualization = angular.module('redash.visualization');
+  var tableVisualization = angular.module('redash.visualization');
 
-    tableVisualization.config(['VisualizationProvider', function(VisualizationProvider) {
-        VisualizationProvider.registerVisualization({
-            type: 'TABLE',
-            name: 'Table',
-            renderTemplate: '<grid-renderer options="visualization.options" query-result="queryResult"></grid-renderer>',
-            skipTypes: true
-        });
-    }]);
+  tableVisualization.config(['VisualizationProvider', function (VisualizationProvider) {
+    VisualizationProvider.registerVisualization({
+      type: 'TABLE',
+      name: 'Table',
+      renderTemplate: '<grid-renderer options="visualization.options" query-result="queryResult"></grid-renderer>',
+      skipTypes: true
+    });
+  }]);
 
-    tableVisualization.directive('gridRenderer', function () {
-        return {
-            restrict: 'E',
-            scope: {
-                queryResult: '=',
-                itemsPerPage: '='
-            },
-            templateUrl: "/views/grid_renderer.html",
-            replace: false,
-            controller: ['$scope', function ($scope) {
-                $scope.gridColumns = [];
-                $scope.gridData = [];
-                $scope.gridConfig = {
-                    isPaginationEnabled: true,
-                    itemsByPage: $scope.itemsPerPage || 15,
-                    maxSize: 8
+  tableVisualization.directive('gridRenderer', function () {
+    return {
+      restrict: 'E',
+      scope: {
+        queryResult: '=',
+        itemsPerPage: '='
+      },
+      templateUrl: "/views/grid_renderer.html",
+      replace: false,
+      controller: ['$scope', '$filter', function ($scope, $filter) {
+        $scope.gridColumns = [];
+        $scope.gridData = [];
+        $scope.gridConfig = {
+          isPaginationEnabled: true,
+          itemsByPage: $scope.itemsPerPage || 15,
+          maxSize: 8
+        };
+
+        $scope.$watch('queryResult && queryResult.getData()', function (data) {
+          if (!data) {
+            return;
+          }
+
+          if ($scope.queryResult.getData() == null) {
+            $scope.gridColumns = [];
+            $scope.gridData = [];
+            $scope.filters = [];
+          } else {
+            $scope.filters = $scope.queryResult.getFilters();
+
+            var prepareGridData = function (data) {
+              var gridData = _.map(data, function (row) {
+                var newRow = {};
+                _.each(row, function (val, key) {
+                  newRow[$scope.queryResult.getColumnCleanName(key)] = val;
+                })
+                return newRow;
+              });
+
+              return gridData;
+            };
+
+            $scope.gridData = prepareGridData($scope.queryResult.getData());
+
+            var columns = $scope.queryResult.getColumns();
+            $scope.gridColumns = _.map($scope.queryResult.getColumnCleanNames(), function (col, i) {
+              var columnDefinition = {
+                'label': $scope.queryResult.getColumnFriendlyNames()[i],
+                'map': col
+              };
+
+              var columnType = columns[i].type;
+
+              if (columnType === 'integer') {
+                columnDefinition.formatFunction = 'number';
+                columnDefinition.formatParameter = 0;
+              } else if (columnType === 'float') {
+                columnDefinition.formatFunction = 'number';
+                columnDefinition.formatParameter = 2;
+              } else if (columnType === 'boolean') {
+                columnDefinition.formatFunction = function (value) {
+                  if (value !== undefined) {
+                    return "" + value;
+                  }
+                  return value;
                 };
+              } else if (columnType === 'date') {
+                columnDefinition.formatFunction = function (value) {
+                  if (value) {
+                    return value.format("DD/MM/YY");
+                  }
+                  return value;
+                };
+              } else if (columnType === 'datetime') {
+                columnDefinition.formatFunction = function (value) {
+                  if (value) {
+                    return value.format("DD/MM/YY HH:mm");
+                  }
+                  return value;
+                };
+              } else {
+                columnDefinition.formatFunction = function (value) {
+                  if (angular.isString(value)) {
+                    value = $filter('linkify')(value);
+                  }
+                  return value;
+                }
+              }
 
-                $scope.$watch('queryResult && queryResult.getData()', function (data) {
-                    if (!data) {
-                        return;
-                    }
-
-                    if ($scope.queryResult.getData() == null) {
-                        $scope.gridColumns = [];
-                        $scope.gridData = [];
-                        $scope.filters = [];
-                    } else {
-
-
-                        $scope.filters = $scope.queryResult.getFilters();
-
-                        var gridData = _.map($scope.queryResult.getData(), function (row) {
-                            var newRow = {};
-                            _.each(row, function (val, key) {
-                                newRow[$scope.queryResult.getColumnCleanName(key)] = val;
-                            })
-                            return newRow;
-                        });
-
-                        $scope.gridColumns = _.map($scope.queryResult.getColumnCleanNames(), function (col, i) {
-                            var columnDefinition = {
-                                'label': $scope.queryResult.getColumnFriendlyNames()[i],
-                                'map': col
-                            };
-
-                            if (gridData.length > 0) {
-                                var exampleData = gridData[0][col];
-                                if (angular.isNumber(exampleData)) {
-                                    columnDefinition['formatFunction'] = 'number';
-                                    columnDefinition['formatParameter'] = 2;
-                                } else if (moment.isMoment(exampleData)) {
-                                    columnDefinition['formatFunction'] = function(value) {
-                                        // TODO: this is very hackish way to determine if we need
-                                        // to show the value as a time or date only. Better solution
-                                        // is to complete #70 and use the information it returns.
-                                        if (value._i.match(/^\d{4}-\d{2}-\d{2}T/)) {
-                                            return value.format("DD/MM/YY HH:mm");
-                                        }
-                                        return value.format("DD/MM/YY");
-                                    }
-                                }
-                            }
-
-                            return columnDefinition;
-                        });
-
-                        $scope.gridData = _.clone(gridData);
-
-                        $scope.$watch('filters', function (filters) {
-                            $scope.gridData = _.filter(gridData, function (row) {
-                                return _.reduce(filters, function (memo, filter) {
-                                    if (filter.current == 'All') {
-                                        return memo && true;
-                                    }
-
-                                    return (memo && row[$scope.queryResult.getColumnCleanName(filter.name)] == filter.current);
-                                }, true);
-                            });
-                        }, true);
-                    }
-                });
-            }]
-        }
-    })
+              return columnDefinition;
+            });
+          }
+        });
+      }]
+    }
+  })
 }());

rd_ui/app/styles/redash.css

@@ -25,6 +25,29 @@ a.navbar-brand {
     margin-top: 5px;
     margin-bottom: 5px;
 }
+.avatar img {
+    width: 40px;
+    height: 40px;
+}
+
+#logout {
+    color: white;
+    position: relative;
+    left: -9px;
+    bottom: -11px;
+}
+
+.details-toggle {
+    cursor: pointer;
+}
+.details-toggle::before {
+    content: '▸';
+    margin-right: 5px;
+}
+.details-toggle.open::before {
+    content: '▾';
+    margin-right: 5px;
+}
 
 .edit-in-place span {
     white-space: pre-line;
@@ -63,10 +86,15 @@ a.navbar-brand {
     margin-bottom: 0px;
 }
 
-.panel-heading > a {
+.panel-heading > a,
+.panel-heading .query-link {
     color: inherit;
 }
 
+.panel-heading .query-link:hover {
+    text-decoration: none;
+}
+
 /* angular-growl */
 .growl {
     position: fixed;
@@ -217,6 +245,9 @@ to add those CSS styles here. */
     background-color: #FF8080;
     border-radius: 50%;
 }
+.nav-tabs > li.rd-tab-btn {
+    float: right;
+}
 
 /* light version of bootstrap's form-control */
 .rd-form-control {
@@ -236,10 +267,18 @@ to add those CSS styles here. */
 .rd-form-control {
      width: 100%;
 }
-visualization-renderer > div {
+pivot-table-renderer > table, grid-renderer > div, visualization-renderer > div {
     overflow: auto;
 }
 
+.rd-widget-textbox p {
+    margin-bottom: 0;
+}
+
+.iframe-container {
+    height: 100%;
+}
+
 /*
 bootstrap's hidden-xs class adds display:block when not hidden
 use this class when you need to keep the original display value
@@ -248,4 +287,4 @@ use this class when you need to keep the original display value
     .rd-hidden-xs {
         display: none !important;
     }
-}
+}

rd_ui/app/views/admin_status.html

@@ -21,30 +21,20 @@
                     Started
                 </li>
                 <li class="list-group-item">
-                    <span class="badge">{{manager.queue_size}}</span>
-                    Queue Size
+                    <span class="badge">{{manager.outdated_queries_count}}</span>
+                    Outdated Queries Count
+                </li>
+
+                <li class="list-group-item" ng-if="flowerUrl">
+                    <a href="/admin/workers">Workers' Status</a>
                 </li>
             </ul>
             <ul class="list-group col-lg-4">
-                <div ng-repeat="worker in workers">
-                    <li class="list-group-item active">Worker {{$index+1}}</li>
-                    <li class="list-group-item">
-                    <span class="badge" am-time-ago="worker.updated_at*1000.0"></span>
-                    Updated
+                <li class="list-group-item active">Queues</li>
+                <li class="list-group-item" ng-repeat="(name, value) in manager.queues">
+                    <span class="badge">{{value.size}}</span>
+                    {{name}} ({{value.data_sources}})
                 </li>
-                <li class="list-group-item">
-                    <span class="badge" am-time-ago="worker.started_at*1000.0"></span>
-                    Started
-                </li>
-                <li class="list-group-item">
-                    <span class="badge">{{worker.jobs_count}}</span>
-                    Jobs Received
-                </li>
-                <li class="list-group-item">
-                    <span class="badge">{{worker.done_jobs_count}}</span>
-                    Jobs Done
-                </li>
-                </div>
             </ul>
         </div>
         <div class="panel-footer">Next refresh: <span am-time-ago="refresh_time"></span></div>

rd_ui/app/views/admin_workers.html

@@ -0,0 +1,3 @@
+<div class="container-fluid iframe-container">
+    <iframe src="{{flowerUrl}}" style="width:100%; height:100%; background-color:transparent;"></iframe>
+</div>

rd_ui/app/views/dashboard.html

@@ -14,6 +14,7 @@
                 </button>
             </span>
     </h2>
+    <filters></filters>
 </div>
 
 <div class="container" id="dashboard">
@@ -21,11 +22,12 @@
         <div ng-repeat="widget in row" class="col-lg-{{widget.width | colWidth}}"
              ng-controller='WidgetCtrl'>
 
-            <div class="panel panel-default">
+            <div class="panel panel-default" ng-if="type=='visualization'">
                 <div class="panel-heading">
-                    <h3 class="panel-title" style="cursor: pointer;" ng-click="open(query, widget.visualization)">
+                    <h3 class="panel-title">
                         <p>
-                            <span ng-bind="query.name"></span>
+                            <span ng-hide="currentUser.hasPermission('view_query')">{{query.name}}</span>
+                            <query-link query="query" visualization="widget.visualization" ng-show="currentUser.hasPermission('view_query')"></query-link>
                         </p>
                         <div class="text-muted" ng-bind="query.description"></div>
                     </h3>
@@ -39,12 +41,32 @@
                           tooltip-placement="bottom">Updated: <span am-time-ago="queryResult.getUpdatedAt()"></span></span>
 
                     <span class="pull-right">
-                        <a class="btn btn-default btn-xs" ng-href="/queries/{{query.id}}#{{widget.visualization.id}}"><span class="glyphicon glyphicon-link"></span></a>
+                        <a class="btn btn-default btn-xs" ng-href="/queries/{{query.id}}#{{widget.visualization.id}}" ng-show="currentUser.hasPermission('view_query')"><span class="glyphicon glyphicon-link"></span></a>
                         <button type="button" class="btn btn-default btn-xs" ng-show="dashboard.canEdit()" ng-click="deleteWidget()" title="Remove Widget"><span class="glyphicon glyphicon-trash"></span></button>
                     </span>
 
+                    <span class="pull-right">
+                        <a class="btn btn-default btn-xs" ng-disabled="!queryResult.getData()" query-result-link target="_self">
+                            <span class="glyphicon glyphicon-cloud-download"></span>
+                        </a>
+                    </span>
+                </div>
+            </div>
+
+            <div class="panel panel-default rd-widget-textbox" ng-if="type=='textbox'" ng-mouseenter="showControls = true" ng-mouseleave="showControls = false">
+                <div class="panel-body">
+                    <div class="row">
+                        <div class="col-lg-11">
+                            <p ng-bind-html="widget.text | markdown"></p>
+                        </div>
+                        <div class="col-lg-1">
+                          <span class="pull-right" ng-show="showControls">
+                              <button type="button" class="btn btn-default btn-xs" ng-show="dashboard.canEdit()" ng-click="deleteWidget()" title="Remove Widget"><span class="glyphicon glyphicon-trash"></span></button>
+                          </span>
+                        </div>
+                    </div>
                 </div>
             </div>
         </div>
     </div>
-</div>
+</div>

rd_ui/app/views/grid_renderer.html

@@ -1,17 +1,4 @@
 <div>
-    <div class="btn-group pull-right" ng-repeat="filter in filters">
-        <button type="button" class="btn btn-default dropdown-toggle" data-toggle="dropdown">
-            {{filter.friendlyName}}: {{filter.current}}<span class="caret"></span>
-        </button>
-        <ul class="dropdown-menu" role="menu">
-            <li ng-repeat="value in filter.values">
-                <a href="#" ng-click="filter.current = value">{{value}}</a>
-            </li>
-            <li class="divider"></li>
-            <li><a href="#" ng-click="filter.current = 'All'">All</a></li>
-        </ul>
-    </div>
-
     <smart-table rows="gridData" columns="gridColumns"
                  config="gridConfig"
                  class="table table-condensed table-hover"></smart-table>

rd_ui/app/views/index.html

@@ -3,7 +3,7 @@
     <div class="list-group" ng-repeat="(name, dashboards) in allDashboards">
         <div class="list-group-item active">
             {{name}}
-            <button type="button" class="btn btn-sm btn-link" data-toggle="modal" href="#new_dashboard_dialog" tooltip="New Dashboard"><span class="glyphicon glyphicon-plus-sign"></span></button>
+            <button ng-show="currentUser.hasPermission('create_dashboard')" type="button" class="btn btn-sm btn-link" data-toggle="modal" href="#new_dashboard_dialog" tooltip="New Dashboard"><span class="glyphicon glyphicon-plus-sign"></span></button>
         </div>
         <div class="list-group-item" ng-repeat="dashboard in dashboards" >
             <button type="button" class="close delete-button" aria-hidden="true" ng-show="dashboard.canEdit()" ng-click="archiveDashboard(dashboard)" tooltip="Delete Dashboard">&times;</button>
@@ -11,7 +11,7 @@
         </div>
     </div>
 
-    <div ng-show="currentUser.is_admin">
+    <div ng-show="currentUser.hasPermission('admin')">
         <div class="list-group">
             <div class="list-group-item active">Admin</div>
             <a href="/admin/status" class="list-group-item">Status</a>

rd_ui/app/views/new_widget_form.html

@@ -6,35 +6,54 @@
                 <h4 class="modal-title">Add Widget</h4>
             </div>
             <div class="modal-body">
-                <p>
-                    <form class="form-inline" role="form" ng-submit="loadVisualizations()">
-                      <div class="form-group">
-                        <input class="form-control" placeholder="Query Id" ng-model="queryId">
-                      </div>
-                      <button type="submit" class="btn btn-primary" ng-disabled="!queryId">
-                        <span class="glyphicon glyphicon-refresh"></span> Load
-                      </button>
-                    </form>
+                <p class="btn-group">
+                    <button type="button" class="btn btn-default" ng-class="{active: isVisualization()}" ng-click="setType('visualization')">Visualization</button>
+                    <button type="button" class="btn btn-default" ng-class="{active: isTextBox()}" ng-click="setType('textbox')">Text Box</button>
                 </p>
 
-                <div ng-show="query">
-                  <div class="form-group">
-                      <label for="">Choose Visualation</label>
-                      <select ng-model="selectedVis" ng-options="vis as vis.name group by vis.type for vis in query.visualizations" class="form-control"></select>
-                  </div>
+                <div ng-show="isTextBox()">
+                    <div class="form-group">
+                        <textarea class="form-control" ng-model="text" rows="3"></textarea>
+                    </div>
+                    <div ng-show="text">
+                        <strong>Preview:</strong>
+                        <p ng-bind-html="text | markdown"></p>
+                    </div>
+                </div>
 
-                  <div class="form-group">
-                      <label for="">Widget Size</label>
-                      <select class="form-control" ng-model="widgetSize" ng-options="c.value as c.name for c in widgetSizes"></select>
-                  </div>
+                <div ng-show="isVisualization()">
+                    <p>
+                        <form class="form-inline" role="form" ng-submit="loadVisualizations()">
+                            <div class="form-group">
+                                <input class="form-control" placeholder="Query Id" ng-model="queryId">
+                            </div>
+                            <button type="submit" class="btn btn-primary" ng-disabled="!queryId">
+                                Load visualizations
+                            </button>
+                        </form>
+                    </p>
+
+                    <div ng-show="query">
+                      <div class="form-group">
+                          <label for="">Choose Visualization</label>
+                          <select ng-model="selectedVis" ng-options="vis as vis.name group by vis.type for vis in query.visualizations" class="form-control"></select>
+                      </div>
+                    </div>
+                </div>
+
+                <div class="form-group">
+                    <label for="">Widget Size</label>
+                    <select class="form-control" ng-model="widgetSize"
+                            ng-options="c.value as c.name for c in widgetSizes"></select>
                 </div>
             </div>
+
             <div class="modal-footer">
                 <button type="button" class="btn btn-default" ng-disabled="saveInProgress" data-dismiss="modal">Close</button>
-                <button type="button" class="btn btn-primary" ng-disabled="saveInProgress" ng-click="saveWidget()">Add to Dashboard</button>
+                <button type="button" class="btn btn-primary" ng-disabled="saveInProgress || !(selectedVis || isTextBox())" ng-click="saveWidget()">Add to Dashboard</button>
             </div>
         </div>
         <!-- /.modal-content -->
     </div>
     <!-- /.modal-dialog -->
-</div>
+</div>

rd_ui/app/views/query.html

@@ -1,28 +1,29 @@
 
 <div class="container">
+
+    <alert-unsaved-changes ng-if="canEdit" is-dirty="isDirty"></alert-unsaved-changes>
+
     <div class="row">
         <div class="col-lg-12">
             <div class="row">
                 <div class="col-lg-10">
                     <h2>
-                        <edit-in-place editable="isOwner" done="saveQuery" ignore-blanks='true' value="query.name"></edit-in-place>
+                        <edit-in-place editable="isQueryOwner" done="saveName" ignore-blanks='true' value="query.name"></edit-in-place>
                     </h2>
                     <p>
                         <em>
-                            <edit-in-place editable="isOwner" done="saveQuery" editor="textarea" placeholder="No description" ignore-blanks='false' value="query.description"></edit-in-place>
+                            <edit-in-place editable="isQueryOwner" done="saveDescription" editor="textarea" placeholder="No description" ignore-blanks='false' value="query.description"></edit-in-place>
                         </em>
                     </p>
                 </div>
-                <div class="col-lg-2" ng-hide="isNewQuery">
-                    <a ng-href="{{sourceHref}}" ng-click="toggleSource()" class="hidden-xs pull-right">
-                        <span ng-show="isSourceVisible">Hide Source</span>
-                        <span ng-show="!isSourceVisible">View Source</span>
-                    </a>
+
+                <div class="col-lg-2">
+                    <div class="rd-hidden-xs pull-right">
+                        <query-source-link></query-source-link>
+                    </div>
                 </div>
             </div>
 
-
-
             <div class="visible-xs">
                 <p>
                     <span class="text-muted">Last update </span>
@@ -31,7 +32,8 @@
                     </strong>
                     &nbsp;
                     <span class="text-muted">Created By </span>
-                    <strong>{{query.user.name}}</strong>
+                    <strong ng-hide="isQueryOwner">{{query.user.name}}</strong>
+                    <strong ng-show="isQueryOwner">You</strong>
                     &nbsp;
                     <span class="text-muted">Runtime </span>
                     <strong ng-show="!queryExecuting">{{queryResult.getRuntime() | durationHumanize}}</strong>
@@ -41,10 +43,7 @@
                     <strong>{{queryResult.getData().length}}</strong>
                 </p>
                 <p>
-                    <a ng-href="{{sourceHref}}" ng-click="toggleSource()">
-                        <span ng-show="isSourceVisible">Hide Source</span>
-                        <span ng-show="!isSourceVisible">View Source</span>
-                    </a>
+                    <query-source-link></query-source-link>
                 </p>
             </div>
         </div>
@@ -54,31 +53,31 @@
 
     <div class="row">
         <div class="col-lg-12">
-            <div ng-show="isSourceVisible">
+            <div ng-show="sourceMode">
                 <p>
                     <button type="button" class="btn btn-primary btn-xs" ng-disabled="queryExecuting" ng-click="executeQuery()">
                         <span class="glyphicon glyphicon-play"></span> Execute
                     </button>
-                    <button type="button" class="btn btn-default btn-xs" ng-click="formatQuery()">
-                        <span class="glyphicon glyphicon-indent-left"></span> Format SQL
-                    </button>
+                    <query-formatter></query-formatter>
                     <span class="pull-right">
-                        <button class="btn btn-xs btn-default rd-hidden-xs" ng-show="canFork" ng-click="duplicateQuery()">
+                        <button class="btn btn-xs btn-default rd-hidden-xs" ng-click="duplicateQuery()">
                             <span class="glyphicon glyphicon-share-alt"></span> Fork
                         </button>
 
                         <button class="btn btn-success btn-xs" ng-show="canEdit" ng-click="saveQuery()">
-                            <span class="glyphicon glyphicon-floppy-disk"> </span> Save<span ng-show="dirty">&#42;</span>
+                            <span class="glyphicon glyphicon-floppy-disk"> </span> Save<span ng-show="isDirty">&#42;</span>
                         </button>
                     </span>
                 </p>
             </div>
 
             <!-- code editor -->
-            <p ng-show="isSourceVisible">
-                <textarea ui-codemirror="editorOptions" ui-refresh="isSourceVisible" ng-model="query.query"></textarea>
-            </p>
-            <hr ng-show="isSourceVisible">
+            <div ng-show="sourceMode">
+                <p>
+                    <query-editor query="query" lock="queryExecuting"></query-editor>
+                </p>
+                <hr>
+            </div>
         </div>
 
     </div>
@@ -94,7 +93,9 @@
             </p>
             <p>
                 <span class="glyphicon glyphicon-user"></span>
-                <span class="text-muted">Created By </span><strong>{{query.user.name}}</strong>
+                <span class="text-muted">Created By </span>
+                <strong ng-hide="isQueryOwner">{{query.user.name}}</strong>
+                <strong ng-show="isQueryOwner">You</strong>
             </p>
             <p>
                 <span class="glyphicon glyphicon-play"></span>
@@ -109,13 +110,19 @@
             <p>
                 <span class="glyphicon glyphicon-refresh"></span>
                 <span class="text-muted">Refresh Interval</span>
-                <select ng-disabled="!isOwner" ng-model="query.ttl" ng-change="saveQuery()" ng-options="c.value as c.name for c in refreshOptions"></select>
+                <query-refresh-select></query-refresh-select>
+            </p>
+
+            <p>
+                <span class="glyphicon glyphicon-hdd"></span>
+                <span class="text-muted">Data Source</span>
+                <select ng-disabled="!isQueryOwner" ng-model="query.data_source_id" ng-change="updateDataSource()" ng-options="ds.id as ds.name for ds in dataSources"></select>
             </p>
 
             <hr>
 
             <p>
-                <a class="btn btn-primary btn-sm" ng-disabled="queryExecuting || !queryResult.getData()" ng-href="{{dataUri}}" download="{{dataFilename}}" target="_self">
+                <a class="btn btn-primary btn-sm" ng-disabled="queryExecuting || !queryResult.getData()" query-result-link target="_self">
                     <span class="glyphicon glyphicon-cloud-download"></span>
                     <span class="rd-hidden-xs">Download Dataset</span>
                 </a>
@@ -135,32 +142,38 @@
             <div class="alert alert-danger" ng-show="queryResult.getError()">Error running query: <strong>{{queryResult.getError()}}</strong></div>
 
             <!-- tabs and data -->
-            <div ng-show="queryResult.getStatus() == 'done'">
+            <div ng-show="showDataset">
                 <div class="row">
                     <div class="col-lg-12">
                         <ul class="nav nav-tabs">
                             <rd-tab tab-id="table" name="Table"></rd-tab>
                             <rd-tab tab-id="pivot" name="Pivot Table"></rd-tab>
-                            <rd-tab tab-id="{{vis.id}}" name="{{vis.name}}" ng-hide="vis.type=='TABLE'" ng-repeat="vis in query.visualizations">
+                            <rd-tab tab-id="{{vis.id}}" name="{{vis.name}}" ng-if="vis.type!='TABLE'" ng-repeat="vis in query.visualizations">
                                 <span class="remove" ng-click="deleteVisualization($event, vis)" ng-show="canEdit"> &times;</span>
                             </rd-tab>
                             <rd-tab tab-id="add" name="&plus; New" removeable="true" ng-show="canEdit"></rd-tab>
+                            <li ng-if="!sourceMode" class="rd-tab-btn"><button class="btn btn-sm btn-default" ng-click="executeQuery()" ng-disabled="queryExecuting" title="Refresh Dataset"><span class="glyphicon glyphicon-refresh"></span></button></li>
                         </ul>
                     </div>
                 </div>
                 <div class="row">
                     <div class="col-lg-12">
-                        <grid-renderer ng-show="selectedTab == 'table'" query-result="queryResult" items-per-page="50"></grid-renderer>
+                        <div ng-show="selectedTab == 'table'" >
+                            <filters></filters>
+                            <grid-renderer query-result="queryResult" items-per-page="50"></grid-renderer>
+                        </div>
+
                         <pivot-table-renderer ng-show="selectedTab == 'pivot'" query-result="queryResult"></pivot-table-renderer>
 
                         <div ng-show="selectedTab == vis.id" ng-repeat="vis in query.visualizations">
                             <visualization-renderer visualization="vis" query-result="queryResult"></visualization-renderer>
+
                             <edit-visulatization-form visualization="vis" query="query" query-result="queryResult" ng-show="canEdit"></edit-visulatization-form>
                         </div>
 
-                        <div ng-show="selectedTab == 'add'">
+                        <div ng-if="canEdit" ng-show="selectedTab == 'add'">
                             <visualization-renderer visualization="newVisualization" query-result="queryResult"></visualization-renderer>
-                            <edit-visulatization-form visualization="newVisualization" query="query" ng-show="canEdit"></edit-visulatization-form>
+                            <edit-visulatization-form visualization="newVisualization" query="query" query-result="queryResult" ng-show="canEdit" open-editor="true" on-new-success="setVisualizationTab"></edit-visulatization-form>
                         </div>
                     </div>
                 </div>

rd_ui/app/views/queryfiddle.html

@@ -1,102 +0,0 @@
-<div class="container">
-    <div class="row">
-        <div class="panel panel-default">
-            <div class="panel-heading">
-                <h3 class="panel-title">
-                    <p>
-                     <edit-in-place editable="currentUser.canEdit(query)" ignore-blanks='true' value="query.name"></edit-in-place>
-                 </p>
-                </h3>
-                <p>
-                    <edit-in-place editable="currentUser.canEdit(query)" editor="textarea" placeholder="No description" ignore-blanks='false' value="query.description" class="text-muted"></edit-in-place>
-                </p>
-            </div>
-            <div class="panel-body">
-                <textarea ui-codemirror="editorOptions" ng-model="query.query"></textarea>
-
-                <div>
-                    <a class="btn btn-default" ng-disabled="queryExecuting || !queryResult.getData()" ng-href="{{dataUri}}" download="{{dataFilename}}" target="_self">
-                        <span class="glyphicon glyphicon-floppy-disk"></span> Download Data Set
-                    </a>
-                    <button type="button" class="btn btn-default center-x" ng-click="formatQuery()"><span class="glyphicon glyphicon-ok"></span> Format SQL</button>
-
-                    <div class="btn-group pull-right">
-                        <button type="button" class="btn btn-default" ng-click="duplicateQuery()">Duplicate</button>
-                        <button type="button" class="btn btn-default" ng-disabled="!currentUser.canEdit(query)" ng-click="saveQuery()">Save
-                            <span ng-show="dirty">&#42;</span>
-                        </button>
-                        <button type="button" class="btn btn-primary" ng-disabled="queryExecuting" ng-click="executeQuery()">Execute</button>
-                    </div>
-                </div>
-
-            </div>
-            <div class="panel-footer">
-                <span ng-show="queryResult.getRuntime()>=0">Query runtime: {{queryResult.getRuntime() | durationHumanize}} | </span>
-                <span ng-show="queryResult.query_result.retrieved_at">Last update time: <span am-time-ago="queryResult.query_result.retrieved_at"></span> | </span>
-                <span ng-show="queryResult.getStatus() == 'done'">Rows: {{queryResult.getData().length}} | </span>
-                Created by: {{query.user.name}}
-                <div class="pull-right">Refresh query: <select ng-model="query.ttl" ng-options="c.value as c.name for c in refreshOptions"></select><br></div>
-            </div>
-        </div>
-
-        <div class="alert alert-info" ng-show="queryResult.getStatus() == 'processing'">
-            Executing query... <rd-timer timestamp="queryResult.getUpdatedAt()"></rd-timer>
-            <button type="button" class="btn btn-warning btn-xs pull-right" ng-disabled="cancelling" ng-click="cancelExecution()">Cancel</button>
-        </div>
-        <div class="alert alert-info" ng-show="queryResult.getStatus() == 'waiting'">
-            Query in queue... <rd-timer timestamp="queryResult.getUpdatedAt()"></rd-timer>
-            <button type="button" class="btn btn-warning btn-xs pull-right" ng-disabled="cancelling" ng-click="cancelExecution()">Cancel</button>
-        </div>
-        <div class="alert alert-danger" ng-show="queryResult.getError()">Error running query: <strong>{{queryResult.getError()}}</strong></div>
-    </div>
-
-    <div class="row" ng-show="queryResult.getStatus() == 'done'">
-        <ul class="nav nav-tabs">
-            <rd-tab tab-id="table" name="Table"></rd-tab>
-            <rd-tab tab-id="pivot" name="Pivot Table"></rd-tab>
-            <!-- hide the table visualization -->
-            <rd-tab tab-id="{{vis.id}}" name="{{vis.name}}" ng-hide="vis.type=='TABLE'" ng-repeat="vis in query.visualizations">
-                <span class="remove" ng-click="deleteVisualization($event, vis)" ng-show="currentUser.canEdit(query)"> &times;</span>
-            </rd-tab>
-            <rd-tab tab-id="add" name="&plus; New" removeable="true" ng-show="currentUser.canEdit(query)"></rd-tab>
-        </ul>
-
-        <div class="col-lg-12" ng-show="selectedTab == 'table'">
-            <grid-renderer query-result="queryResult" items-per-page="50"></grid-renderer>
-        </div>
-
-        <div class="col-lg-12" ng-show="selectedTab == 'pivot'">
-            <pivot-table-renderer query-result="queryResult"></pivot-table-renderer>
-        </div>
-
-        <div class="col-lg-12" ng-show="selectedTab == vis.id" ng-repeat="vis in query.visualizations">
-            <div class="row" ng-show="currentUser.canEdit(query)">
-                <p>
-                    <div class="col-lg-12">
-                        <edit-visulatization-form visualization="vis" query="query" query-result="queryResult"></edit-visulatization-form>
-                    </div>
-                </p>
-            </div>
-            <div class="row">
-                <p>
-                    <div class="col-lg-12">
-                        <visualization-renderer visualization="vis" query-result="queryResult"></visualization-renderer>
-                    </div>
-                </p>
-            </div>
-        </div>
-
-        <div class="col-lg-12" ng-show="selectedTab == 'add'">
-            <div class="row">
-                 <p>
-                    <div class="col-lg-6">
-                        <edit-visulatization-form visualization="newVisualization" query="query"></edit-visulatization-form>
-                    </div>
-                    <div class="col-lg-6">
-                        <visualization-renderer visualization="newVisualization" query-result="queryResult"></visualization-renderer>
-                    </div>
-                </p>
-            </div>
-        </div>
-    </div>
-</div>

rd_ui/app/views/visualizations/chart_editor.html

@@ -1,7 +1,100 @@
-<div class="form-group">
-    <label class="control-label">Chart Type</label>
-    <select required ng-model="visualization.options.series.type" ng-options="value as key for (key, value) in seriesTypes" class="form-control"></select>
+<div class="form-horizontal">
+    <div class="panel panel-default">
+        <div class="panel-body">
+            <div class="form-group">
+                <label class="control-label col-sm-2">Stacking</label>
 
-    <label class="control-label">Stacking</label>
-    <select required ng-model="stacking" ng-options="value as key for (key, value) in stackingOptions" class="form-control"></select>
-</div>
+                <div class="col-sm-10">
+                    <select required ng-model="stacking"
+                            ng-options="value as key for (key, value) in stackingOptions"
+                            class="form-control"></select>
+                </div>
+            </div>
+            <div class="form-group">
+                <label class="control-label col-sm-2">X Axis Type</label>
+
+                <div class="col-sm-10">
+                    <select required ng-model="xAxisType" ng-options="value as key for (key, value) in xAxisOptions"
+                            class="form-control"></select>
+                </div>
+            </div>
+
+            <div class="form-group">
+                <label class="control-label col-sm-2">Series Type</label>
+
+                <div class="col-sm-10">
+                    <select required ng-options="value as key for (key, value) in seriesTypes"
+                            ng-model="globalSeriesType" class="form-control"></select>
+                </div>
+            </div>
+        </div>
+    </div>
+
+    <div class="row">
+        <div class="col-lg-6">
+            <div class="list-group">
+                <div class="list-group-item active">
+                    Columns Mapping
+                </div>
+                <div class="list-group-item">
+                    <div class="form-group" ng-repeat="column in columns">
+                        <label class="control-label col-sm-4">{{column.name}}</label>
+
+                        <div class="col-sm-8">
+                            <select ng-options="value as key for (key, value) in columnTypes" class="form-control"
+                                    ng-model="columnTypeSelection[column.name]"></select>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+
+        <div class="col-lg-6" ng-if="series.length > 0">
+            <div class="list-group" ng-repeat="seriesName in series">
+                <div class="list-group-item active">
+                    {{seriesName}}
+                </div>
+                <div class="list-group-item">
+                    <div class="form-group">
+                        <label class="control-label col-sm-3">Type</label>
+
+                        <div class="col-sm-9">
+                            <select required ng-model="visualization.options.seriesOptions[seriesName].type"
+                                    ng-options="value as key for (key, value) in seriesTypes"
+                                    class="form-control"></select>
+                        </div>
+                    </div>
+                    <div class="form-group">
+                        <label class="control-label col-sm-3">zIndex</label>
+
+                        <div class="col-sm-9">
+                            <select required ng-model="visualization.options.seriesOptions[seriesName].zIndex"
+                                    ng-options="o as o for o in zIndexes"
+                                    class="form-control"></select>
+                        </div>
+                    </div>
+                    <div class="form-group">
+                        <label class="control-label col-sm-3">y Axis</label>
+
+                        <div class="col-sm-9">
+                            <select required ng-model="visualization.options.seriesOptions[seriesName].yAxis"
+                                    ng-options="o[0] as o[1] for o in yAxes"
+                                    class="form-control"></select>
+                        </div>
+                    </div>
+                    <div class="form-group">
+                        <label class="control-label col-sm-3">Name</label>
+
+                        <div class="col-sm-9">
+                            <input name="seriesName" type="text" class="form-control"
+                                   ng-model="visualization.options.seriesOptions[seriesName].name"
+                                   placeholder="{{seriesName}}">
+                        </div>
+                    </div>
+                </div>
+            </div>
+
+
+        </div>
+    </div>
+</div>

rd_ui/app/views/visualizations/cohort_editor.html

@@ -0,0 +1,15 @@
+<div class="form-group">
+    <label class="control-label">Time Label</label>
+    <input type="text" class="form-control" ng-model="cohortOptions.timeLabel">
+    <label class="control-label">People Label</label>
+    <input type="text" class="form-control" ng-model="cohortOptions.peopleLabel">
+
+    <label class="control-label">Bucket Column</label>
+    <select ng-model="bucket_column" ng-options="value as key for (key, value) in columns" class="form-control"></select>
+    <label class="control-label">Bucket Total Value Column</label>
+    <select ng-model="total_column" ng-options="value as key for (key, value) in columns" class="form-control"></select>
+    <label class="control-label">Day Number Column</label>
+    <select ng-model="value_column" ng-options="value as key for (key, value) in columns" class="form-control"></select>
+    <label class="control-label">Day Value Column</label>
+    <select ng-model="day_column" ng-options="value as key for (key, value) in columns" class="form-control"></select>
+</div>

rd_ui/app/views/visualizations/edit_visualization.html

@@ -1,18 +1,27 @@
-<form role="form" name="visForm" ng-submit="submit()">
-  <div class="form-group">
-    <label class="control-label">Name</label>
-    <input name="name" type="text" class="form-control" ng-model="visualization.name" placeholder="{{visualization.type | capitalize}}">
-  </div>
+<div>
+  <span ng-click="openEditor=!openEditor" class="details-toggle" ng-class="{open: openEditor}">Edit</span>
 
-  <div class="form-group">
-    <label class="control-label">Visualization Type</label>
-    <select required ng-model="visualization.type" ng-options="value as key for (key, value) in visTypes" class="form-control" ng-change="typeChanged()"></select>
-  </div>
+  <form ng-if="openEditor" role="form" name="visForm" ng-submit="submit()">
+    <div class="form-group">
+      <label class="control-label">Name</label>
+      <input name="name" type="text" class="form-control" ng-model="visualization.name" placeholder="{{visualization.type | capitalize}}">
+    </div>
 
-  <visualization-options-editor></visualization-options-editor>
+    <div class="form-group">
+      <label class="control-label">Visualization Type</label>
+      <select required ng-model="visualization.type" ng-options="value as key for (key, value) in visTypes" class="form-control" ng-change="typeChanged()"></select>
+    </div>
 
-  <div class="form-group">
-    <button type="submit" class="btn btn-primary">Save</button>
-  </div>
+    <visualization-options-editor></visualization-options-editor>
 
-</form>
+    <div class="form-group" ng-if="editRawOptions">
+      <label class="control-label">Advanced</label>
+      <textarea json-text ng-model="visualization.options" class="form-control" rows="10"></textarea>
+    </div>
+
+    <div class="form-group">
+      <button type="submit" class="btn btn-primary">Save</button>
+    </div>
+
+  </form>
+</div>

rd_ui/app/views/visualizations/filters.html

@@ -0,0 +1,8 @@
+<div class="well well-sm" ng-show="filters">
+    <div ng-repeat="filter in filters">
+        {{filter.friendlyName}}:
+        <select ui-select2='select2Options' ng-model="filter.current" ng-multiple="{{filter.multiple}}">
+            <option ng-repeat="value in filter.values" value="{{value}}">{{value}}</option>
+        </select>
+    </div>
+</div>

rd_ui/bower.json

@@ -11,21 +11,27 @@
     "moment": "2.1.0",
     "angular-ui-bootstrap": "0.5.0",
     "angular-ui-codemirror": "0.0.5",
-    "highcharts": "3.0.1",
+    "highcharts": "3.0.10",
     "underscore": "1.5.1",
-    "angular-resource": "1.0.7",
+    "angular-resource": "1.2.15",
     "angular-growl": "0.3.1",
     "angular-route": "1.2.7",
-    "pivottable": "https://github.com/arikfr/pivottable.git",
+    "pivottable": "~1.1.1",
     "cornelius": "https://github.com/restorando/cornelius.git",
     "gridster": "0.2.0",
-    "mousetrap": "~1.4.6"
+    "mousetrap": "~1.4.6",
+    "angular-ui-select2": "~0.0.5",
+    "jquery-ui": "~1.10.4",
+    "underscore.string": "~2.3.3",
+    "marked": "~0.3.2",
+    "bucky": "~0.2.6",
+    "pace": "~0.5.1"
   },
   "devDependencies": {
     "angular-mocks": "~1.0.7",
     "angular-scenario": "~1.0.7"
   },
   "resolutions": {
-    "angular": "~1.2.7"
+    "angular": "1.2.7"
   }
 }

rd_ui/karma-e2e.conf.js

@@ -1,58 +0,0 @@
-// Karma E2E configuration
-
-// base path, that will be used to resolve files and exclude
-basePath = '';
-
-// list of files / patterns to load in the browser
-files = [
-  ANGULAR_SCENARIO,
-  ANGULAR_SCENARIO_ADAPTER,
-  'test/e2e/**/*.js'
-];
-
-// list of files to exclude
-exclude = [];
-
-// test results reporter to use
-// possible values: dots || progress || growl
-reporters = ['progress'];
-
-// web server port
-port = 8080;
-
-// cli runner port
-runnerPort = 9100;
-
-// enable / disable colors in the output (reporters and logs)
-colors = true;
-
-// level of logging
-// possible values: LOG_DISABLE || LOG_ERROR || LOG_WARN || LOG_INFO || LOG_DEBUG
-logLevel = LOG_INFO;
-
-// enable / disable watching file and executing tests whenever any file changes
-autoWatch = false;
-
-// Start these browsers, currently available:
-// - Chrome
-// - ChromeCanary
-// - Firefox
-// - Opera
-// - Safari (only Mac)
-// - PhantomJS
-// - IE (only Windows)
-browsers = ['Chrome'];
-
-// If browser does not capture in given timeout [ms], kill it
-captureTimeout = 5000;
-
-// Continuous Integration mode
-// if true, it capture browsers, run tests and exit
-singleRun = false;
-
-// Uncomment the following lines if you are using grunt's server to run the tests
-// proxies = {
-//   '/': 'http://localhost:9000/'
-// };
-// URL root prevent conflicts with the site root
-// urlRoot = '_karma_';

rd_ui/karma.conf.js

@@ -1,56 +0,0 @@
-// Karma configuration
-
-// base path, that will be used to resolve files and exclude
-basePath = '';
-
-// list of files / patterns to load in the browser
-files = [
-  JASMINE,
-  JASMINE_ADAPTER,
-  'app/bower_components/angular/angular.js',
-  'app/bower_components/angular-mocks/angular-mocks.js',
-  'app/scripts/*.js',
-  'app/scripts/**/*.js',
-  'test/mock/**/*.js',
-  'test/spec/**/*.js'
-];
-
-// list of files to exclude
-exclude = [];
-
-// test results reporter to use
-// possible values: dots || progress || growl
-reporters = ['progress'];
-
-// web server port
-port = 8080;
-
-// cli runner port
-runnerPort = 9100;
-
-// enable / disable colors in the output (reporters and logs)
-colors = true;
-
-// level of logging
-// possible values: LOG_DISABLE || LOG_ERROR || LOG_WARN || LOG_INFO || LOG_DEBUG
-logLevel = LOG_INFO;
-
-// enable / disable watching file and executing tests whenever any file changes
-autoWatch = false;
-
-// Start these browsers, currently available:
-// - Chrome
-// - ChromeCanary
-// - Firefox
-// - Opera
-// - Safari (only Mac)
-// - PhantomJS
-// - IE (only Windows)
-browsers = ['Chrome'];
-
-// If browser does not capture in given timeout [ms], kill it
-captureTimeout = 5000;
-
-// Continuous Integration mode
-// if true, it capture browsers, run tests and exit
-singleRun = false;

rd_ui/package.json

@@ -1,38 +1,39 @@
 {
-  "name": "rd-ui",
-  "version": "0.1.0",
-  "dependencies": {
-  },
+  "name": "rdui",
+  "version": "0.0.0",
+  "dependencies": {},
   "devDependencies": {
-    "grunt": "git+https://github.com/gruntjs/grunt.git#08a3af5",
-    "grunt-contrib-copy": "~0.4.1",
-    "grunt-contrib-concat": "~0.3.0",
-    "grunt-contrib-coffee": "~0.7.0",
-    "grunt-contrib-uglify": "~0.2.0",
-    "grunt-contrib-compass": "~0.5.0",
-    "grunt-contrib-jshint": "~0.6.0",
-    "grunt-contrib-cssmin": "~0.6.0",
-    "grunt-contrib-connect": "~0.3.0",
-    "grunt-contrib-clean": "~0.5.0",
-    "grunt-contrib-htmlmin": "~0.1.3",
-    "grunt-contrib-imagemin": "~0.2.0",
-    "grunt-contrib-watch": "~0.5.2",
-    "grunt-autoprefixer": "~0.2.0",
-    "grunt-usemin": "~0.1.11",
-    "grunt-svgmin": "~0.2.0",
-    "grunt-rev": "~0.1.0",
-    "grunt-open": "~0.2.0",
-    "grunt-concurrent": "~0.3.0",
-    "load-grunt-tasks": "~0.1.0",
-    "connect-livereload": "~0.2.0",
-    "grunt-google-cdn": "~0.2.0",
-    "grunt-ngmin": "~0.0.2",
-    "time-grunt": "~0.1.0",
-    "bower": "~1.2.7",
-    "grunt-cli": "~0.1.9"
+    "grunt": "^0.4.1",
+    "grunt-autoprefixer": "^0.7.3",
+    "grunt-concurrent": "^0.5.0",
+    "grunt-contrib-clean": "^0.5.0",
+    "grunt-contrib-concat": "^0.4.0",
+    "grunt-contrib-connect": "^0.7.1",
+    "grunt-contrib-copy": "^0.5.0",
+    "grunt-contrib-cssmin": "^0.9.0",
+    "grunt-contrib-htmlmin": "^0.3.0",
+    "grunt-contrib-imagemin": "^0.7.0",
+    "grunt-contrib-jshint": "^0.10.0",
+    "grunt-contrib-uglify": "^0.4.0",
+    "grunt-contrib-watch": "^0.6.1",
+    "grunt-filerev": "^0.2.1",
+    "grunt-google-cdn": "^0.4.0",
+    "grunt-newer": "^0.7.0",
+    "grunt-ngmin": "^0.0.3",
+    "grunt-svgmin": "^0.4.0",
+    "grunt-usemin": "^2.1.1",
+    "grunt-wiredep": "^1.7.0",
+    "jshint-stylish": "^0.2.0",
+    "load-grunt-tasks": "^0.4.0",
+    "time-grunt": "^0.3.1",
+    "karma-jasmine": "~0.1.5",
+    "grunt-karma": "~0.8.3",
+    "karma-phantomjs-launcher": "~0.1.4",
+    "karma": "~0.12.19",
+    "karma-ng-html2js-preprocessor": "~0.1.0"
   },
   "engines": {
-    "node": ">=0.8.0"
+    "node": ">=0.10.0"
   },
   "scripts": {
     "test": "grunt test"

rd_ui/test/.jshintrc

@@ -29,6 +29,7 @@
     "expect": false,
     "inject": false,
     "it": false,
+    "jasmine": false,
     "spyOn": false
   }
 }

rd_ui/test/karma.conf.js

@@ -0,0 +1,131 @@
+// Karma configuration
+// http://karma-runner.github.io/0.12/config/configuration-file.html
+// Generated on 2014-07-30 using
+// generator-karma 0.8.3
+
+module.exports = function(config) {
+  'use strict';
+
+  config.set({
+    // enable / disable watching file and executing tests whenever any file changes
+    autoWatch: true,
+
+    // base path, that will be used to resolve files and exclude
+    basePath: '../',
+
+    // testing framework to use (jasmine/mocha/qunit/...)
+    frameworks: ['jasmine'],
+
+    // list of files / patterns to load in the browser
+    files: [
+      'app/bower_components/jquery/jquery.js',
+      'app/bower_components/jquery-ui/ui/jquery-ui.js',
+
+      'app/bower_components/angular/angular.js',
+      'app/bower_components/angular-route/angular-route.js',
+      'app/bower_components/angular-mocks/angular-mocks.js',
+
+      'app/bower_components/bootstrap/js/collapse.js',
+      'app/bower_components/bootstrap/js/modal.js',
+      'app/bower_components/angular-resource/angular-resource.js',
+      'app/bower_components/underscore/underscore.js',
+      'app/bower_components/moment/moment.js',
+      'app/bower_components/angular-moment/angular-moment.js',
+      'app/bower_components/codemirror/lib/codemirror.js',
+      'app/bower_components/codemirror/addon/edit/matchbrackets.js',
+      'app/bower_components/codemirror/addon/edit/closebrackets.js',
+      'app/bower_components/codemirror/mode/sql/sql.js',
+      'app/bower_components/codemirror/mode/javascript/javascript.js',
+      'app/bower_components/angular-ui-codemirror/ui-codemirror.js',
+      'app/bower_components/highcharts/highcharts.js',
+      'app/bower_components/highcharts/modules/exporting.js',
+      'app/bower_components/gridster/dist/jquery.gridster.js',
+      'app/bower_components/angular-growl/build/angular-growl.js',
+      'app/bower_components/pivottable/dist/pivot.js',
+      'app/bower_components/cornelius/src/cornelius.js',
+      'app/bower_components/mousetrap/mousetrap.js',
+      'app/bower_components/mousetrap/plugins/global-bind/mousetrap-global-bind.js',
+      'app/bower_components/select2/select2.js',
+      'app/bower_components/angular-ui-select2/src/select2.js',
+      'app/bower_components/underscore.string/lib/underscore.string.js',
+      'app/bower_components/marked/lib/marked.js',
+      'app/scripts/ng_highchart.js',
+      'app/scripts/ng_smart_table.js',
+      'app/scripts/ui-bootstrap-tpls-0.5.0.min.js',
+      'app/bower_components/bucky/bucky.js',
+      'app/bower_components/pace/pace.js',
+
+      'app/scripts/app.js',
+      'app/scripts/services/services.js',
+      'app/scripts/services/resources.js',
+      'app/scripts/services/notifications.js',
+      'app/scripts/services/dashboards.js',
+      'app/scripts/controllers/controllers.js',
+      'app/scripts/controllers/dashboard.js',
+      'app/scripts/controllers/admin_controllers.js',
+      'app/scripts/controllers/query_view.js',
+      'app/scripts/controllers/query_source.js',
+      'app/scripts/visualizations/base.js',
+      'app/scripts/visualizations/chart.js',
+      'app/scripts/visualizations/cohort.js',
+      'app/scripts/visualizations/table.js',
+      'app/scripts/visualizations/pivot.js',
+      'app/scripts/directives/directives.js',
+      'app/scripts/directives/query_directives.js',
+      'app/scripts/directives/dashboard_directives.js',
+      'app/scripts/filters.js',
+
+      'app/views/**/*.html',
+
+      'test/mocks/*.js',
+      'test/unit/*.js'
+    ],
+
+    // generate js files from html templates
+    preprocessors: {
+      'app/views/**/*.html': 'ng-html2js'
+    },
+
+    // list of files / patterns to exclude
+    exclude: [],
+
+    // web server port
+    port: 8080,
+
+    // Start these browsers, currently available:
+    // - Chrome
+    // - ChromeCanary
+    // - Firefox
+    // - Opera
+    // - Safari (only Mac)
+    // - PhantomJS
+    // - IE (only Windows)
+    browsers: [
+      'PhantomJS'
+    ],
+
+    // Which plugins to enable
+    plugins: [
+      'karma-phantomjs-launcher',
+      'karma-jasmine',
+      'karma-ng-html2js-preprocessor'
+    ],
+
+    // Continuous Integration mode
+    // if true, it capture browsers, run tests and exit
+    singleRun: false,
+
+    colors: true,
+
+    // level of logging
+    // possible values: LOG_DISABLE || LOG_ERROR || LOG_WARN || LOG_INFO || LOG_DEBUG
+    logLevel: config.LOG_INFO,
+
+    // Uncomment the following lines if you are using grunt's server to run the tests
+    // proxies: {
+    //   '/': 'http://localhost:9000/'
+    // },
+    // URL root prevent conflicts with the site root
+    // urlRoot: '_karma_'
+  });
+};

rd_ui/test/mocks/redash_mocks.js

@@ -0,0 +1,108 @@
+featureFlags = [];
+currentUser = {
+  id: 1,
+  name: 'John Mock',
+  email: 'john@example.com',
+  groups: ['default'],
+  permissions: [],
+  canEdit: function(object) {
+    var user_id = object.user_id || (object.user && object.user.id);
+    return user_id && (user_id == currentUser.id);
+  },
+  hasPermission: function(permission) {
+    return this.permissions.indexOf(permission) != -1;
+  }
+};
+
+
+angular.module('redashMocks', [])
+  .value('MockData', {
+    query: {
+      "ttl": -1,
+      "query": "select name from users;",
+      "id": 1803,
+      "description": "",
+      "name": "my test query",
+      "created_at": "2014-01-07T16:11:31.859528+02:00",
+      "query_hash": "c89c235bc73e462e9702debc56adc309",
+
+      "user": {
+        "email": "amirn@everything.me",
+        "id": 48,
+        "name": "Amir Nissim"
+      },
+
+      "visualizations": [{
+        "description": "",
+        "options": {},
+        "type": "TABLE",
+        "id": 636,
+        "name": "Table"
+      }],
+
+      "api_key": "123456789",
+
+      "data_source_id": 1,
+
+      "latest_query_data_id": 106632,
+
+      "latest_query_data": {
+        "retrieved_at": "2014-07-29T10:49:10.951364+03:00",
+        "query_hash": "c89c235bc73e462e9702debc56adc309",
+        "query": "select name from users;",
+        "runtime": 0.0139260292053223,
+        "data": {
+          "rows": [{
+            "name": "Amir Nissim"
+          }, {
+            "name": "Arik Fraimovich"
+          }],
+          "columns": [{
+            "friendly_name": "name",
+            "type": null,
+            "name": "name"
+          }, {
+            "friendly_name": "mail::filter",
+            "type": null,
+            "name": "mail::filter"
+          }]
+        },
+        "id": 106632,
+        "data_source_id": 1
+      }
+
+    },
+
+    queryResult: {
+      "job": {},
+      "query_result": {
+        "retrieved_at": "2014-08-04T13:33:45.563486+03:00",
+        "query_hash": "9951c38c9cf00e6ee8aecce026b51c19",
+        "query": "select name as \"name::filter\" from users",
+        "runtime": 0.00896096229553223,
+        "data": {
+          "rows": [],
+          "columns": [{
+            "friendly_name": "name::filter",
+            "type": null,
+            "name": "name::filter"
+          }]
+        },
+        "id": 106673,
+        "data_source_id": 1
+      },
+      "status": "done",
+      "filters": [],
+      "filterFreeze": "test@example.com",
+      "updatedAt": "2014-08-05T13:13:40.833Z",
+      "columnNames": ["name::filter"],
+      "filteredData": [{
+        "name::filter": "test@example.com"
+      }],
+      "columns": [{
+        "friendly_name": "name::filter",
+        "type": null,
+        "name": "name::filter"
+      }]
+    }
+  });

rd_ui/test/unit/example_test.js

@@ -0,0 +1,5 @@
+describe('example test', function() {
+  it('should expect the obvious', function() {
+    expect(0).toBe(0);
+  });
+});

rd_ui/test/unit/test_query_view.js

@@ -0,0 +1,34 @@
+'use strict';
+
+describe('QueryViewCtrl', function() {
+  var scope;
+  var MockData;
+
+  beforeEach(module('redash', 'redashMocks'));
+
+  beforeEach(inject(function($injector, $controller, $rootScope, Query, _MockData_) {
+    MockData = _MockData_;
+    scope = $rootScope.$new();
+
+    var route = {
+      current: {
+        locals: {
+          query: new Query(MockData.query)
+        }
+      }
+    };
+
+    $controller('QueryViewCtrl', {$scope: scope, $route: route});
+  }));
+
+  it('should have a query', function() {
+    expect(scope.query).toBeDefined();
+  });
+
+  it('should update the executing state', function() {
+    expect(scope.queryExecuting).toBe(false);
+    scope.executeQuery();
+    expect(scope.queryExecuting).toBe(true);
+  });
+
+});

rd_ui/test/unit/test_visualization_renderer.js

@@ -0,0 +1,89 @@
+'use strict';
+
+describe('VisualizationRenderer', function() {
+  var element;
+  var scope;
+
+  var filters = [{
+    "name": "name::filter",
+    "friendlyName": "Name",
+    "values": ["test@example.com", "amirn@example.com"],
+    "multiple": false
+  }];
+
+  beforeEach(module('redash', 'redashMocks'));
+
+  // loading templates
+  beforeEach(module('app/views/grid_renderer.html',
+    'app/views/visualizations/filters.html'));
+
+  // serving templates
+  beforeEach(inject(function($httpBackend, $templateCache) {
+    $httpBackend.whenGET('/views/grid_renderer.html')
+      .respond($templateCache.get('app/views/grid_renderer.html'));
+
+    $httpBackend.whenGET('/views/visualizations/filters.html')
+      .respond($templateCache.get('app/views/visualizations/filters.html'));
+  }));
+
+  // directive setup
+  beforeEach(inject(function($rootScope, $compile, MockData, QueryResult) {
+    var qr = new QueryResult(MockData.queryResult)
+    qr.filters = filters;
+
+    $rootScope.queryResult = qr;
+
+    element = angular.element(
+      '<visualization-renderer query-result="queryResult">' +
+      '</visualization-renderer>');
+  }));
+
+
+  describe('scope', function() {
+    beforeEach(inject(function($rootScope, $compile) {
+      $compile(element)($rootScope);
+
+      // we will test the isolated scope of the directive
+      scope = element.isolateScope();
+      scope.$digest();
+    }));
+
+    it('should have filters', function() {
+      expect(scope.filters).toBeDefined();
+    });
+  });
+
+
+  /*describe('URL binding', function() {
+
+    beforeEach(inject(function($rootScope, $compile, $location) {
+      spyOn($location, 'search').andCallThrough();
+
+      // set initial search
+      var initialSearch = {};
+      initialSearch[filters[0].friendlyName] = filters[0].values[0];
+      $location.search('filters', initialSearch);
+
+      $compile(element)($rootScope);
+
+      // we will test the isolated scope of the directive
+      scope = element.isolateScope();
+      scope.$digest();
+    }));
+
+    it('should update scope from URL',
+      inject(function($location) {
+        expect($location.search).toHaveBeenCalled();
+        expect(scope.filters[0].current).toEqual(filters[0].values[0]);
+      }));
+
+    it('should update URL from scope',
+      inject(function($location) {
+        scope.filters[0].current = 'newValue';
+        scope.$digest();
+
+        var searchFilters = angular.fromJson($location.search().filters);
+        expect(searchFilters[filters[0].friendlyName]).toEqual('newValue');
+    }));
+  });*/
+});

redash/__init__.py

@@ -1,47 +1,35 @@
-import json
+import logging
 import urlparse
-from flask import Flask, make_response
-from flask.ext.restful import Api
-from flask_peewee.db import Database
-
 import redis
 from statsd import StatsClient
-from redash import settings, utils
 
-__version__ = '0.3.5'
+from redash import settings, events
 
-app = Flask(__name__,
-            template_folder=settings.STATIC_ASSETS_PATH,
-            static_folder=settings.STATIC_ASSETS_PATH,
-            static_path='/static')
-
-api = Api(app)
-
-# configure our database
-settings.DATABASE_CONFIG.update({'threadlocals': True})
-app.config['DATABASE'] = settings.DATABASE_CONFIG
-db = Database(app)
-
-from redash.authentication import setup_authentication
-auth = setup_authentication(app)
-
-@api.representation('application/json')
-def json_representation(data, code, headers=None):
-    resp = make_response(json.dumps(data, cls=utils.JSONEncoder), code)
-    resp.headers.extend(headers or {})
-    return resp
+__version__ = '0.4.0'
 
 
-redis_url = urlparse.urlparse(settings.REDIS_URL)
-if redis_url.path:
-    redis_db = redis_url.path[1]
-else:
-    redis_db = 0
+def setup_logging():
+    handler = logging.StreamHandler()
+    formatter = logging.Formatter('[%(asctime)s][PID:%(process)d][%(levelname)s][%(name)s] %(message)s')
+    handler.setFormatter(formatter)
+    logging.getLogger().addHandler(handler)
+    logging.getLogger().setLevel(settings.LOG_LEVEL)
 
-redis_connection = redis.StrictRedis(host=redis_url.hostname, port=redis_url.port, db=redis_db, password=redis_url.password)
-statsd_client = StatsClient(host=settings.STATSD_HOST, port=settings.STATSD_PORT, prefix=settings.STATSD_PREFIX)
+    events.setup_logging(settings.EVENTS_LOG_PATH, settings.EVENTS_CONSOLE_OUTPUT)
 
-from redash import data
-data_manager = data.Manager(redis_connection, db, statsd_client)
 
-from redash import controllers
+def create_redis_connection():
+    redis_url = urlparse.urlparse(settings.REDIS_URL)
+    if redis_url.path:
+        redis_db = redis_url.path[1]
+    else:
+        redis_db = 0
+
+    r = redis.StrictRedis(host=redis_url.hostname, port=redis_url.port, db=redis_db, password=redis_url.password)
+
+    return r
+
+
+setup_logging()
+redis_connection = create_redis_connection()
+statsd_client = StatsClient(host=settings.STATSD_HOST, port=settings.STATSD_PORT, prefix=settings.STATSD_PREFIX)

redash/authentication.py

@@ -1,17 +1,20 @@
 import functools
 import hashlib
 import hmac
-from flask import current_app, request, make_response, g, redirect, url_for
-from flask.ext.googleauth import GoogleAuth, login
-from flask.ext.login import LoginManager, login_user, current_user
 import time
 import logging
+
+from flask import request, make_response, redirect, url_for
+from flask.ext.login import LoginManager, login_user, current_user
+from flask.ext.googleauth import GoogleAuth, login
 from werkzeug.contrib.fixers import ProxyFix
+
 from redash import models, settings
 
 login_manager = LoginManager()
 logger = logging.getLogger('authentication')
 
+
 def sign(key, path, expires):
     if not key:
         return None
@@ -35,18 +38,15 @@ class HMACAuthentication(object):
             calculated_signature = sign(query.api_key, request.path, expires)
 
             if query.api_key and signature == calculated_signature:
+                login_user(models.ApiUser(query.api_key), remember=False)
                 return True
 
         return False
 
-    @staticmethod
-    def is_user_logged_in():
-        return current_user.is_authenticated()
-
     def required(self, fn):
         @functools.wraps(fn)
         def decorated(*args, **kwargs):
-            if self.is_user_logged_in():
+            if current_user.is_authenticated():
                 return fn(*args, **kwargs)
 
             if self.api_key_authentication():
@@ -76,8 +76,7 @@ def create_and_login_user(app, user):
             user_object.save()
     except models.User.DoesNotExist:
         logger.debug("Creating user object (%r)", user.name)
-        user_object = models.User.create(name=user.name, email=user.email,
-                                  is_admin=(user.email in settings.ADMINS))
+        user_object = models.User.create(name=user.name, email=user.email, groups = models.User.DEFAULT_GROUPS)
 
     login_user(user_object, remember=True)
 
@@ -98,6 +97,7 @@ def setup_authentication(app):
             openid_auth._OPENID_ENDPOINT = "https://www.google.com/a/%s/o8/ud?be=o8" % settings.GOOGLE_APPS_DOMAIN
 
     login_manager.init_app(app)
+    login_manager.anonymous_user = models.AnonymousUser
     app.wsgi_app = ProxyFix(app.wsgi_app)
     app.secret_key = settings.COOKIE_SECRET
 

redash/cache.py

@@ -0,0 +1,8 @@
+from flask import make_response
+from functools import update_wrapper
+
+ONE_YEAR = 60 * 60 * 24 * 365.25
+
+headers = {
+  'Cache-Control': 'max-age=%d' % ONE_YEAR
+}

redash/controllers.py

@@ -17,12 +17,16 @@ from flask.ext.restful import Resource, abort
 from flask_login import current_user, login_user, logout_user
 
 import sqlparse
-from redash import settings, utils
-from redash import data
+import events
+from permissions import require_permission
 
-from redash import app, auth, api, redis_connection, data_manager
-from redash import models
+from redash import redis_connection, statsd_client, models, settings, utils, __version__
+from redash.wsgi import app, auth, api
 
+import logging
+from tasks import QueryTask
+
+from cache import headers as cache_headers
 
 @app.route('/ping', methods=['GET'])
 def ping():
@@ -42,13 +46,21 @@ def index(**kwargs):
 
     user = {
         'gravatar_url': gravatar_url,
-        'is_admin': current_user.is_admin,
         'id': current_user.id,
         'name': current_user.name,
-        'email': current_user.email
+        'email': current_user.email,
+        'groups': current_user.groups,
+        'permissions': current_user.permissions
     }
 
-    return render_template("index.html", user=json.dumps(user), analytics=settings.ANALYTICS)
+    features = {
+        'clientSideMetrics': settings.CLIENT_SIDE_METRICS,
+        'flowerUrl': settings.CELERY_FLOWER_URL
+    }
+
+    return render_template("index.html", user=json.dumps(user), name=settings.NAME,
+                           features=json.dumps(features),
+                           analytics=settings.ANALYTICS)
 
 
 @app.route('/login', methods=['GET', 'POST'])
@@ -68,6 +80,7 @@ def login():
             return redirect(request.args.get('next') or '/')
 
     return render_template("login.html",
+                           name=settings.NAME,
                            analytics=settings.ANALYTICS,
                            next=request.args.get('next'),
                            username=request.form.get('username', ''),
@@ -83,22 +96,35 @@ def logout():
 
 @app.route('/status.json')
 @auth.required
+@require_permission('admin')
 def status_api():
     status = {}
     info = redis_connection.info()
     status['redis_used_memory'] = info['used_memory_human']
-
+    status['version'] = __version__
     status['queries_count'] = models.Query.select().count()
     status['query_results_count'] = models.QueryResult.select().count()
     status['dashboards_count'] = models.Dashboard.select().count()
     status['widgets_count'] = models.Widget.select().count()
 
-    status['workers'] = [redis_connection.hgetall(w)
-                         for w in redis_connection.smembers('workers')]
+    status['workers'] = []
 
-    manager_status = redis_connection.hgetall('manager:status')
+    manager_status = redis_connection.hgetall('redash:status')
     status['manager'] = manager_status
-    status['manager']['queue_size'] = redis_connection.zcard('jobs')
+    status['manager']['outdated_queries_count'] = models.Query.outdated_queries().count()
+
+    queues = {}
+    for ds in models.DataSource.select():
+        for queue in (ds.queue_name, ds.scheduled_queue_name):
+            queues.setdefault(queue, set())
+            queues[queue].add(ds.name)
+
+    status['manager']['queues'] = {}
+    for queue, sources in queues.iteritems():
+        status['manager']['queues'][queue] = {
+            'data_sources': ', '.join(sources),
+            'size': redis_connection.llen(queue)
+        }
 
     return jsonify(status)
 
@@ -123,6 +149,40 @@ class BaseResource(Resource):
     def current_user(self):
         return current_user._get_current_object()
 
+    def dispatch_request(self, *args, **kwargs):
+        with statsd_client.timer('requests.{}.{}'.format(request.endpoint, request.method.lower())):
+            response = super(BaseResource, self).dispatch_request(*args, **kwargs)
+        return response
+
+
+class EventAPI(BaseResource):
+    def post(self):
+        events_list = request.get_json(force=True)
+        for event in events_list:
+            events.record_event(event)
+
+
+api.add_resource(EventAPI, '/api/events', endpoint='events')
+
+
+class MetricsAPI(BaseResource):
+    def post(self):
+        for stat_line in request.data.split():
+            stat, value = stat_line.split(':')
+            statsd_client._send_stat('client.{}'.format(stat), value, 1)
+
+        return "OK."
+
+api.add_resource(MetricsAPI, '/api/metrics/v1/send', endpoint='metrics')
+
+
+class DataSourceListAPI(BaseResource):
+    def get(self):
+        data_sources = [ds.to_dict() for ds in models.DataSource.all()]
+        return data_sources
+
+api.add_resource(DataSourceListAPI, '/api/data_sources', endpoint='data_sources')
+
 
 class DashboardListAPI(BaseResource):
     def get(self):
@@ -131,6 +191,7 @@ class DashboardListAPI(BaseResource):
 
         return dashboards
 
+    @require_permission('create_dashboard')
     def post(self):
         dashboard_properties = request.get_json(force=True)
         dashboard = models.Dashboard(name=dashboard_properties['name'],
@@ -149,6 +210,7 @@ class DashboardAPI(BaseResource):
 
         return dashboard.to_dict(with_widgets=True)
 
+    @require_permission('edit_dashboard')
     def post(self, dashboard_slug):
         dashboard_properties = request.get_json(force=True)
         # TODO: either convert all requests to use slugs or ids
@@ -159,6 +221,7 @@ class DashboardAPI(BaseResource):
 
         return dashboard.to_dict(with_widgets=True)
 
+    @require_permission('edit_dashboard')
     def delete(self, dashboard_slug):
         dashboard = models.Dashboard.get_by_slug(dashboard_slug)
         dashboard.is_archived = True
@@ -169,6 +232,7 @@ api.add_resource(DashboardAPI, '/api/dashboards/<dashboard_slug>', endpoint='das
 
 
 class WidgetListAPI(BaseResource):
+    @require_permission('edit_dashboard')
     def post(self):
         widget_properties = request.get_json(force=True)
         widget_properties['options'] = json.dumps(widget_properties['options'])
@@ -200,6 +264,7 @@ class WidgetListAPI(BaseResource):
 
 
 class WidgetAPI(BaseResource):
+    @require_permission('edit_dashboard')
     def delete(self, widget_id):
         widget = models.Widget.get(models.Widget.id == widget_id)
         # TODO: reposition existing ones
@@ -216,25 +281,28 @@ api.add_resource(WidgetAPI, '/api/widgets/<int:widget_id>', endpoint='widget')
 
 
 class QueryListAPI(BaseResource):
+    @require_permission('create_query')
     def post(self):
         query_def = request.get_json(force=True)
-        # id, created_at, api_key
         for field in ['id', 'created_at', 'api_key', 'visualizations', 'latest_query_data']:
             query_def.pop(field, None)
 
         query_def['user'] = self.current_user
+        query_def['data_source'] = query_def.pop('data_source_id')
         query = models.Query(**query_def)
         query.save()
 
         query.create_default_visualizations()
 
-        return query.to_dict(with_result=False)
+        return query.to_dict()
 
+    @require_permission('view_query')
     def get(self):
-        return [q.to_dict(with_result=False, with_stats=True) for q in models.Query.all_queries()]
+        return [q.to_dict(with_stats=True) for q in models.Query.all_queries()]
 
 
 class QueryAPI(BaseResource):
+    @require_permission('edit_query')
     def post(self, query_id):
         query_def = request.get_json(force=True)
         for field in ['id', 'created_at', 'api_key', 'visualizations', 'latest_query_data', 'user']:
@@ -243,12 +311,16 @@ class QueryAPI(BaseResource):
         if 'latest_query_data_id' in query_def:
             query_def['latest_query_data'] = query_def.pop('latest_query_data_id')
 
+        if 'data_source_id' in query_def:
+            query_def['data_source'] = query_def.pop('data_source_id')
+
         models.Query.update_instance(query_id, **query_def)
 
         query = models.Query.get_by_id(query_id)
 
-        return query.to_dict(with_result=False, with_visualizations=True)
+        return query.to_dict(with_visualizations=True)
 
+    @require_permission('view_query')
     def get(self, query_id):
         q = models.Query.get(models.Query.id == query_id)
         if q:
@@ -261,6 +333,7 @@ api.add_resource(QueryAPI, '/api/queries/<query_id>', endpoint='query')
 
 
 class VisualizationListAPI(BaseResource):
+    @require_permission('edit_query')
     def post(self):
         kwargs = request.get_json(force=True)
         kwargs['options'] = json.dumps(kwargs['options'])
@@ -273,11 +346,13 @@ class VisualizationListAPI(BaseResource):
 
 
 class VisualizationAPI(BaseResource):
+    @require_permission('edit_query')
     def post(self, visualization_id):
         kwargs = request.get_json(force=True)
         if 'options' in kwargs:
             kwargs['options'] = json.dumps(kwargs['options'])
         kwargs.pop('id', None)
+        kwargs.pop('query_id', None)
 
         update = models.Visualization.update(**kwargs).where(models.Visualization.id == visualization_id)
         update.execute()
@@ -286,6 +361,7 @@ class VisualizationAPI(BaseResource):
 
         return vis.to_dict(with_query=False)
 
+    @require_permission('edit_query')
     def delete(self, visualization_id):
         vis = models.Visualization.get(models.Visualization.id == visualization_id)
         vis.delete_instance()
@@ -295,9 +371,28 @@ api.add_resource(VisualizationAPI, '/api/visualizations/<visualization_id>', end
 
 
 class QueryResultListAPI(BaseResource):
+    @require_permission('execute_query')
     def post(self):
         params = request.json
-        
+
+        if settings.FEATURE_TABLES_PERMISSIONS:
+            metadata = utils.SQLMetaData(params['query'])
+
+            if metadata.has_non_select_dml_statements or metadata.has_ddl_statements:
+                return {
+                    'job': {
+                        'error': 'Only SELECT statements are allowed'
+                    }
+                }
+
+            if len(metadata.used_tables - current_user.allowed_tables) > 0 and '*' not in current_user.allowed_tables:
+                logging.warning('Permission denied for user %s to table %s', self.current_user.name, metadata.used_tables)
+                return {
+                    'job': {
+                        'error': 'Access denied for table(s): %s' % (metadata.used_tables)
+                    }
+                }
+
         models.ActivityLog(
             user=self.current_user,
             type=models.ActivityLog.QUERY_EXECUTION,
@@ -307,65 +402,73 @@ class QueryResultListAPI(BaseResource):
         if params['ttl'] == 0:
             query_result = None
         else:
-            query_result = data_manager.get_query_result(params['query'], int(params['ttl']))
+            query_result = models.QueryResult.get_latest(params['data_source_id'], params['query'], int(params['ttl']))
 
         if query_result:
-            return {'query_result': query_result.to_dict(parse_data=True)}
+            return {'query_result': query_result.to_dict()}
         else:
-            job = data_manager.add_job(params['query'], data.Job.HIGH_PRIORITY)
+            data_source = models.DataSource.get_by_id(params['data_source_id'])
+            job = QueryTask.add_task(params['query'], data_source)
             return {'job': job.to_dict()}
 
 
 class QueryResultAPI(BaseResource):
-    def get(self, query_result_id):
-        query_result = data_manager.get_query_result_by_id(query_result_id)
-        if query_result:
-            return {'query_result': query_result.to_dict(parse_data=True)}
-        else:
-            abort(404)
+    @staticmethod
+    def csv_response(query_result):
+        s = cStringIO.StringIO()
 
+        query_data = json.loads(query_result.data)
+        writer = csv.DictWriter(s, fieldnames=[col['name'] for col in query_data['columns']])
+        writer.writer = utils.UnicodeWriter(s)
+        writer.writeheader()
+        for row in query_data['rows']:
+            for k, v in row.iteritems():
+                if isinstance(v, numbers.Number) and (v > 1000 * 1000 * 1000 * 100):
+                    row[k] = datetime.datetime.fromtimestamp(v/1000.0)
 
-class CsvQueryResultsAPI(BaseResource):
-    def get(self, query_id, query_result_id=None):
-        if not query_result_id:
+            writer.writerow(row)
+
+        headers = {'Content-Type': "text/csv; charset=UTF-8"}
+        headers.update(cache_headers)
+        return make_response(s.getvalue(), 200, headers)
+
+    @require_permission('view_query')
+    def get(self, query_id=None, query_result_id=None, filetype='json'):
+        if query_result_id is None and query_id is not None:
             query = models.Query.get(models.Query.id == query_id)
             if query:
                 query_result_id = query._data['latest_query_data']
 
-        query_result = query_result_id and data_manager.get_query_result_by_id(query_result_id)
+        if query_result_id:
+            query_result = models.QueryResult.get_by_id(query_result_id)
+
         if query_result:
-            s = cStringIO.StringIO()
+            if filetype == 'json':
+                data = json.dumps({'query_result': query_result.to_dict()}, cls=utils.JSONEncoder)
+                return make_response(data, 200, cache_headers)
+            else:
+                return self.csv_response(query_result)
 
-            query_data = json.loads(query_result.data)
-            writer = csv.DictWriter(s, fieldnames=[col['name'] for col in query_data['columns']])
-            writer.writer = utils.UnicodeWriter(s)
-            writer.writeheader()
-            for row in query_data['rows']:
-                for k, v in row.iteritems():
-                    if isinstance(v, numbers.Number) and (v > 1000 * 1000 * 1000 * 100):
-                        row[k] = datetime.datetime.fromtimestamp(v/1000.0)
-
-                writer.writerow(row)
-
-            return make_response(s.getvalue(), 200, {'Content-Type': "text/csv; charset=UTF-8"})
         else:
             abort(404)
 
-api.add_resource(CsvQueryResultsAPI, '/api/queries/<query_id>/results/<query_result_id>.csv',
-                 '/api/queries/<query_id>/results.csv',
-                 endpoint='csv_query_results')
+
 api.add_resource(QueryResultListAPI, '/api/query_results', endpoint='query_results')
-api.add_resource(QueryResultAPI, '/api/query_results/<query_result_id>', endpoint='query_result')
+api.add_resource(QueryResultAPI,
+                 '/api/query_results/<query_result_id>',
+                 '/api/queries/<query_id>/results.<filetype>',
+                 '/api/queries/<query_id>/results/<query_result_id>.<filetype>',
+                 endpoint='query_result')
 
 
 class JobAPI(BaseResource):
     def get(self, job_id):
         # TODO: if finished, include the query result
-        job = data.Job.load(data_manager.redis_connection, job_id)
+        job = QueryTask(job_id=job_id)
         return {'job': job.to_dict()}
 
     def delete(self, job_id):
-        job = data.Job.load(data_manager.redis_connection, job_id)
+        job = QueryTask(job_id=job_id)
         job.cancel()
 
 api.add_resource(JobAPI, '/api/jobs/<job_id>', endpoint='job')

redash/data/__init__.py

@@ -1,2 +0,0 @@
-from manager import Manager
-from worker import Job

redash/data/manager.py

@@ -1,221 +0,0 @@
-"""
-Data manager. Used to manage and coordinate execution of queries.
-"""
-from contextlib import contextmanager
-import collections
-import json
-import time
-import logging
-import psycopg2
-import qr
-import redis
-from redash.data import worker
-from redash.utils import gen_query_hash
-
-class QueryResult(collections.namedtuple('QueryData', 'id query data runtime retrieved_at query_hash')):
-    def to_dict(self, parse_data=False):
-        d = self._asdict()
-
-        if parse_data and d['data']:
-            d['data'] = json.loads(d['data'])
-
-        return d
-
-
-class Manager(object):
-    def __init__(self, redis_connection, db, statsd_client):
-        self.statsd_client = statsd_client
-        self.redis_connection = redis_connection
-        self.db = db
-        self.workers = []
-        self.queue = qr.PriorityQueue("jobs", **self.redis_connection.connection_pool.connection_kwargs)
-        self.max_retries = 5
-        self.status = {
-            'last_refresh_at': 0,
-            'started_at': time.time()
-        }
-
-        self._save_status()
-
-    # TODO: Use our Django Models
-    def get_query_result_by_id(self, query_result_id):
-        with self.db_transaction() as cursor:
-            sql = "SELECT id, query, data, runtime, retrieved_at, query_hash FROM query_results " \
-                  "WHERE id=%s LIMIT 1"
-            cursor.execute(sql, (query_result_id,))
-            query_result = cursor.fetchone()
-
-        if query_result:
-            query_result = QueryResult(*query_result)
-
-        return query_result
-
-    def get_query_result(self, query, ttl=0):
-        query_hash = gen_query_hash(query)
-
-        with self.db_transaction() as cursor:
-            sql = "SELECT id, query, data, runtime, retrieved_at, query_hash FROM query_results " \
-                  "WHERE query_hash=%s " \
-                  "AND retrieved_at < now() at time zone 'utc' - interval '%s second'" \
-                  "ORDER BY retrieved_at DESC LIMIT 1"
-            cursor.execute(sql, (query_hash, psycopg2.extensions.AsIs(ttl)))
-            query_result = cursor.fetchone()
-
-        if query_result:
-            query_result = QueryResult(*query_result)
-
-        return query_result
-
-    def add_job(self, query, priority):
-        query_hash = gen_query_hash(query)
-        logging.info("[Manager][%s] Inserting job with priority=%s", query_hash, priority)
-        try_count = 0
-        job = None
-
-        while try_count < self.max_retries:
-            try_count += 1
-
-            pipe = self.redis_connection.pipeline()
-            try:
-                pipe.watch('query_hash_job:%s' % query_hash)
-                job_id = pipe.get('query_hash_job:%s' % query_hash)
-                if job_id:
-                    logging.info("[Manager][%s] Found existing job: %s", query_hash, job_id)
-                    job = worker.Job.load(self.redis_connection, job_id)
-                else:
-                    job = worker.Job(self.redis_connection, query, priority)
-                    pipe.multi()
-                    job.save(pipe)
-                    logging.info("[Manager][%s] Created new job: %s", query_hash, job.id)
-                    self.queue.push(job.id, job.priority)
-                break
-
-            except redis.WatchError:
-                continue
-
-        if not job:
-            logging.error("[Manager][%s] Failed adding job for query.", query_hash)
-
-        return job
-
-    def report_status(self):
-        workers = [self.redis_connection.hgetall(w)
-                   for w in self.redis_connection.smembers('workers')]
-
-        for w in workers:
-            self.statsd_client.gauge('worker_{}.seconds_since_update'.format(w['id']),
-                                     time.time() - float(w['updated_at']))
-            self.statsd_client.gauge('worker_{}.jobs_received'.format(w['id']), int(w['jobs_count']))
-            self.statsd_client.gauge('worker_{}.jobs_done'.format(w['id']), int(w['done_jobs_count']))
-
-        manager_status = self.redis_connection.hgetall('manager:status')
-        self.statsd_client.gauge('manager.seconds_since_refresh',
-                                 time.time() - float(manager_status['last_refresh_at']))
-
-    def refresh_queries(self):
-        sql = """SELECT first_value(t1."query") over(partition by t1.query_hash)
-                 FROM "queries" AS t1
-                 INNER JOIN "query_results" AS t2 ON (t1."latest_query_data_id" = t2."id")
-                 WHERE ((t1."ttl" > 0) AND ((t2."retrieved_at" + t1.ttl * interval '1 second') <
-                                           now() at time zone 'utc'));
-        """
-
-        self.status['last_refresh_at'] = time.time()
-        self._save_status()
-
-        logging.info("Refreshing queries...")
-        queries = self.run_query(sql)
-
-        for query in queries:
-            self.add_job(query[0], worker.Job.LOW_PRIORITY)
-
-        self.statsd_client.gauge('manager.outdated_queries', len(queries))
-        self.statsd_client.gauge('manager.queue_size', self.redis_connection.zcard('jobs'))
-
-        logging.info("Done refreshing queries... %d" % len(queries))
-
-    def store_query_result(self, query, data, run_time, retrieved_at):
-        query_result_id = None
-        query_hash = gen_query_hash(query)
-        sql = "INSERT INTO query_results (query_hash, query, data, runtime, retrieved_at) " \
-              "VALUES (%s, %s, %s, %s, %s) RETURNING id"
-        with self.db_transaction() as cursor:
-            cursor.execute(sql, (query_hash, query, data, run_time, retrieved_at))
-            if cursor.rowcount == 1:
-                query_result_id = cursor.fetchone()[0]
-                logging.info("[Manager][%s] Inserted query data; id=%s", query_hash, query_result_id)
-
-                sql = "UPDATE queries SET latest_query_data_id=%s WHERE query_hash=%s"
-                cursor.execute(sql, (query_result_id, query_hash))
-
-                logging.info("[Manager][%s] Updated %s queries.", query_hash, cursor.rowcount)
-            else:
-                logging.error("[Manager][%s] Failed inserting query data.", query_hash)
-        return query_result_id
-
-    def run_query(self, *args):
-        sql = args[0]
-        logging.debug("running query: %s %s", sql, args[1:])
-
-        with self.db_transaction() as cursor:
-            cursor.execute(sql, args[1:])
-            if cursor.description:
-                data = list(cursor)
-            else:
-                data = cursor.rowcount
-
-        return data
-
-    def start_workers(self, workers_count, connection_type, connection_string):
-        if self.workers:
-            return self.workers
-
-        if connection_type == 'mysql':
-            from redash.data import query_runner_mysql
-            runner = query_runner_mysql.mysql(connection_string)
-        elif connection_type == 'graphite':
-            from redash.data import query_runner_graphite
-            connection_params = json.loads(connection_string)
-            if connection_params['auth']:
-                connection_params['auth'] = tuple(connection_params['auth'])
-            else:
-                connection_params['auth'] = None
-            runner = query_runner_graphite.graphite(connection_params)
-        elif connection_type == 'bigquery':
-            from redash.data import query_runner_bigquery
-            connection_params = json.loads(connection_string)
-            runner = query_runner_bigquery.bigquery(connection_params)
-        else:
-            from redash.data import query_runner
-            runner = query_runner.redshift(connection_string)
-
-        redis_connection_params = self.redis_connection.connection_pool.connection_kwargs
-        self.workers = [worker.Worker(worker_id, self, redis_connection_params, runner)
-                        for worker_id in range(workers_count)]
-        for w in self.workers:
-            w.start()
-
-        return self.workers
-
-    def stop_workers(self):
-        for w in self.workers:
-            w.continue_working = False
-            w.join()
-
-    @contextmanager
-    def db_transaction(self):
-        self.db.connect_db()
-
-        cursor = self.db.database.get_cursor()
-        try:
-            yield cursor
-        except:
-            self.db.database.rollback()
-            raise
-        else:
-            self.db.database.commit()
-        finally:
-            self.db.close_db(None)
-
-    def _save_status(self):
-        self.redis_connection.hmset('manager:status', self.status)

redash/data/query_runner.py

@@ -1,69 +1,30 @@
-"""
-QueryRunner is the function that the workers use, to execute queries. This is the Redshift
-(PostgreSQL in fact) version, but easily we can write another to support additional databases
-(MySQL and others).
-
-Because the worker just pass the query, this can be used with any data store that has some sort of
-query language (for example: HiveQL).
-"""
-import logging
 import json
-import sys
-import select
 
-import psycopg2
 
-from redash.utils import JSONEncoder
+def get_query_runner(connection_type, connection_string):
+    if connection_type == 'mysql':
+        from redash.data import query_runner_mysql
+        runner = query_runner_mysql.mysql(connection_string)
+    elif connection_type == 'graphite':
+        from redash.data import query_runner_graphite
+        connection_params = json.loads(connection_string)
+        if connection_params['auth']:
+            connection_params['auth'] = tuple(connection_params['auth'])
+        else:
+            connection_params['auth'] = None
+        runner = query_runner_graphite.graphite(connection_params)
+    elif connection_type == 'bigquery':
+        from redash.data import query_runner_bigquery
+        connection_params = json.loads(connection_string)
+        runner = query_runner_bigquery.bigquery(connection_params)
+    elif connection_type == 'script':
+        from redash.data import query_runner_script
+        runner = query_runner_script.script(connection_string)
+    elif connection_type == 'url':
+        from redash.data import query_runner_url
+        runner = query_runner_url.url(connection_string)
+    else:
+        from redash.data import query_runner_pg
+        runner = query_runner_pg.pg(connection_string)
 
-def redshift(connection_string):
-    def column_friendly_name(column_name):
-        return column_name
- 
-    def wait(conn):
-        while 1:
-            state = conn.poll()
-            if state == psycopg2.extensions.POLL_OK:
-                break
-            elif state == psycopg2.extensions.POLL_WRITE:
-                select.select([], [conn.fileno()], [])
-            elif state == psycopg2.extensions.POLL_READ:
-                select.select([conn.fileno()], [], [])
-            else:
-                raise psycopg2.OperationalError("poll() returned %s" % state)
- 
-    def query_runner(query):
-        connection = psycopg2.connect(connection_string, async=True)
-        wait(connection)
- 
-        cursor = connection.cursor()
- 
-        try:
-            cursor.execute(query)
-            wait(connection)
- 
-            column_names = [col.name for col in cursor.description]
- 
-            rows = [dict(zip(column_names, row)) for row in cursor]
-            columns = [{'name': col.name,
-                        'friendly_name': column_friendly_name(col.name),
-                        'type': None} for col in cursor.description]
- 
-            data = {'columns': columns, 'rows': rows}
-            json_data = json.dumps(data, cls=JSONEncoder)
-            error = None
-            cursor.close()
-        except psycopg2.DatabaseError as e:
-            json_data = None
-            error = e.message
-        except KeyboardInterrupt:
-            connection.cancel()
-            error = "Query cancelled by user."
-            json_data = None
-        except Exception as e:
-            raise sys.exc_info()[1], None, sys.exc_info()[2]
-        finally:
-            connection.close()
- 
-        return json_data, error
- 
-    return query_runner
+    return runner

redash/data/query_runner_bigquery.py

@@ -1,7 +1,9 @@
+import datetime
 import httplib2
 import json
 import logging
 import sys
+import time
 
 try:
     import apiclient.errors
@@ -14,6 +16,39 @@ except ImportError:
 
 from redash.utils import JSONEncoder
 
+types_map = {
+    'INTEGER': 'integer',
+    'FLOAT': 'float',
+    'BOOLEAN': 'boolean',
+    'STRING': 'string',
+    'TIMESTAMP': 'datetime',
+}
+
+def transform_row(row, fields):
+    column_index = 0
+    row_data = {}
+
+    for cell in row["f"]:
+        field = fields[column_index]
+        cell_value = cell['v']
+
+        if cell_value is None:
+            pass
+        # Otherwise just cast the value
+        elif field['type'] == 'INTEGER':
+            cell_value = int(cell_value)
+        elif field['type'] == 'FLOAT':
+            cell_value = float(cell_value)
+        elif field['type'] == 'BOOLEAN':
+            cell_value = cell_value.lower() == "true"
+        elif field['type'] == 'TIMESTAMP':
+            cell_value = datetime.datetime.fromtimestamp(float(cell_value))
+
+        row_data[field["name"]] = cell_value
+        column_index += 1
+
+    return row_data
+
 def bigquery(connection_string):
     def load_key(filename):
         f = file(filename, "rb")
@@ -27,12 +62,22 @@ def bigquery(connection_string):
             "https://www.googleapis.com/auth/bigquery",
         ]
 
-        credentials = SignedJwtAssertionCredentials(connection_string["serviceAccount"], load_key(connection_string["privateKey"]), scope=scope)
+        credentials = SignedJwtAssertionCredentials(connection_string["serviceAccount"],
+                                                    load_key(connection_string["privateKey"]), scope=scope)
         http = httplib2.Http()
         http = credentials.authorize(http)
 
         return build("bigquery", "v2", http=http)
 
+    def get_query_results(jobs, project_id, job_id, start_index):
+        query_reply = jobs.getQueryResults(projectId=project_id, jobId=job_id, startIndex=start_index).execute()
+        logging.debug('query_reply %s', query_reply)
+        if not query_reply['jobComplete']:
+            time.sleep(10)
+            return get_query_results(jobs, project_id, job_id, start_index)
+
+        return query_reply
+
     def query_runner(query):
         bigquery_service = get_bigquery_service()
 
@@ -52,44 +97,39 @@ def bigquery(connection_string):
         try:
             insert_response = jobs.insert(projectId=project_id, body=job_data).execute()
             current_row = 0
-            query_reply = jobs.getQueryResults(projectId=project_id, jobId=insert_response['jobReference']['jobId'], startIndex=current_row).execute()
+            query_reply = get_query_results(jobs, project_id=project_id,
+                                            job_id=insert_response['jobReference']['jobId'], start_index=current_row)
+
+            logging.debug("bigquery replied: %s", query_reply)
 
             rows = []
-            field_names = []
-            for f in query_reply["schema"]["fields"]:
-                field_names.append(f["name"])
 
-            while(("rows" in query_reply) and current_row < query_reply['totalRows']):
+            while ("rows" in query_reply) and current_row < query_reply['totalRows']:
                 for row in query_reply["rows"]:
-                    row_data = {}
-                    column_index = 0
-                    for cell in row["f"]:
-                        row_data[field_names[column_index]] = cell["v"]
-                        column_index += 1
-
-                    rows.append(row_data)
+                    rows.append(transform_row(row, query_reply["schema"]["fields"]))
 
                 current_row += len(query_reply['rows'])
-                query_reply = jobs.getQueryResults(projectId=project_id, jobId=query_reply['jobReference']['jobId'], startIndex=current_row).execute()
+                query_reply = jobs.getQueryResults(projectId=project_id, jobId=query_reply['jobReference']['jobId'],
+                                                   startIndex=current_row).execute()
 
-            columns = [{'name': name,
-                        'friendly_name': name,
-                        'type': None} for name in field_names]
+            columns = [{'name': f["name"],
+                        'friendly_name': f["name"],
+                        'type': types_map.get(f['type'], "string")} for f in query_reply["schema"]["fields"]]
 
             data = {
-                "columns" : columns,
-                "rows" : rows
+                "columns": columns,
+                "rows": rows
             }
             error = None
 
             json_data = json.dumps(data, cls=JSONEncoder)
         except apiclient.errors.HttpError, e:
             json_data = None
-            error = e.args[1]
+            error = e.content
         except KeyboardInterrupt:
             error = "Query cancelled by user."
             json_data = None
-        except Exception as e:
+        except Exception:
             raise sys.exc_info()[1], None, sys.exc_info()[2]
 
         return json_data, error

redash/data/query_runner_mysql.py

@@ -18,7 +18,7 @@ def mysql(connection_string):
     
     def query_runner(query):
         connections_params = [entry.split('=')[1] for entry in connection_string.split(';')]
-        connection = MySQLdb.connect(*connections_params)
+        connection = MySQLdb.connect(*connections_params, charset="utf8", use_unicode=True)
         cursor = connection.cursor()
 
         logging.debug("mysql got query: %s", query)
@@ -61,4 +61,4 @@ def mysql(connection_string):
         return json_data, error
         
     
-    return query_runner
+    return query_runner

redash/data/query_runner_pg.py

@@ -0,0 +1,110 @@
+"""
+QueryRunner is the function that the workers use, to execute queries. This is the PostgreSQL
+version, but easily we can write another to support additional databases (MySQL and others).
+
+Because the worker just pass the query, this can be used with any data store that has some sort of
+query language (for example: HiveQL).
+"""
+import json
+import sys
+import select
+import logging
+import psycopg2
+
+from redash.utils import JSONEncoder
+
+types_map = {
+    20: 'integer',
+    21: 'integer',
+    23: 'integer',
+    700: 'float',
+    1700: 'float',
+    701: 'float',
+    16: 'boolean',
+    1082: 'date',
+    1114: 'datetime',
+    1184: 'datetime',
+    1014: 'string',
+    1015: 'string',
+    1008: 'string',
+    1009: 'string',
+    2951: 'string'
+}
+
+
+def pg(connection_string):
+    def column_friendly_name(column_name):
+        return column_name
+
+    def wait(conn):
+        while 1:
+            try:
+                state = conn.poll()
+                if state == psycopg2.extensions.POLL_OK:
+                    break
+                elif state == psycopg2.extensions.POLL_WRITE:
+                    select.select([], [conn.fileno()], [])
+                elif state == psycopg2.extensions.POLL_READ:
+                    select.select([conn.fileno()], [], [])
+                else:
+                    raise psycopg2.OperationalError("poll() returned %s" % state)
+            except select.error:
+                raise psycopg2.OperationalError("select.error received")
+
+    def query_runner(query):
+        connection = psycopg2.connect(connection_string, async=True)
+        wait(connection)
+
+        cursor = connection.cursor()
+
+        try:
+            cursor.execute(query)
+            wait(connection)
+
+            # While set would be more efficient here, it sorts the data which is not what we want, but due to the small
+            # size of the data we can assume it's ok.
+            column_names = []
+            columns = []
+            duplicates_counter = 1
+
+            for column in cursor.description:
+                # TODO: this deduplication needs to be generalized and reused in all query runners.
+                column_name = column.name
+                if column_name in column_names:
+                    column_name = column_name + str(duplicates_counter)
+                    duplicates_counter += 1
+
+                column_names.append(column_name)
+
+                columns.append({
+                    'name': column_name,
+                    'friendly_name': column_friendly_name(column_name),
+                    'type': types_map.get(column.type_code, None)
+                })
+
+            rows = [dict(zip(column_names, row)) for row in cursor]
+
+            data = {'columns': columns, 'rows': rows}
+            json_data = json.dumps(data, cls=JSONEncoder)
+            error = None
+            cursor.close()
+        except (select.error, OSError) as e:
+            logging.exception(e)
+            error = "Query interrupted. Please retry."
+            json_data = None
+        except psycopg2.DatabaseError as e:
+            logging.exception(e)
+            json_data = None
+            error = e.message
+        except KeyboardInterrupt:
+            connection.cancel()
+            error = "Query cancelled by user."
+            json_data = None
+        except Exception as e:
+            raise sys.exc_info()[1], None, sys.exc_info()[2]
+        finally:
+            connection.close()
+
+        return json_data, error
+
+    return query_runner

redash/data/query_runner_script.py

@@ -0,0 +1,48 @@
+import json
+import logging
+import sys
+import os
+import subprocess
+
+# We use subprocess.check_output because we are lazy.
+# If someone will really want to run this on Python < 2.7 they can easily update the code to run
+# Popen, check the retcodes and other things and read the standard output to a variable.
+if not "check_output" in subprocess.__dict__:
+    print "ERROR: This runner uses subprocess.check_output function which exists in Python 2.7"
+
+def script(connection_string):
+
+    def query_runner(query):
+        try:
+            json_data = None
+            error = None
+
+            # Poor man's protection against running scripts from output the scripts directory
+            if connection_string.find("../") > -1:
+                return None, "Scripts can only be run from the configured scripts directory"
+
+            query = query.strip()
+
+            script = os.path.join(connection_string, query)
+            if not os.path.exists(script):
+                return None, "Script '%s' not found in script directory" % query
+
+            output = subprocess.check_output(script, shell=False)
+            if output != None:
+                output = output.strip()
+                if output != "":
+                    return output, None
+
+            error = "Error reading output"
+        except subprocess.CalledProcessError as e:
+            return None, str(e)
+        except KeyboardInterrupt:
+            error = "Query cancelled by user."
+            json_data = None
+        except Exception as e:
+            raise sys.exc_info()[1], None, sys.exc_info()[2]
+
+        return json_data, error
+
+    query_runner.annotate_query = False
+    return query_runner

redash/data/query_runner_url.py

@@ -0,0 +1,45 @@
+import json
+import logging
+import sys
+import os
+import urllib2
+
+def url(connection_string):
+
+    def query_runner(query):
+        base_url = connection_string
+
+        try:
+            json_data = None
+            error = None
+
+            query = query.strip()
+
+            if base_url is not None and base_url != "":
+                if query.find("://") > -1:
+                    return None, "Accepting only relative URLs to '%s'" % base_url
+
+            if base_url is None:
+                base_url = ""
+
+            url = base_url + query
+
+            json_data = urllib2.urlopen(url).read().strip()
+
+            if not json_data:
+                error = "Error reading data from '%s'" % url
+
+            return json_data, error
+
+        except urllib2.URLError as e:
+            return None, str(e)
+        except KeyboardInterrupt:
+            error = "Query cancelled by user."
+            json_data = None
+        except Exception as e:
+            raise sys.exc_info()[1], None, sys.exc_info()[2]
+
+        return json_data, error
+
+    query_runner.annotate_query = False
+    return query_runner

redash/data/worker.py

@@ -1,264 +0,0 @@
-"""
-Worker implementation to execute incoming queries.
-"""
-import json
-import logging
-import os
-import threading
-import uuid
-import datetime
-import time
-import signal
-import setproctitle
-import redis
-from statsd import StatsClient
-from redash.utils import gen_query_hash
-from redash import settings
-
-class Job(object):
-    HIGH_PRIORITY = 1
-    LOW_PRIORITY = 2
-
-    WAITING = 1
-    PROCESSING = 2
-    DONE = 3
-    FAILED = 4
-
-    def __init__(self, redis_connection, query, priority,
-                 job_id=None,
-                 wait_time=None, query_time=None,
-                 updated_at=None, status=None, error=None, query_result_id=None,
-                 process_id=0):
-        self.redis_connection = redis_connection
-        self.query = query
-        self.priority = priority
-        self.query_hash = gen_query_hash(self.query)
-        self.query_result_id = query_result_id
-        if process_id == 'None':
-            self.process_id = None
-        else:
-            self.process_id = int(process_id)
-
-        if job_id is None:
-            self.id = str(uuid.uuid1())
-            self.new_job = True
-            self.wait_time = 0
-            self.query_time = 0
-            self.error = None
-            self.updated_at = time.time() # job_dict.get('updated_at', time.time())
-            self.status = self.WAITING # int(job_dict.get('status', self.WAITING))
-        else:
-            self.id = job_id
-            self.new_job = False
-            self.error = error
-            self.wait_time = wait_time
-            self.query_time = query_time
-            self.updated_at = updated_at
-            self.status = status
-
-    def to_dict(self):
-        return {
-            'query': self.query,
-            'priority': self.priority,
-            'id': self.id,
-            'wait_time': self.wait_time,
-            'query_time': self.query_time,
-            'updated_at': self.updated_at,
-            'status': self.status,
-            'error': self.error,
-            'query_result_id': self.query_result_id,
-            'process_id': self.process_id
-        }
-
-    @staticmethod
-    def _redis_key(job_id):
-        return 'job:%s' % job_id
-
-    def cancel(self):
-        # TODO: Race condition:
-        # it's possible that it will be picked up by worker while processing the cancel order
-        if self.is_finished():
-            return
-
-        if self.status == self.PROCESSING:
-            os.kill(self.process_id, signal.SIGINT)
-        else:
-            self.done(None, "Interrupted/Cancelled while running.")
-
-    def save(self, pipe=None):
-        if not pipe:
-            pipe = self.redis_connection.pipeline()
-
-        if self.new_job:
-            pipe.set('query_hash_job:%s' % self.query_hash, self.id)
-
-        if self.is_finished():
-            pipe.delete('query_hash_job:%s' % self.query_hash)
-
-        pipe.sadd('jobs_set', self.id)
-        pipe.hmset(self._redis_key(self.id), self.to_dict())
-        pipe.publish(self._redis_key(self.id), json.dumps(self.to_dict()))
-        pipe.execute()
-
-    def processing(self, process_id):
-        self.status = self.PROCESSING
-        self.process_id = process_id
-        self.wait_time = time.time() - self.updated_at
-        self.updated_at = time.time()
-        self.save()
-
-    def is_finished(self):
-        return self.status in (self.FAILED, self.DONE)
-
-    def done(self, query_result_id, error):
-        if error:
-            self.status = self.FAILED
-        else:
-            self.status = self.DONE
-
-        self.query_result_id = query_result_id
-        self.error = error
-        self.query_time = time.time() - self.updated_at
-        self.updated_at = time.time()
-        self.save()
-
-    def __str__(self):
-        return "<Job:%s,priority:%d,status:%d>" % (self.id, self.priority, self.status)
-
-    @classmethod
-    def _load(cls, redis_connection, job_id):
-        return redis_connection.hgetall(cls._redis_key(job_id))
-
-    @classmethod
-    def load(cls, redis_connection, job_id):
-        job_dict = cls._load(redis_connection, job_id)
-        job = None
-        if job_dict:
-            job = Job(redis_connection, job_id=job_dict['id'], query=job_dict['query'].decode('utf-8'),
-                      priority=int(job_dict['priority']), updated_at=float(job_dict['updated_at']),
-                      status=int(job_dict['status']), wait_time=float(job_dict['wait_time']),
-                      query_time=float(job_dict['query_time']), error=job_dict['error'],
-                      query_result_id=job_dict['query_result_id'],
-                      process_id=job_dict['process_id'])
-
-        return job
-
-
-class Worker(threading.Thread):
-    def __init__(self, worker_id, manager, redis_connection_params, query_runner, sleep_time=0.1):
-        self.manager = manager
-
-        self.statsd_client = StatsClient(host=settings.STATSD_HOST, port=settings.STATSD_PORT,
-                                         prefix=settings.STATSD_PREFIX)
-        self.redis_connection_params = {k: v for k, v in redis_connection_params.iteritems()
-                                        if k in ('host', 'db', 'password', 'port')}
-        self.continue_working = True
-        self.query_runner = query_runner
-        self.sleep_time = sleep_time
-        self.child_pid = None
-        self.worker_id = worker_id
-        self.status = {
-            'id': self.worker_id,
-            'jobs_count': 0,
-            'cancelled_jobs_count': 0,
-            'done_jobs_count': 0,
-            'updated_at': time.time(),
-            'started_at': time.time()
-        }
-        self._save_status()
-        self.manager.redis_connection.sadd('workers', self._key)
-
-        super(Worker, self).__init__(name="Worker-%s" % self.worker_id)
-
-    def set_title(self, title=None):
-        base_title = "redash worker:%s" % self.worker_id
-        if title:
-            full_title = "%s - %s" % (base_title, title)
-        else:
-            full_title = base_title
-
-        setproctitle.setproctitle(full_title)
-
-    def run(self):
-        logging.info("[%s] started.", self.name)
-        while self.continue_working:
-            job_id = self.manager.queue.pop()
-            if job_id:
-                self._update_status('jobs_count')
-                logging.info("[%s] Processing %s", self.name, job_id)
-                self._fork_and_process(job_id)
-                if self.child_pid == 0:
-                    return
-            else:
-                time.sleep(self.sleep_time)
-
-    def _update_status(self, counter):
-        self.status['updated_at'] = time.time()
-        self.status[counter] += 1
-        self._save_status()
-
-    @property
-    def _key(self):
-        return 'worker:%s' % self.worker_id
-
-    def _save_status(self):
-        self.manager.redis_connection.hmset(self._key, self.status)
-
-    def _fork_and_process(self, job_id):
-        self.child_pid = os.fork()
-        if self.child_pid == 0:
-            self.set_title("processing %s" % job_id)
-            self._process(job_id)
-        else:
-            logging.info("[%s] Waiting for pid: %d", self.name, self.child_pid)
-            _, status = os.waitpid(self.child_pid, 0)
-            self._update_status('done_jobs_count')
-            if status > 0:
-                job = Job.load(self.manager.redis_connection, job_id)
-                if not job.is_finished():
-                    self._update_status('cancelled_jobs_count')
-                    logging.info("[%s] process interrupted and job %s hasn't finished; registering interruption in job",
-                                 self.name, job_id)
-                    job.done(None, "Interrupted/Cancelled while running.")
-
-            logging.info("[%s] Finished Processing %s (pid: %d status: %d)",
-                         self.name, job_id, self.child_pid, status)
-
-    def _process(self, job_id):
-        redis_connection = redis.StrictRedis(**self.redis_connection_params)
-        job = Job.load(redis_connection, job_id)
-        if job.is_finished():
-            logging.warning("[%s][%s] tried to process finished job.", self.name, job)
-            return
-
-        pid = os.getpid()
-        job.processing(pid)
-
-        logging.info("[%s][%s] running query...", self.name, job.id)
-        start_time = time.time()
-        self.set_title("running query %s" % job_id)
-
-        if getattr(self.query_runner, 'annotate_query', True):
-            annotated_query = "/* Pid: %s, Job Id: %s, Query hash: %s, Priority: %s */ %s" % \
-                              (pid, job.id, job.query_hash, job.priority, job.query)
-        else:
-            annotated_query = job.query
-
-        # TODO: here's the part that needs to be forked, not all of the worker process...
-        with self.statsd_client.timer('worker_{}.query_runner.run_time'.format(self.worker_id)):
-            data, error = self.query_runner(annotated_query)
-
-        run_time = time.time() - start_time
-        logging.info("[%s][%s] query finished... data length=%s, error=%s",
-                     self.name, job.id, data and len(data), error)
-
-        # TODO: it is possible that storing the data will fail, and we will need to retry
-        # while we already marked the job as done
-        query_result_id = None
-        if not error:
-            self.set_title("storing results %s" % job_id)
-            query_result_id = self.manager.store_query_result(job.query, data, run_time,
-                                                              datetime.datetime.utcnow())
-
-        self.set_title("marking job as done %s" % job_id)
-        job.done(query_result_id, error)

redash/events.py

@@ -0,0 +1,23 @@
+import logging
+import json
+
+logger = logging.getLogger("redash.events")
+logger.propagate = False
+
+
+def setup_logging(log_path, console_output=False):
+    if log_path:
+        fh = logging.FileHandler(log_path)
+        formatter = logging.Formatter('%(message)s')
+        fh.setFormatter(formatter)
+        logger.addHandler(fh)
+
+    if console_output:
+        handler = logging.StreamHandler()
+        formatter = logging.Formatter('[%(name)s] %(message)s')
+        handler.setFormatter(formatter)
+        logger.addHandler(handler)
+
+
+def record_event(event):
+    logger.info(json.dumps(event))

redash/import_export.py

@@ -1,16 +1,19 @@
+import contextlib
 import json
 from redash import models
 from flask.ext.script import Manager
 
 
 class Importer(object):
-    def __init__(self, object_mapping=None):
+    def __init__(self, object_mapping=None, data_source=None):
         if object_mapping is None:
             object_mapping = {}
         self.object_mapping = object_mapping
+        self.data_source = data_source
 
     def import_query_result(self, query_result):
         query_result = self._get_or_create(models.QueryResult, query_result['id'],
+                                           data_source=self.data_source,
                                            data=json.dumps(query_result['data']),
                                            query_hash=query_result['query_hash'],
                                            retrieved_at=query_result['retrieved_at'],
@@ -29,7 +32,8 @@ class Importer(object):
                                         query=query['query'],
                                         query_hash=query['query_hash'],
                                         description=query['description'],
-                                        latest_query_data=query_result)
+                                        latest_query_data=query_result,
+                                        data_source=self.data_source)
 
         return new_query
 
@@ -108,18 +112,47 @@ class Importer(object):
 import_manager = Manager(help="import utilities")
 export_manager = Manager(help="export utilities")
 
-@import_manager.command
-def dashboard(mapping_filename, dashboard_filename, user_id):
+
+@contextlib.contextmanager
+def importer_with_mapping_file(mapping_filename):
     with open(mapping_filename) as f:
         mapping = json.loads(f.read())
 
-    user = models.User.get_by_id(user_id)
-    with open(dashboard_filename) as f:
-        dashboard = json.loads(f.read())
-
-        importer = Importer(object_mapping=mapping)
-        importer.import_dashboard(user, dashboard)
+    importer = Importer(object_mapping=mapping, data_source=get_data_source())
+    yield importer
 
     with open(mapping_filename, 'w') as f:
         f.write(json.dumps(importer.object_mapping, indent=2))
 
+
+def get_data_source():
+    try:
+        data_source = models.DataSource.get(models.DataSource.name=="Import")
+    except models.DataSource.DoesNotExist:
+        data_source = models.DataSource.create(name="Import", type="import", options='{}')
+
+    return data_source
+
+@import_manager.command
+def query(mapping_filename, query_filename, user_id):
+    user = models.User.get_by_id(user_id)
+    with open(query_filename) as f:
+        query = json.loads(f.read())
+
+    with importer_with_mapping_file(mapping_filename) as importer:
+        imported_query = importer.import_query(user, query)
+
+        print "New query id: {}".format(imported_query.id)
+
+
+@import_manager.command
+def dashboard(mapping_filename, dashboard_filename, user_id):
+    user = models.User.get_by_id(user_id)
+    with open(dashboard_filename) as f:
+        dashboard = json.loads(f.read())
+
+    with importer_with_mapping_file(mapping_filename) as importer:
+        importer.import_dashboard(user, dashboard)
+
+
+

redash/models.py

@@ -1,26 +1,114 @@
 import json
 import hashlib
+import logging
+import os
+import threading
 import time
 import datetime
-from flask.ext.peewee.utils import slugify
-from flask.ext.login import UserMixin
-from passlib.apps import custom_app_context as pwd_context
+import itertools
+
 import peewee
-from redash import db, utils
+from passlib.apps import custom_app_context as pwd_context
+from playhouse.postgres_ext import ArrayField
+from flask.ext.login import UserMixin, AnonymousUserMixin
+
+from redash import utils, settings
 
 
-class BaseModel(db.Model):
+class Database(object):
+    def __init__(self):
+        self.database_config = dict(settings.DATABASE_CONFIG)
+        self.database_name = self.database_config.pop('name')
+        self.database = peewee.PostgresqlDatabase(self.database_name, **self.database_config)
+        self.app = None
+        self.pid = os.getpid()
+
+    def init_app(self, app):
+        self.app = app
+        self.register_handlers()
+
+    def connect_db(self):
+        self._check_pid()
+        self.database.connect()
+
+    def close_db(self, exc):
+        self._check_pid()
+        if not self.database.is_closed():
+            self.database.close()
+
+    def _check_pid(self):
+        current_pid = os.getpid()
+        if self.pid != current_pid:
+            logging.info("New pid detected (%d!=%d); resetting database lock.", self.pid, current_pid)
+            self.pid = os.getpid()
+            self.database._conn_lock = threading.Lock()
+
+    def register_handlers(self):
+        self.app.before_request(self.connect_db)
+        self.app.teardown_request(self.close_db)
+
+
+db = Database()
+
+
+class BaseModel(peewee.Model):
+    class Meta:
+        database = db.database
+
     @classmethod
     def get_by_id(cls, model_id):
         return cls.get(cls.id == model_id)
 
 
+class AnonymousUser(AnonymousUserMixin):
+    @property
+    def permissions(self):
+        return []
+
+
+class ApiUser(UserMixin):
+    def __init__(self, api_key):
+        self.id = api_key
+
+    @property
+    def permissions(self):
+        return ['view_query']
+
+
+class Group(BaseModel):
+    DEFAULT_PERMISSIONS = ['create_dashboard', 'create_query', 'edit_dashboard', 'edit_query',
+                           'view_query', 'view_source', 'execute_query']
+
+    id = peewee.PrimaryKeyField()
+    name = peewee.CharField(max_length=100)
+    permissions = ArrayField(peewee.CharField, default=DEFAULT_PERMISSIONS)
+    tables = ArrayField(peewee.CharField)
+    created_at = peewee.DateTimeField(default=datetime.datetime.now)
+
+    class Meta:
+        db_table = 'groups'
+
+    def to_dict(self):
+        return {
+            'id': self.id,
+            'name': self.name,
+            'permissions': self.permissions,
+            'tables': self.tables,
+            'created_at': self.created_at
+        }
+
+    def __unicode__(self):
+        return unicode(self.id)
+
+
 class User(BaseModel, UserMixin):
+    DEFAULT_GROUPS = ['default']
+
     id = peewee.PrimaryKeyField()
     name = peewee.CharField(max_length=320)
     email = peewee.CharField(max_length=320, index=True, unique=True)
     password_hash = peewee.CharField(max_length=128, null=True)
-    is_admin = peewee.BooleanField(default=False)
+    groups = ArrayField(peewee.CharField, default=DEFAULT_GROUPS)
 
     class Meta:
         db_table = 'users'
@@ -29,10 +117,32 @@ class User(BaseModel, UserMixin):
         return {
             'id': self.id,
             'name': self.name,
-            'email': self.email,
-            'is_admin': self.is_admin
+            'email': self.email
         }
 
+    def __init__(self, *args, **kwargs):
+        super(User, self).__init__(*args, **kwargs)
+        self._allowed_tables = None
+
+    @property
+    def permissions(self):
+        # TODO: this should be cached.
+        return list(itertools.chain(*[g.permissions for g in
+                                      Group.select().where(Group.name << self.groups)]))
+
+    @property
+    def allowed_tables(self):
+        # TODO: cache this as weel
+        if self._allowed_tables is None:
+            self._allowed_tables = set([t.lower() for t in itertools.chain(*[g.tables for g in
+                                        Group.select().where(Group.name << self.groups)])])
+
+        return self._allowed_tables
+
+    @classmethod
+    def get_by_email(cls, email):
+        return cls.get(cls.email == email)
+
     def __unicode__(self):
         return '%r, %r' % (self.name, self.email)
 
@@ -45,10 +155,10 @@ class User(BaseModel, UserMixin):
 
 class ActivityLog(BaseModel):
     QUERY_EXECUTION = 1
-    
+
     id = peewee.PrimaryKeyField()
     user = peewee.ForeignKeyField(User)
-    type = peewee.IntegerField() # 1 for query execution
+    type = peewee.IntegerField()
     activity = peewee.TextField()
     created_at = peewee.DateTimeField(default=datetime.datetime.now)
 
@@ -68,8 +178,33 @@ class ActivityLog(BaseModel):
         return unicode(self.id)
 
 
+class DataSource(BaseModel):
+    id = peewee.PrimaryKeyField()
+    name = peewee.CharField()
+    type = peewee.CharField()
+    options = peewee.TextField()
+    queue_name = peewee.CharField(default="queries")
+    scheduled_queue_name = peewee.CharField(default="queries")
+    created_at = peewee.DateTimeField(default=datetime.datetime.now)
+
+    class Meta:
+        db_table = 'data_sources'
+
+    def to_dict(self):
+        return {
+            'id': self.id,
+            'name': self.name,
+            'type': self.type
+        }
+
+    @classmethod
+    def all(cls):
+        return cls.select().order_by(cls.id.asc())
+
+
 class QueryResult(BaseModel):
     id = peewee.PrimaryKeyField()
+    data_source = peewee.ForeignKeyField(DataSource)
     query_hash = peewee.CharField(max_length=32, index=True)
     query = peewee.TextField()
     data = peewee.TextField()
@@ -85,16 +220,51 @@ class QueryResult(BaseModel):
             'query_hash': self.query_hash,
             'query': self.query,
             'data': json.loads(self.data),
+            'data_source_id': self._data.get('data_source', None),
             'runtime': self.runtime,
             'retrieved_at': self.retrieved_at
         }
 
+    @classmethod
+    def get_latest(cls, data_source, query, ttl=0):
+        query_hash = utils.gen_query_hash(query)
+
+        if ttl == -1:
+            query = cls.select().where(cls.query_hash == query_hash,
+                                       cls.data_source == data_source).order_by(cls.retrieved_at.desc())
+        else:
+            query = cls.select().where(cls.query_hash == query_hash, cls.data_source == data_source,
+                                       peewee.SQL("retrieved_at + interval '%s second' >= now() at time zone 'utc'",
+                                                  ttl)).order_by(cls.retrieved_at.desc())
+
+        return query.first()
+
+    @classmethod
+    def store_result(cls, data_source_id, query_hash, query, data, run_time, retrieved_at):
+        query_result = cls.create(query_hash=query_hash,
+                                  query=query,
+                                  runtime=run_time,
+                                  data_source=data_source_id,
+                                  retrieved_at=retrieved_at,
+                                  data=data)
+
+        logging.info("Inserted query (%s) data; id=%s", query_hash, query_result.id)
+
+        updated_count = Query.update(latest_query_data=query_result).\
+            where(Query.query_hash==query_hash, Query.data_source==data_source_id).\
+            execute()
+
+        logging.info("Updated %s queries with result (%s).", updated_count, query_hash)
+
+        return query_result
+
     def __unicode__(self):
         return u"%d | %s | %s" % (self.id, self.query_hash, self.retrieved_at)
 
 
 class Query(BaseModel):
     id = peewee.PrimaryKeyField()
+    data_source = peewee.ForeignKeyField(DataSource)
     latest_query_data = peewee.ForeignKeyField(QueryResult, null=True)
     name = peewee.CharField(max_length=255)
     description = peewee.CharField(max_length=4096, null=True)
@@ -115,7 +285,7 @@ class Query(BaseModel):
                                             type="TABLE", options="{}")
         table_visualization.save()
 
-    def to_dict(self, with_result=True, with_stats=False, with_visualizations=False, with_user=True):
+    def to_dict(self, with_stats=False, with_visualizations=False, with_user=True):
         d = {
             'id': self.id,
             'latest_query_data_id': self._data.get('latest_query_data', None),
@@ -126,6 +296,7 @@ class Query(BaseModel):
             'ttl': self.ttl,
             'api_key': self.api_key,
             'created_at': self.created_at,
+            'data_source_id': self._data.get('data_source', None)
         }
 
         if with_user:
@@ -144,9 +315,6 @@ class Query(BaseModel):
             d['visualizations'] = [vis.to_dict(with_query=False)
                                    for vis in self.visualizations]
 
-        if with_result and self.latest_query_data:
-            d['latest_query_data'] = self.latest_query_data.to_dict()
-
         return d
 
     @classmethod
@@ -163,6 +331,23 @@ class Query(BaseModel):
 
         return q
 
+    @classmethod
+    def outdated_queries(cls):
+        # TODO: this will only find scheduled queries that were executed before. I think this is
+        # a reasonable assumption, but worth revisiting.
+        outdated_queries_ids = cls.select(
+            peewee.Func('first_value', cls.id).over(partition_by=[cls.query_hash, cls.data_source])) \
+            .join(QueryResult) \
+            .where(cls.ttl > 0,
+                   (QueryResult.retrieved_at +
+                    (cls.ttl * peewee.SQL("interval '1 second'"))) <
+                   peewee.SQL("(now() at time zone 'utc')"))
+
+        queries = cls.select(cls, DataSource).join(DataSource) \
+            .where(cls.id << outdated_queries_ids)
+
+        return queries
+
     @classmethod
     def update_instance(cls, query_id, **kwargs):
         if 'query' in kwargs:
@@ -192,6 +377,7 @@ class Dashboard(BaseModel):
     user_email = peewee.CharField(max_length=360, null=True)
     user = peewee.ForeignKeyField(User)
     layout = peewee.TextField()
+    dashboard_filters_enabled = peewee.BooleanField(default=False)
     is_archived = peewee.BooleanField(default=False, index=True)
     created_at = peewee.DateTimeField(default=datetime.datetime.now)
 
@@ -202,15 +388,29 @@ class Dashboard(BaseModel):
         layout = json.loads(self.layout)
 
         if with_widgets:
-            widgets = Widget.select(Widget, Visualization, Query, QueryResult, User)\
+            widgets = Widget.select(Widget, Visualization, Query, User)\
                 .where(Widget.dashboard == self.id)\
-                .join(Visualization)\
-                .join(Query)\
-                .join(User)\
-                .switch(Query)\
-                .join(QueryResult)
+                .join(Visualization, join_type=peewee.JOIN_LEFT_OUTER)\
+                .join(Query, join_type=peewee.JOIN_LEFT_OUTER)\
+                .join(User, join_type=peewee.JOIN_LEFT_OUTER)
             widgets = {w.id: w.to_dict() for w in widgets}
-            widgets_layout = map(lambda row: map(lambda widget_id: widgets.get(widget_id, None), row), layout)
+
+            # The following is a workaround for cases when the widget object gets deleted without the dashboard layout
+            # updated. This happens for users with old databases that didn't have a foreign key relationship between
+            # visualizations and widgets.
+            # It's temporary until better solution is implemented (we probably should move the position information
+            # to the widget).
+            widgets_layout = []
+            for row in layout:
+                new_row = []
+                for widget_id in row:
+                    widget = widgets.get(widget_id, None)
+                    if widget:
+                        new_row.append(widget)
+
+                widgets_layout.append(new_row)
+
+            # widgets_layout = map(lambda row: map(lambda widget_id: widgets.get(widget_id, None), row), layout)
         else:
             widgets_layout = None
 
@@ -220,6 +420,7 @@ class Dashboard(BaseModel):
             'name': self.name,
             'user_id': self._data['user'],
             'layout': layout,
+            'dashboard_filters_enabled': self.dashboard_filters_enabled,
             'widgets': widgets_layout
         }
 
@@ -229,11 +430,11 @@ class Dashboard(BaseModel):
 
     def save(self, *args, **kwargs):
         if not self.slug:
-            self.slug = slugify(self.name)
+            self.slug = utils.slugify(self.name)
 
             tries = 1
             while self.select().where(Dashboard.slug == self.slug).first() is not None:
-                self.slug = slugify(self.name) + "_{0}".format(tries)
+                self.slug = utils.slugify(self.name) + "_{0}".format(tries)
                 tries += 1
 
         super(Dashboard, self).save(*args, **kwargs)
@@ -273,8 +474,8 @@ class Visualization(BaseModel):
 
 class Widget(BaseModel):
     id = peewee.PrimaryKeyField()
-    visualization = peewee.ForeignKeyField(Visualization, related_name='widgets')
-
+    visualization = peewee.ForeignKeyField(Visualization, related_name='widgets', null=True)
+    text = peewee.TextField(null=True)
     width = peewee.IntegerField()
     options = peewee.TextField()
     dashboard = peewee.ForeignKeyField(Dashboard, related_name='widgets', index=True)
@@ -288,18 +489,44 @@ class Widget(BaseModel):
         db_table = 'widgets'
 
     def to_dict(self):
-        return {
+        d = {
             'id': self.id,
             'width': self.width,
             'options': json.loads(self.options),
-            'visualization': self.visualization.to_dict(),
-            'dashboard_id': self._data['dashboard']
+            'dashboard_id': self._data['dashboard'],
+            'text': self.text
         }
 
+        if self.visualization and self.visualization.id:
+            d['visualization'] = self.visualization.to_dict()
+
+        return d
+
     def __unicode__(self):
         return u"%s" % self.id
 
-all_models = (User, QueryResult, Query, Dashboard, Visualization, Widget, ActivityLog)
+
+class Event(BaseModel):
+    user = peewee.ForeignKeyField(User, related_name="events")
+    action = peewee.CharField()
+    object_type = peewee.CharField()
+    object_id = peewee.CharField(null=True)
+    additional_properties = peewee.TextField(null=True)
+    created_at = peewee.DateTimeField(default=datetime.datetime.now)
+
+    class Meta:
+        db_table = 'events'
+
+    def __unicode__(self):
+        return u"%s,%s,%s,%s" % (self._data['user'], self.action, self.object_type, self.object_id)
+
+
+all_models = (DataSource, User, QueryResult, Query, Dashboard, Visualization, Widget, ActivityLog, Group, Event)
+
+
+def init_db():
+    Group.insert(name='admin', permissions=['admin'], tables=['*']).execute()
+    Group.insert(name='default', permissions=Group.DEFAULT_PERMISSIONS, tables=['*']).execute()
 
 
 def create_db(create_tables, drop_tables):
@@ -311,7 +538,7 @@ def create_db(create_tables, drop_tables):
             db.database.execute_sql('DROP TABLE %s CASCADE' % model._meta.db_table)
             #model.drop_table()
 
-        if create_tables:
+        if create_tables and not model.table_exists():
             model.create_table()
 
-    db.close_db(None)
+    db.close_db(None)

redash/permissions.py

@@ -0,0 +1,27 @@
+import functools
+from flask.ext.login import current_user
+from flask.ext.restful import abort
+
+
+class require_permissions(object):
+    def __init__(self, permissions):
+        self.permissions = permissions
+
+    def __call__(self, fn):
+        @functools.wraps(fn)
+        def decorated(*args, **kwargs):
+            has_permissions = reduce(lambda a, b: a and b,
+                                      map(lambda permission: permission in current_user.permissions,
+                                          self.permissions),
+                                      True)
+
+            if has_permissions:
+                return fn(*args, **kwargs)
+            else:
+                abort(403)
+
+        return decorated
+
+
+def require_permission(permission):
+    return require_permissions((permission,))

redash/settings.py

@@ -5,9 +5,7 @@ import urlparse
 
 def parse_db_url(url):
     url_parts = urlparse.urlparse(url)
-    connection = {
-        'engine': 'peewee.PostgresqlDatabase',
-    }
+    connection = {'threadlocals': True}
 
     if url_parts.hostname and not url_parts.path:
         connection['name'] = url_parts.hostname
@@ -38,35 +36,41 @@ def parse_boolean(str):
     return json.loads(str.lower())
 
 
-REDIS_URL = os.environ.get('REDASH_REDIS_URL', "redis://localhost:6379")
+NAME = os.environ.get('REDASH_NAME', 're:dash')
+
+REDIS_URL = os.environ.get('REDASH_REDIS_URL', "redis://localhost:6379/0")
 
 STATSD_HOST = os.environ.get('REDASH_STATSD_HOST', "127.0.0.1")
 STATSD_PORT = int(os.environ.get('REDASH_STATSD_PORT', "8125"))
 STATSD_PREFIX = os.environ.get('REDASH_STATSD_PREFIX', "redash")
 
-# "pg", "graphite" or "mysql"
+# The following is kept for backward compatability, and shouldn't be used any more.
 CONNECTION_ADAPTER = os.environ.get("REDASH_CONNECTION_ADAPTER", "pg")
-# Connection string for the database that is used to run queries against. Examples:
-# -- mysql:    CONNECTION_STRING = "Server=;User=;Pwd=;Database="
-# -- pg:       CONNECTION_STRING = "user= password= host= port=5439 dbname="
-# -- graphite: CONNECTION_STRING = {"url": "https://graphite.yourcompany.com", "auth": ["user", "password"], "verify": true}
-# -- bigquery: CONNECTION_STRING = {"serviceAccount" : "43242343247-fjdfakljr3r2@developer.gserviceaccount.com", "privateKey" : "/somewhere/23fjkfjdsfj21312-privatekey.p12", "projectId" : "myproject-123" }
-#    to obtain bigquery credentials follow the guidelines at https://developers.google.com/bigquery/authorization#service-accounts
 CONNECTION_STRING = os.environ.get("REDASH_CONNECTION_STRING", "user= password= host= port=5439 dbname=")
 
 # Connection settings for re:dash's own database (where we store the queries, results, etc)
 DATABASE_CONFIG = parse_db_url(os.environ.get("REDASH_DATABASE_URL", "postgresql://postgres"))
 
+# Celery related settings
+CELERY_BROKER = os.environ.get("REDASH_CELERY_BROKER", REDIS_URL)
+CELERY_BACKEND = os.environ.get("REDASH_CELERY_BACKEND", REDIS_URL)
+CELERY_FLOWER_URL = os.environ.get("REDASH_CELERY_FLOWER_URL", "/flower")
+
 # Google Apps domain to allow access from; any user with email in this Google Apps will be allowed
 # access
 GOOGLE_APPS_DOMAIN = os.environ.get("REDASH_GOOGLE_APPS_DOMAIN", "")
 GOOGLE_OPENID_ENABLED = parse_boolean(os.environ.get("REDASH_GOOGLE_OPENID_ENABLED", "true"))
 PASSWORD_LOGIN_ENABLED = parse_boolean(os.environ.get("REDASH_PASSWORD_LOGIN_ENABLED", "false"))
-# Email addresses of admin users (comma separated)
-ADMINS = array_from_string(os.environ.get("REDASH_ADMINS", ''))
 ALLOWED_EXTERNAL_USERS = array_from_string(os.environ.get("REDASH_ALLOWED_EXTERNAL_USERS", ''))
 STATIC_ASSETS_PATH = fix_assets_path(os.environ.get("REDASH_STATIC_ASSETS_PATH", "../rd_ui/app/"))
 WORKERS_COUNT = int(os.environ.get("REDASH_WORKERS_COUNT", "2"))
+JOB_EXPIRY_TIME = int(os.environ.get("REDASH_JOB_EXPIRY_TIME", 3600*6))
 COOKIE_SECRET = os.environ.get("REDASH_COOKIE_SECRET", "c292a0a3aa32397cdb050e233733900f")
 LOG_LEVEL = os.environ.get("REDASH_LOG_LEVEL", "INFO")
-ANALYTICS = os.environ.get("REDASH_ANALYTICS", "")
+EVENTS_LOG_PATH = os.environ.get("REDASH_EVENTS_LOG_PATH", "")
+EVENTS_CONSOLE_OUTPUT = parse_boolean(os.environ.get("REDASH_EVENTS_CONSOLE_OUTPUT", "false"))
+CLIENT_SIDE_METRICS = parse_boolean(os.environ.get("REDASH_CLIENT_SIDE_METRICS", "false"))
+ANALYTICS = os.environ.get("REDASH_ANALYTICS", "")
+
+# Features:
+FEATURE_TABLES_PERMISSIONS = parse_boolean(os.environ.get("REDASH_FEATURE_TABLES_PERMISSIONS", "false"))

redash/tasks.py

@@ -0,0 +1,248 @@
+import time
+import datetime
+import logging
+import redis
+from celery import Task
+from celery.result import AsyncResult
+from celery.utils.log import get_task_logger
+from redash import redis_connection, models, statsd_client, settings
+from redash.utils import gen_query_hash
+from redash.worker import celery
+from redash.data.query_runner import get_query_runner
+
+logger = get_task_logger(__name__)
+
+
+class BaseTask(Task):
+    abstract = True
+
+    def after_return(self, *args, **kwargs):
+        models.db.close_db(None)
+
+    def __call__(self, *args, **kwargs):
+        models.db.connect_db()
+        return super(BaseTask, self).__call__(*args, **kwargs)
+
+
+class QueryTask(object):
+    MAX_RETRIES = 5
+
+    # TODO: this is mapping to the old Job class statuses. Need to update the client side and remove this
+    STATUSES = {
+        'PENDING': 1,
+        'STARTED': 2,
+        'SUCCESS': 3,
+        'FAILURE': 4,
+        'REVOKED': 4
+    }
+
+    def __init__(self, job_id=None, async_result=None):
+        if async_result:
+            self._async_result = async_result
+        else:
+            self._async_result = AsyncResult(job_id, app=celery)
+
+    @property
+    def id(self):
+        return self._async_result.id
+
+    @classmethod
+    def add_task(cls, query, data_source, scheduled=False):
+        query_hash = gen_query_hash(query)
+        logging.info("[Manager][%s] Inserting job", query_hash)
+        try_count = 0
+        job = None
+
+        while try_count < cls.MAX_RETRIES:
+            try_count += 1
+
+            pipe = redis_connection.pipeline()
+            try:
+                pipe.watch(cls._job_lock_id(query_hash, data_source.id))
+                job_id = pipe.get(cls._job_lock_id(query_hash, data_source.id))
+                if job_id:
+                    logging.info("[Manager][%s] Found existing job: %s", query_hash, job_id)
+
+                    job = cls(job_id=job_id)
+                    if job.ready():
+                        logging.info("[%s] job found is ready (%s), removing lock", query_hash, job.celery_status)
+                        redis_connection.delete(QueryTask._job_lock_id(query_hash, data_source.id))
+                        job = None
+
+                if not job:
+                    pipe.multi()
+
+                    if scheduled:
+                        queue_name = data_source.scheduled_queue_name
+                    else:
+                        queue_name = data_source.queue_name
+
+                    result = execute_query.apply_async(args=(query, data_source.id), queue=queue_name)
+                    job = cls(async_result=result)
+                    logging.info("[Manager][%s] Created new job: %s", query_hash, job.id)
+                    pipe.set(cls._job_lock_id(query_hash, data_source.id), job.id, settings.JOB_EXPIRY_TIME)
+                    pipe.execute()
+                break
+
+            except redis.WatchError:
+                continue
+
+        if not job:
+            logging.error("[Manager][%s] Failed adding job for query.", query_hash)
+
+        return job
+
+    def to_dict(self):
+        if self._async_result.status == 'STARTED':
+            updated_at = self._async_result.result.get('start_time', 0)
+        else:
+            updated_at = 0
+
+        if self._async_result.failed() and isinstance(self._async_result.result, Exception):
+            error = self._async_result.result.message
+        elif self._async_result.status == 'REVOKED':
+            error = 'Query execution cancelled.'
+        else:
+            error = ''
+
+        if self._async_result.successful():
+            query_result_id = self._async_result.result
+        else:
+            query_result_id = None
+
+        return {
+            'id': self._async_result.id,
+            'updated_at': updated_at,
+            'status': self.STATUSES[self._async_result.status],
+            'error': error,
+            'query_result_id': query_result_id,
+        }
+
+    @property
+    def is_cancelled(self):
+        return self._async_result.status == 'REVOKED'
+
+    @property
+    def celery_status(self):
+        return self._async_result.status
+
+    def ready(self):
+        return self._async_result.ready()
+
+    def cancel(self):
+        return self._async_result.revoke(terminate=True)
+
+    @staticmethod
+    def _job_lock_id(query_hash, data_source_id):
+        return "query_hash_job:%s:%s" % (data_source_id, query_hash)
+
+
+@celery.task(base=BaseTask)
+def refresh_queries():
+    # self.status['last_refresh_at'] = time.time()
+    # self._save_status()
+
+    logger.info("Refreshing queries...")
+
+    outdated_queries_count = 0
+    for query in models.Query.outdated_queries():
+        # TODO: this should go into lower priority
+        QueryTask.add_task(query.query, query.data_source, scheduled=True)
+        outdated_queries_count += 1
+
+    statsd_client.gauge('manager.outdated_queries', outdated_queries_count)
+    # TODO: decide if we still need this
+    # statsd_client.gauge('manager.queue_size', self.redis_connection.zcard('jobs'))
+
+    logger.info("Done refreshing queries. Found %d outdated queries." % outdated_queries_count)
+
+    status = redis_connection.hgetall('redash:status')
+    now = time.time()
+
+    redis_connection.hmset('redash:status', {
+        'outdated_queries_count': outdated_queries_count,
+        'last_refresh_at': now
+    })
+
+    statsd_client.gauge('manager.seconds_since_refresh', now - float(status.get('last_refresh_at', now)))
+
+
+@celery.task(base=BaseTask)
+def cleanup_tasks():
+    # in case of cold restart of the workers, there might be jobs that still have their "lock" object, but aren't really
+    # going to run. this job removes them.
+
+    lock_keys = redis_connection.keys("query_hash_job:*") # TODO: use set instead of keys command
+    query_tasks = [QueryTask(job_id=j) for j in redis_connection.mget(lock_keys)]
+
+    logger.info("Found %d locks", len(query_tasks))
+
+    inspect = celery.control.inspect()
+    active_tasks = inspect.active()
+    if active_tasks is None:
+        active_tasks = []
+    else:
+        active_tasks = active_tasks.values()
+
+    all_tasks = set()
+    for task_list in active_tasks:
+        for task in task_list:
+            all_tasks.add(task['id'])
+
+    logger.info("Active jobs count: %d", len(all_tasks))
+
+    for i, t in enumerate(query_tasks):
+        if t.ready():
+            # if locked task is ready already (failed, finished, revoked), we don't need the lock anymore
+            logger.warning("%s is ready (%s), removing lock.", lock_keys[i], t.celery_status)
+            redis_connection.delete(lock_keys[i])
+
+        if t.celery_status == 'STARTED' and t.id not in all_tasks:
+            logger.warning("Couldn't find active job for: %s, removing lock.", lock_keys[i])
+            redis_connection.delete(lock_keys[i])
+
+
+@celery.task(bind=True, base=BaseTask, track_started=True)
+def execute_query(self, query, data_source_id):
+    # TODO: maybe this should be a class?
+    start_time = time.time()
+
+    logger.info("Loading data source (%d)...", data_source_id)
+
+    # TODO: we should probably cache data sources in Redis
+    data_source = models.DataSource.get_by_id(data_source_id)
+
+    self.update_state(state='STARTED', meta={'start_time': start_time, 'custom_message': ''})
+
+    logger.info("Executing query:\n%s", query)
+
+    query_hash = gen_query_hash(query)
+    query_runner = get_query_runner(data_source.type, data_source.options)
+
+    if getattr(query_runner, 'annotate_query', True):
+        # TODO: anotate with queu ename
+        annotated_query = "/* Task Id: %s, Query hash: %s */ %s" % \
+                          (self.request.id, query_hash, query)
+    else:
+        annotated_query = query
+
+    with statsd_client.timer('query_runner.{}.{}.run_time'.format(data_source.type, data_source.name)):
+        data, error = query_runner(annotated_query)
+
+    run_time = time.time() - start_time
+    logger.info("Query finished... data length=%s, error=%s", data and len(data), error)
+
+    self.update_state(state='STARTED', meta={'start_time': start_time, 'error': error, 'custom_message': ''})
+
+    # Delete query_hash
+    redis_connection.delete(QueryTask._job_lock_id(query_hash, data_source.id))
+
+    # TODO: it is possible that storing the data will fail, and we will need to retry
+    # while we already marked the job as done
+    if not error:
+        query_result = models.QueryResult.store_result(data_source.id, query_hash, query, data, run_time, datetime.datetime.utcnow())
+    else:
+        raise Exception(error)
+
+    return query_result.id
+

redash/utils.py

@@ -6,10 +6,66 @@ import datetime
 import json
 import re
 import hashlib
+import sqlparse
 
 COMMENTS_REGEX = re.compile("/\*.*?\*/")
 
 
+class SQLMetaData(object):
+    TABLE_SELECTION_KEYWORDS = ('FROM', 'JOIN', 'LEFT JOIN', 'FULL JOIN', 'RIGHT JOIN', 'CROSS JOIN', 'INNER JOIN',
+                                'OUTER JOIN', 'LEFT OUTER JOIN', 'RIGHT OUTER JOIN', 'FULL OUTER JOIN')
+
+    def __init__(self, sql):
+        self.sql = sql
+        self.parsed_sql = sqlparse.parse(self.sql)
+
+        self.has_ddl_statements = self._find_ddl_statements()
+        self.has_non_select_dml_statements = self._find_dml_statements()
+        self.used_tables = self._find_tables()
+
+    def _find_ddl_statements(self):
+        for statement in self.parsed_sql:
+            if len([x for x in statement.flatten() if x.ttype == sqlparse.tokens.DDL]):
+                return True
+
+        return False
+
+    def _find_tables(self):
+        tables = set()
+        for statement in self.parsed_sql:
+            tables.update(self.extract_table_names(statement.tokens))
+
+        return tables
+
+    def extract_table_names(self, tokens):
+        tables = set()
+        tokens = [t for t in tokens if t.ttype not in (sqlparse.tokens.Whitespace, sqlparse.tokens.Newline)]
+
+        for i in range(len(tokens)):
+            if tokens[i].is_group():
+                tables.update(self.extract_table_names(tokens[i].tokens))
+            else:
+                if tokens[i].ttype == sqlparse.tokens.Keyword and tokens[i].normalized in self.TABLE_SELECTION_KEYWORDS:
+                    if isinstance(tokens[i + 1], sqlparse.sql.Identifier):
+                        tables.add(tokens[i + 1].value)
+
+                    if isinstance(tokens[i + 1], sqlparse.sql.IdentifierList):
+                        tables.update(set([t.value for t in tokens[i+1].get_identifiers()]))
+        return tables
+
+    def _find_dml_statements(self):
+        for statement in self.parsed_sql:
+            for token in statement.flatten():
+                if token.ttype == sqlparse.tokens.DML and token.normalized != 'SELECT':
+                    return True
+
+        return False
+
+
+def slugify(s):
+    return re.sub('[^a-z0-9_\-]+', '-', s.lower())
+
+
 def gen_query_hash(sql):
     """Returns hash of the given query after stripping all comments, line breaks and multiple
     spaces, and lower casing all text.

redash/worker.py

@@ -0,0 +1,25 @@
+from celery import Celery
+from datetime import timedelta
+from redash import settings
+
+
+celery = Celery('redash',
+                broker=settings.CELERY_BROKER,
+                include='redash.tasks')
+
+celery.conf.update(CELERY_RESULT_BACKEND=settings.CELERY_BACKEND,
+                   CELERYBEAT_SCHEDULE={
+                       'refresh_queries': {
+                           'task': 'redash.tasks.refresh_queries',
+                           'schedule': timedelta(seconds=30)
+                       },
+                       'cleanup_tasks': {
+                           'task': 'redash.tasks.cleanup_tasks',
+                           'schedule': timedelta(minutes=5)
+                       },
+                   },
+                   CELERY_TIMEZONE='UTC')
+
+
+if __name__ == '__main__':
+    celery.start()

redash/wsgi.py

@@ -0,0 +1,32 @@
+import json
+from flask import Flask, make_response
+from flask.ext.restful import Api
+
+from redash import settings, utils
+from redash.models import db
+
+__version__ = '0.4.0'
+
+app = Flask(__name__,
+            template_folder=settings.STATIC_ASSETS_PATH,
+            static_folder=settings.STATIC_ASSETS_PATH,
+            static_path='/static')
+
+
+api = Api(app)
+
+# configure our database
+settings.DATABASE_CONFIG.update({'threadlocals': True})
+app.config['DATABASE'] = settings.DATABASE_CONFIG
+db.init_app(app)
+
+from redash.authentication import setup_authentication
+auth = setup_authentication(app)
+
+@api.representation('application/json')
+def json_representation(data, code, headers=None):
+    resp = make_response(json.dumps(data, cls=utils.JSONEncoder), code)
+    resp.headers.extend(headers or {})
+    return resp
+
+from redash import controllers

requirements.txt

@@ -5,25 +5,21 @@ Flask-Login==0.2.9
 passlib==1.6.2
 Jinja2==2.7.2
 MarkupSafe==0.18
-WTForms==1.0.5
 Werkzeug==0.9.4
 aniso8601==0.82
-atfork==0.1.2
 blinker==1.3
-flask-peewee==0.6.5
 itsdangerous==0.23
-peewee==2.2.0
-psycopg2==2.5.1
+peewee==2.2.2
+psycopg2==2.5.2
 python-dateutil==2.1
 pytz==2013.9
-qr==0.6.0
 redis==2.7.5
 requests==2.2.0
-setproctitle==1.1.8
 six==1.5.2
 sqlparse==0.1.8
 wsgiref==0.1.2
-wtf-peewee==0.2.2
 Flask-Script==0.6.6
 honcho==0.5.0
 statsd==2.1.2
+gunicorn==18.0
+celery==3.1.11

tests/__init__.py

@@ -1,24 +1,21 @@
+import logging
 from unittest import TestCase
-from redash import settings, db, app
-import redash.models
-
-# TODO: this isn't pretty...
+from redash import settings
 settings.DATABASE_CONFIG = {
     'name': 'circle_test',
-    'engine': 'peewee.PostgresqlDatabase',
     'threadlocals': True
 }
-app.config['DATABASE'] = settings.DATABASE_CONFIG
-db.load_database()
 
-for model in redash.models.all_models:
-    model._meta.database = db.database
+from redash import models
+
+logging.getLogger('peewee').setLevel(logging.INFO)
 
 
 class BaseTestCase(TestCase):
     def setUp(self):
-        redash.models.create_db(True, True)
+        models.create_db(True, True)
+        models.init_db()
 
     def tearDown(self):
-        db.close_db(None)
-        redash.models.create_db(False, True)
+        models.db.close_db(None)
+        models.create_db(False, True)

tests/factories.py

@@ -1,5 +1,6 @@
 import datetime
 import redash.models
+from redash.utils import gen_query_hash
 
 
 class ModelFactory(object):
@@ -40,7 +41,13 @@ class Sequence(object):
 
 user_factory = ModelFactory(redash.models.User,
                             name='John Doe', email=Sequence('test{}@example.com'),
-                            is_admin=False)
+                            groups=['default'])
+
+
+data_source_factory = ModelFactory(redash.models.DataSource,
+                                   name='Test',
+                                   type='pg',
+                                   options='')
 
 
 dashboard_factory = ModelFactory(redash.models.Dashboard,
@@ -52,14 +59,16 @@ query_factory = ModelFactory(redash.models.Query,
                              description='',
                              query='SELECT 1',
                              ttl=-1,
-                             user=user_factory.create)
+                             user=user_factory.create,
+                             data_source=data_source_factory.create)
 
 query_result_factory = ModelFactory(redash.models.QueryResult,
                                     data='{"columns":{}, "rows":[]}',
                                     runtime=1,
-                                    retrieved_at=datetime.datetime.now(),
-                                    query=query_factory.create,
-                                    query_hash='')
+                                    retrieved_at=datetime.datetime.utcnow,
+                                    query="SELECT 1",
+                                    query_hash=gen_query_hash('SELECT 1'),
+                                    data_source=data_source_factory.create)
 
 visualization_factory = ModelFactory(redash.models.Visualization,
                                      type='CHART',
@@ -73,4 +82,4 @@ widget_factory = ModelFactory(redash.models.Widget,
                               width=1,
                               options='{}',
                               dashboard=dashboard_factory.create,
-                              visualization=visualization_factory.create)
+                              visualization=visualization_factory.create)

tests/test_authentication.py

@@ -36,7 +36,7 @@ class TestCreateAndLoginUser(BaseTestCase):
     def test_creates_vaild_new_user(self):
         openid_user = ObjectDict({'email': 'test@example.com', 'name': 'Test User'})
 
-        with patch.multiple(settings, GOOGLE_APPS_DOMAIN='example.com', ADMINS=['admin@example.com']), \
+        with patch.multiple(settings, GOOGLE_APPS_DOMAIN='example.com'), \
              patch('redash.authentication.login_user') as login_user_mock:
 
             create_and_login_user(None, openid_user)
@@ -44,20 +44,6 @@ class TestCreateAndLoginUser(BaseTestCase):
             self.assertTrue(login_user_mock.called)
             user = models.User.get(models.User.email == openid_user.email)
 
-            self.assertFalse(user.is_admin)
-
-    def test_creates_vaild_new_user_and_sets_is_admin(self):
-        openid_user = ObjectDict({'email': 'admin@example.com', 'name': 'Test User'})
-
-        with patch.multiple(settings, GOOGLE_APPS_DOMAIN='example.com', ADMINS=['admin@example.com']), \
-             patch('redash.authentication.login_user') as login_user_mock:
-
-            create_and_login_user(None, openid_user)
-
-            self.assertTrue(login_user_mock.called)
-            user = models.User.get(models.User.email == openid_user.email)
-            self.assertTrue(user.is_admin)
-
     def test_ignores_invliad_user(self):
         user = ObjectDict({'email': 'test@whatever.com'})
 

tests/test_controllers.py

@@ -7,8 +7,9 @@ from flask.ext.login import current_user
 from mock import patch
 from tests import BaseTestCase
 from tests.factories import dashboard_factory, widget_factory, visualization_factory, query_factory, \
-    query_result_factory, user_factory
-from redash import app, models, settings
+    query_result_factory, user_factory, data_source_factory
+from redash import models, settings
+from redash.wsgi import app
 from redash.utils import json_dumps
 from redash.authentication import sign
 
@@ -75,10 +76,22 @@ class IndexTest(BaseTestCase, AuthenticationTestMixin):
         super(IndexTest, self).setUp()
 
 
-class StatusTest(BaseTestCase, AuthenticationTestMixin):
-    def setUp(self):
-        self.paths = ['/status.json']
-        super(StatusTest, self).setUp()
+class StatusTest(BaseTestCase):
+    def test_returns_data_for_admin(self):
+        admin = user_factory.create(groups=['admin', 'default'])
+        with app.test_client() as c, authenticated_user(c, user=admin):
+            rv = c.get('/status.json')
+            self.assertEqual(rv.status_code, 200)
+
+    def test_returns_403_for_non_admin(self):
+        with app.test_client() as c, authenticated_user(c):
+            rv = c.get('/status.json')
+            self.assertEqual(rv.status_code, 403)
+
+    def test_redirects_non_authenticated_user(self):
+        with app.test_client() as c:
+            rv = c.get('/status.json')
+            self.assertEqual(rv.status_code, 302)
 
 
 class DashboardAPITest(BaseTestCase, AuthenticationTestMixin):
@@ -170,6 +183,23 @@ class WidgetAPITest(BaseTestCase):
                            [rv4.json['widget']['id']]])
         self.assertEquals(rv4.json['new_row'], True)
 
+    def test_create_text_widget(self):
+        dashboard = dashboard_factory.create()
+
+        data = {
+            'visualization_id': None,
+            'text': 'Sample text.',
+            'dashboard_id': dashboard.id,
+            'options': {},
+            'width': 2
+        }
+
+        with app.test_client() as c, authenticated_user(c):
+            rv = json_request(c.post, '/api/widgets', data=data)
+
+        self.assertEquals(rv.status_code, 200)
+        self.assertEquals(rv.json['widget']['text'], 'Sample text.')
+
     def test_delete_widget(self):
         widget = widget_factory.create()
 
@@ -199,10 +229,12 @@ class QueryAPITest(BaseTestCase, AuthenticationTestMixin):
 
     def test_create_query(self):
         user = user_factory.create()
+        data_source = data_source_factory.create()
         query_data = {
             'name': 'Testing',
             'query': 'SELECT 1',
-            'ttl': 3600
+            'ttl': 3600,
+            'data_source_id': data_source.id
         }
 
         with app.test_client() as c, authenticated_user(c, user=user):
@@ -290,14 +322,16 @@ class JobAPITest(BaseTestCase, AuthenticationTestMixin):
 class CsvQueryResultAPITest(BaseTestCase, AuthenticationTestMixin):
     def setUp(self):
         super(CsvQueryResultAPITest, self).setUp()
+
         self.paths = []
         self.query_result = query_result_factory.create()
-        self.path = '/api/queries/{0}/results/{1}.csv'.format(self.query_result.query.id, self.query_result.id)
+        self.query = query_factory.create()
+        self.path = '/api/queries/{0}/results/{1}.csv'.format(self.query.id, self.query_result.id)
 
     # TODO: factor out the HMAC authentication tests
 
     def signature(self, expires):
-        return sign(self.query_result.query.api_key, self.path, expires)
+        return sign(self.query.api_key, self.path, expires)
 
     def test_redirect_when_unauthenticated(self):
         with app.test_client() as c:
@@ -306,34 +340,34 @@ class CsvQueryResultAPITest(BaseTestCase, AuthenticationTestMixin):
 
     def test_redirect_for_wrong_signature(self):
         with app.test_client() as c:
-            rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query_result.query.id, self.query_result.id), query_string={'signature': 'whatever', 'expires': 0})
+            rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query.id, self.query_result.id), query_string={'signature': 'whatever', 'expires': 0})
             self.assertEquals(rv.status_code, 302)
 
     def test_redirect_for_correct_signature_and_wrong_expires(self):
         with app.test_client() as c:
-            rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query_result.query.id, self.query_result.id), query_string={'signature': self.signature(0), 'expires': 0})
+            rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query.id, self.query_result.id), query_string={'signature': self.signature(0), 'expires': 0})
             self.assertEquals(rv.status_code, 302)
 
     def test_redirect_for_correct_signature_and_no_expires(self):
         with app.test_client() as c:
-            rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query_result.query.id, self.query_result.id), query_string={'signature': self.signature(time.time()+3600)})
+            rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query.id, self.query_result.id), query_string={'signature': self.signature(time.time()+3600)})
             self.assertEquals(rv.status_code, 302)
 
     def test_redirect_for_correct_signature_and_expires_too_long(self):
         with app.test_client() as c:
             expires = time.time()+(10*3600)
-            rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query_result.query.id, self.query_result.id), query_string={'signature': self.signature(expires), 'expires': expires})
+            rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query.id, self.query_result.id), query_string={'signature': self.signature(expires), 'expires': expires})
             self.assertEquals(rv.status_code, 302)
 
     def test_returns_200_for_correct_signature(self):
         with app.test_client() as c:
-            expires = time.time()+3600
-            rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query_result.query.id, self.query_result.id), query_string={'signature': self.signature(expires), 'expires': expires})
+            expires = time.time()+1800
+            rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query.id, self.query_result.id), query_string={'signature': self.signature(expires), 'expires': expires})
             self.assertEquals(rv.status_code, 200)
 
     def test_returns_200_for_authenticated_user(self):
         with app.test_client() as c, authenticated_user(c):
-            rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query_result.query.id, self.query_result.id))
+            rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query.id, self.query_result.id))
             self.assertEquals(rv.status_code, 200)
 
 
@@ -360,6 +394,7 @@ class TestLogin(BaseTestCase):
             self.assertFalse(login_user_mock.called)
 
     def test_submit_correct_user_and_password(self):
+
         user = user_factory.create()
         user.hash_password('password')
         user.save()

tests/test_import.py

@@ -3,7 +3,7 @@ import os.path
 from tests import BaseTestCase
 from redash import models
 from redash import import_export
-from factories import user_factory, dashboard_factory
+from factories import user_factory, dashboard_factory, data_source_factory
 
 
 class ImportTest(BaseTestCase):
@@ -15,7 +15,7 @@ class ImportTest(BaseTestCase):
             self.user = user_factory.create()
 
     def test_imports_dashboard_correctly(self):
-        importer = import_export.Importer()
+        importer = import_export.Importer(data_source=data_source_factory.create())
         dashboard = importer.import_dashboard(self.user, self.dashboard)
 
         self.assertIsNotNone(dashboard)
@@ -31,7 +31,7 @@ class ImportTest(BaseTestCase):
         self.assertEqual(models.QueryResult.select().count(), dashboard.widgets.count()-1)
 
     def test_imports_updates_existing_models(self):
-        importer = import_export.Importer()
+        importer = import_export.Importer(data_source=data_source_factory.create())
         importer.import_dashboard(self.user, self.dashboard)
 
         self.dashboard['name'] = 'Testing #2'
@@ -47,7 +47,7 @@ class ImportTest(BaseTestCase):
             }
         }
 
-        importer = import_export.Importer(object_mapping=mapping)
+        importer = import_export.Importer(object_mapping=mapping, data_source=data_source_factory.create())
         imported_dashboard = importer.import_dashboard(self.user, self.dashboard)
 
         self.assertEqual(imported_dashboard, dashboard)

tests/test_models.py

@@ -1,6 +1,8 @@
+import datetime
 from tests import BaseTestCase
 from redash import models
-from factories import dashboard_factory, query_factory
+from factories import dashboard_factory, query_factory, data_source_factory, query_result_factory
+from redash.utils import gen_query_hash
 
 
 class DashboardTest(BaseTestCase):
@@ -25,4 +27,125 @@ class QueryTest(BaseTestCase):
 
         q = models.Query.get_by_id(q.id)
 
-        self.assertNotEquals(old_hash, q.query_hash)
+        self.assertNotEquals(old_hash, q.query_hash)
+
+
+class QueryResultTest(BaseTestCase):
+    def setUp(self):
+        super(QueryResultTest, self).setUp()
+
+    def test_get_latest_returns_none_if_not_found(self):
+        ds = data_source_factory.create()
+        found_query_result = models.QueryResult.get_latest(ds, "SELECT 1", 60)
+        self.assertIsNone(found_query_result)
+
+    def test_get_latest_returns_when_found(self):
+        qr = query_result_factory.create()
+        found_query_result = models.QueryResult.get_latest(qr.data_source, qr.query, 60)
+
+        self.assertEqual(qr, found_query_result)
+
+    def test_get_latest_works_with_data_source_id(self):
+        qr = query_result_factory.create()
+        found_query_result = models.QueryResult.get_latest(qr.data_source.id, qr.query, 60)
+
+        self.assertEqual(qr, found_query_result)
+
+    def test_get_latest_doesnt_return_query_from_different_data_source(self):
+        qr = query_result_factory.create()
+        data_source = data_source_factory.create()
+        found_query_result = models.QueryResult.get_latest(data_source, qr.query, 60)
+
+        self.assertIsNone(found_query_result)
+
+    def test_get_latest_doesnt_return_if_ttl_expired(self):
+        yesterday = datetime.datetime.now() - datetime.timedelta(days=1)
+        qr = query_result_factory.create(retrieved_at=yesterday)
+
+        found_query_result = models.QueryResult.get_latest(qr.data_source, qr.query, ttl=60)
+
+        self.assertIsNone(found_query_result)
+
+    def test_get_latest_returns_if_ttl_not_expired(self):
+        yesterday = datetime.datetime.now() - datetime.timedelta(seconds=30)
+        qr = query_result_factory.create(retrieved_at=yesterday)
+
+        found_query_result = models.QueryResult.get_latest(qr.data_source, qr.query, ttl=120)
+
+        self.assertEqual(found_query_result, qr)
+
+    def test_get_latest_returns_the_most_recent_result(self):
+        yesterday = datetime.datetime.now() - datetime.timedelta(seconds=30)
+        old_qr = query_result_factory.create(retrieved_at=yesterday)
+        qr = query_result_factory.create()
+
+        found_query_result = models.QueryResult.get_latest(qr.data_source, qr.query, 60)
+
+        self.assertEqual(found_query_result.id, qr.id)
+
+    def test_get_latest_returns_the_last_cached_result_for_negative_ttl(self):
+        yesterday = datetime.datetime.now() + datetime.timedelta(days=-100)
+        very_old = query_result_factory.create(retrieved_at=yesterday)
+
+        yesterday = datetime.datetime.now() + datetime.timedelta(days=-1)
+        qr = query_result_factory.create(retrieved_at=yesterday)
+        found_query_result = models.QueryResult.get_latest(qr.data_source, qr.query, -1)
+
+        self.assertEqual(found_query_result.id, qr.id)
+
+class TestQueryResultStoreResult(BaseTestCase):
+    def setUp(self):
+        super(TestQueryResultStoreResult, self).setUp()
+        self.data_source = data_source_factory.create()
+        self.query = "SELECT 1"
+        self.query_hash = gen_query_hash(self.query)
+        self.runtime = 123
+        self.utcnow = datetime.datetime.utcnow()
+        self.data = "data"
+
+    def test_stores_the_result(self):
+        query_result = models.QueryResult.store_result(self.data_source.id, self.query_hash, self.query,
+                                                          self.data, self.runtime, self.utcnow)
+
+        self.assertEqual(query_result.data, self.data)
+        self.assertEqual(query_result.runtime, self.runtime)
+        self.assertEqual(query_result.retrieved_at, self.utcnow)
+        self.assertEqual(query_result.query, self.query)
+        self.assertEqual(query_result.query_hash, self.query_hash)
+        self.assertEqual(query_result.data_source, self.data_source)
+
+    def test_updates_existing_queries(self):
+        query1 = query_factory.create(query=self.query, data_source=self.data_source)
+        query2 = query_factory.create(query=self.query, data_source=self.data_source)
+        query3 = query_factory.create(query=self.query, data_source=self.data_source)
+
+        query_result = models.QueryResult.store_result(self.data_source.id, self.query_hash, self.query, self.data,
+                                                       self.runtime, self.utcnow)
+
+        self.assertEqual(models.Query.get_by_id(query1.id)._data['latest_query_data'], query_result.id)
+        self.assertEqual(models.Query.get_by_id(query2.id)._data['latest_query_data'], query_result.id)
+        self.assertEqual(models.Query.get_by_id(query3.id)._data['latest_query_data'], query_result.id)
+
+    def test_doesnt_update_queries_with_different_hash(self):
+        query1 = query_factory.create(query=self.query, data_source=self.data_source)
+        query2 = query_factory.create(query=self.query, data_source=self.data_source)
+        query3 = query_factory.create(query=self.query + "123", data_source=self.data_source)
+
+        query_result = models.QueryResult.store_result(self.data_source.id, self.query_hash, self.query, self.data,
+                                                       self.runtime, self.utcnow)
+
+        self.assertEqual(models.Query.get_by_id(query1.id)._data['latest_query_data'], query_result.id)
+        self.assertEqual(models.Query.get_by_id(query2.id)._data['latest_query_data'], query_result.id)
+        self.assertNotEqual(models.Query.get_by_id(query3.id)._data['latest_query_data'], query_result.id)
+
+    def test_doesnt_update_queries_with_different_data_source(self):
+        query1 = query_factory.create(query=self.query, data_source=self.data_source)
+        query2 = query_factory.create(query=self.query, data_source=self.data_source)
+        query3 = query_factory.create(query=self.query, data_source=data_source_factory.create())
+
+        query_result = models.QueryResult.store_result(self.data_source.id, self.query_hash, self.query, self.data,
+                                                       self.runtime, self.utcnow)
+
+        self.assertEqual(models.Query.get_by_id(query1.id)._data['latest_query_data'], query_result.id)
+        self.assertEqual(models.Query.get_by_id(query2.id)._data['latest_query_data'], query_result.id)
+        self.assertNotEqual(models.Query.get_by_id(query3.id)._data['latest_query_data'], query_result.id)