Merge pull request #165 from erans/master

Make sure qr serialization will always be in JSON

.gitignore

@@ -10,3 +10,4 @@ rd_ui/dist
 Berksfile.lock
 redash/dump.rdb
 .env
+.ruby-version

bin/run

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+source .env
+"$@"

bin/test_multithreading.py

@@ -0,0 +1,63 @@
+"""
+Script to test concurrency (multithreading/multiprocess) issues with the workers. Use with caution.
+"""
+import json
+import atfork
+atfork.monkeypatch_os_fork_functions()
+import atfork.stdlib_fixer
+atfork.stdlib_fixer.fix_logging_module()
+
+import time
+from redash.data import worker
+from redash import models, data_manager, redis_connection
+
+if __name__ == '__main__':
+    models.create_db(True, False)
+
+    print "Creating data source..."
+    data_source = models.DataSource.create(name="Concurrency", type="pg", options="dbname=postgres")
+
+    print "Clear jobs/hashes:"
+    redis_connection.delete("jobs")
+    query_hashes = redis_connection.keys("query_hash_*")
+    if query_hashes:
+        redis_connection.delete(*query_hashes)
+
+    starting_query_results_count = models.QueryResult.select().count()
+    jobs_count = 5000
+    workers_count = 10
+
+    print "Creating jobs..."
+    for i in xrange(jobs_count):
+        query = "SELECT {}".format(i)
+        print "Inserting: {}".format(query)
+        data_manager.add_job(query=query, priority=worker.Job.LOW_PRIORITY,
+                             data_source=data_source)
+
+    print "Starting workers..."
+    workers = data_manager.start_workers(workers_count)
+
+    print "Waiting for jobs to be done..."
+    keep_waiting = True
+    while keep_waiting:
+        results_count = models.QueryResult.select().count() - starting_query_results_count
+        print "QueryResults: {}".format(results_count)
+        time.sleep(5)
+        if results_count == jobs_count:
+            print "Yay done..."
+            keep_waiting = False
+
+    data_manager.stop_workers()
+
+    qr_count = 0
+    for qr in models.QueryResult.select():
+        number = int(qr.query.split()[1])
+        data_number = json.loads(qr.data)['rows'][0].values()[0]
+
+        if number != data_number:
+            print "Oops? {} != {} ({})".format(number, data_number, qr.id)
+        qr_count += 1
+
+    print "Verified {} query results.".format(qr_count)
+
+    print "Done."

manage.py

@@ -10,6 +10,7 @@ atfork.stdlib_fixer.fix_logging_module()
 import logging
 import time
 from redash import settings, app, db, models, data_manager, __version__
+from redash.import_export import import_manager
 from flask.ext.script import Manager, prompt_pass
 
 manager = Manager(app)
@@ -32,12 +33,13 @@ def runworkers():
 
         logging.info("Cleaning old workers: %s", old_workers)
 
-        data_manager.start_workers(settings.WORKERS_COUNT, settings.CONNECTION_ADAPTER, settings.CONNECTION_STRING)
+        data_manager.start_workers(settings.WORKERS_COUNT)
         logging.info("Workers started.")
 
         while True:
             try:
                 data_manager.refresh_queries()
+                data_manager.report_status()
             except Exception as e:
                 logging.error("Something went wrong with refreshing queries...")
                 logging.exception(e)
@@ -50,6 +52,15 @@ def runworkers():
 def make_shell_context():
     return dict(app=app, db=db, models=models)
 
+@manager.command
+def check_settings():
+    from types import ModuleType
+
+    for name in dir(settings):
+        item = getattr(settings, name)
+        if not callable(item) and not name.startswith("__") and not isinstance(item, ModuleType):
+            print "{} = {}".format(name, item)
+
 @database_manager.command
 def create_tables():
     """Creates the database tables."""
@@ -67,16 +78,24 @@ def drop_tables():
 
 @users_manager.option('email', help="User's email")
 @users_manager.option('name', help="User's full name")
-@users_manager.option('--admin', dest='is_admin', default=False, help="set user as admin")
-@users_manager.option('--google', dest='google_auth', default=False, help="user uses Google Auth to login")
-def create(email, name, is_admin=False, google_auth=False):
+@users_manager.option('--admin', dest='is_admin', action="store_true", default=False, help="set user as admin")
+@users_manager.option('--google', dest='google_auth', action="store_true", default=False, help="user uses Google Auth to login")
+@users_manager.option('--password', dest='password', default=None, help="Password for users who don't use Google Auth (leave blank for prompt).")
+@users_manager.option('--permissions', dest='permissions', default=models.User.DEFAULT_PERMISSIONS, help="Comma seperated list of permissions (leave blank for default).")
+def create(email, name, permissions, is_admin=False, google_auth=False, password=None):
     print "Creating user (%s, %s)..." % (email, name)
     print "Admin: %r" % is_admin
     print "Login with Google Auth: %r\n" % google_auth
+    if isinstance(permissions, basestring):
+        permissions = permissions.split(',')
+        permissions.remove('') # in case it was empty string
 
-    user = models.User(email=email, name=name, is_admin=is_admin)
+    if is_admin:
+        permissions += ['admin']
+
+    user = models.User(email=email, name=name, permissions=permissions)
     if not google_auth:
-        password = prompt_pass("Password")
+        password = password or prompt_pass("Password")
         user.hash_password(password)
 
     try:
@@ -92,10 +111,7 @@ def delete(email):
 
 manager.add_command("database", database_manager)
 manager.add_command("users", users_manager)
+manager.add_command("import", import_manager)
 
 if __name__ == '__main__':
-    channel = logging.StreamHandler()
-    logging.getLogger().addHandler(channel)
-    logging.getLogger().setLevel(settings.LOG_LEVEL)
-
     manager.run()

migrations/add_permissions_to_user.py

@@ -0,0 +1,13 @@
+from playhouse.migrate import Migrator
+from redash import db
+from redash import models
+
+
+if __name__ == '__main__':
+    db.connect_db()
+    migrator = Migrator(db.database)
+    with db.database.transaction():
+        migrator.add_column(models.User, models.User.permissions, 'permissions')
+        models.User.update(permissions=['admin'] + models.User.DEFAULT_PERMISSIONS).where(models.User.is_admin == True).execute()
+
+    db.close_db(None)

migrations/add_view_query_permission.py

@@ -0,0 +1,13 @@
+import peewee
+from redash import db
+from redash import models
+
+
+if __name__ == '__main__':
+    db.connect_db()
+
+    previous_default_permissions = models.User.DEFAULT_PERMISSIONS[:]
+    previous_default_permissions.remove('view_query')
+    models.User.update(permissions=peewee.fn.array_append(models.User.permissions, 'view_query')).where(peewee.SQL("'view_source' = any(permissions)")).execute()
+
+    db.close_db(None)

migrations/create_activity_log.py

@@ -0,0 +1,11 @@
+from redash import db
+from redash import models
+
+if __name__ == '__main__':
+    db.connect_db()
+
+    if not models.ActivityLog.table_exists():
+        print "Creating activity_log table..."
+        models.ActivityLog.create_table()
+
+    db.close_db(None)

migrations/create_data_sources.py

@@ -0,0 +1,48 @@
+import logging
+import peewee
+from playhouse.migrate import Migrator
+from redash import db
+from redash import models
+from redash import settings
+
+if __name__ == '__main__':
+    db.connect_db()
+
+    if not models.DataSource.table_exists():
+        print "Creating data_sources table..."
+        models.DataSource.create_table()
+
+        default_data_source = models.DataSource.create(name="Default",
+                                                       type=settings.CONNECTION_ADAPTER,
+                                                       options=settings.CONNECTION_STRING)
+    else:
+        default_data_source = models.DataSource.select().first()
+
+    migrator = Migrator(db.database)
+    models.Query.data_source.null = True
+    models.QueryResult.data_source.null = True
+    try:
+        with db.database.transaction():
+            migrator.add_column(models.Query, models.Query.data_source, "data_source_id")
+    except peewee.ProgrammingError:
+        print "Failed to create data_source_id column -- assuming it already exists"
+
+    try:
+        with db.database.transaction():
+            migrator.add_column(models.QueryResult, models.QueryResult.data_source, "data_source_id")
+    except peewee.ProgrammingError:
+        print "Failed to create data_source_id column -- assuming it already exists"
+
+    print "Updating data source to existing one..."
+    models.Query.update(data_source=default_data_source.id).execute()
+    models.QueryResult.update(data_source=default_data_source.id).execute()
+
+    with db.database.transaction():
+        print "Setting data source to non nullable..."
+        migrator.set_nullable(models.Query, models.Query.data_source, False)
+
+    with db.database.transaction():
+        print "Setting data source to non nullable..."
+        migrator.set_nullable(models.QueryResult, models.QueryResult.data_source, False)
+
+    db.close_db(None)

rd_ui/Gruntfile.js

@@ -231,7 +231,7 @@ module.exports = function (grunt) {
         files: [{
           expand: true,
           cwd: '<%= yeoman.app %>',
-          src: ['*.html', 'views/*.html'],
+          src: ['*.html', 'views/**/*.html'],
           dest: '<%= yeoman.dist %>'
         }]
       }

rd_ui/app/index.html

@@ -4,7 +4,7 @@
 <!--[if IE 8]>         <html class="no-js lt-ie9" ng-app="redash" ng-controller='MainCtrl'> <![endif]-->
 <!--[if gt IE 8]><!--> <html class="no-js" ng-app="redash" ng-controller='MainCtrl'> <!--<![endif]-->
 <head>
-    <title ng-bind="'re:dash | ' + pageTitle"></title>
+    <title ng-bind="'{{name}} | ' + pageTitle"></title>
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
 
@@ -29,7 +29,7 @@
             <span class="icon-bar"></span>
             <span class="icon-bar"></span>
         </button>
-        <a class="navbar-brand" href="/"><strong>re:dash</strong></a>
+        <a class="navbar-brand" href="/"><strong>{{name}}</strong></a>
     </div>
     {% raw %}
     <div class="collapse navbar-collapse navbar-ex1-collapse">
@@ -51,21 +51,24 @@
                     <li ng-repeat="dashboard in otherDashboards">
                         <a role="menu-item" ng-href="/dashboard/{{dashboard.slug}}" ng-bind="dashboard.name"></a>
                     </li>
-                    <li class="divider"></li>
-                    <li><a data-toggle="modal" href="#new_dashboard_dialog">New Dashboard</a></li>
+                    <li class="divider" ng-show="currentUser.hasPermission('create_dashboard')"></li>
+                    <li><a data-toggle="modal" href="#new_dashboard_dialog" ng-show="currentUser.hasPermission('create_dashboard')">New Dashboard</a></li>
                 </ul>
             </li>
-            <li class="dropdown">
+            <li class="dropdown" ng-show="currentUser.hasPermission('view_query')">
                 <a href="#" class="dropdown-toggle" data-toggle="dropdown">Queries <b class="caret"></b></a>
                 <ul class="dropdown-menu">
-                    <li><a href="/queries/new">New Query</a></li>
+                    <li ng-show="currentUser.hasPermission('create_query')"><a href="/queries/new">New Query</a></li>
                     <li><a href="/queries">Queries</a></li>
                 </ul>
             </li>
         </ul>
         <ul class="nav navbar-nav navbar-right">
-            <p class="navbar-text avatar">
-                <img ng-src="{{currentUser.gravatar_url}}" class="img-circle" alt="{{currentUser.name}}" width="40" height="40"/>
+            <p class="navbar-text avatar" ng-show="currentUser.id" ng-cloak>
+                <img ng-src="{{currentUser.gravatar_url}}" class="img-circle" alt="{{currentUser.name}}"/>
+                <a target="_self" href="/logout" id="logout" title="Logout">
+                    <span class="glyphicon glyphicon-log-out"></span>
+                </a>
             </p>
         </ul>
     </div>
@@ -97,27 +100,39 @@
 <script src="/bower_components/angular-ui-codemirror/ui-codemirror.js"></script>
 <script src="/bower_components/highcharts/highcharts.js"></script>
 <script src="/bower_components/highcharts/modules/exporting.js"></script>
-<script src="/scripts/ng-highchart.js"></script>
-<script src="/scripts/smart-table.js"></script>
-<script src="/scripts/ui-bootstrap-tpls-0.5.0.min.js"></script>
 <script src="/bower_components/gridster/dist/jquery.gridster.js"></script>
 <script src="/bower_components/angular-growl/build/angular-growl.js"></script>
 <script src="/bower_components/pivottable/examples/pivot.js"></script>
 <script src="/bower_components/cornelius/src/cornelius.js"></script>
 <script src="/bower_components/mousetrap/mousetrap.js"></script>
 <script src="/bower_components/mousetrap/plugins/global-bind/mousetrap-global-bind.js"></script>
+
+<script src="/scripts/ng_highchart.js"></script>
+<script src="/scripts/ng_smart_table.js"></script>
+<script src="/scripts/ui-bootstrap-tpls-0.5.0.min.js"></script>
+
 <!-- endbuild -->
 
 <!-- build:js({.tmp,app}) /scripts/scripts.js -->
 <script src="/scripts/app.js"></script>
-<script src="/scripts/controllers.js"></script>
-<script src="/scripts/admin_controllers.js"></script>
-<script src="/scripts/directives.js"></script>
-<script src="/scripts/services.js"></script>
-<script src="/scripts/filters.js"></script>
+<script src="/scripts/services/services.js"></script>
+<script src="/scripts/services/resources.js"></script>
 <script src="/scripts/services/notifications.js"></script>
 <script src="/scripts/services/dashboards.js"></script>
-<script src="/scripts/query_fiddle/renderers.js"></script>
+<script src="/scripts/controllers/controllers.js"></script>
+<script src="/scripts/controllers/dashboard.js"></script>
+<script src="/scripts/controllers/admin_controllers.js"></script>
+<script src="/scripts/controllers/query_view.js"></script>
+<script src="/scripts/controllers/query_source.js"></script>
+<script src="/scripts/visualizations/base.js"></script>
+<script src="/scripts/visualizations/chart.js"></script>
+<script src="/scripts/visualizations/cohort.js"></script>
+<script src="/scripts/visualizations/table.js"></script>
+<script src="/scripts/visualizations/pivot.js"></script>
+<script src="/scripts/directives/directives.js"></script>
+<script src="/scripts/directives/query_directives.js"></script>
+<script src="/scripts/directives/dashboard_directives.js"></script>
+<script src="/scripts/filters.js"></script>
 <!-- endbuild -->
 
 <script>
@@ -128,6 +143,10 @@
         return user_id && (user_id == currentUser.id);
     };
 
+    currentUser.hasPermission = function(permission) {
+        return this.permissions.indexOf(permission) != -1;
+    }
+
     {{ analytics|safe }}
 </script>
 

rd_ui/app/login.html

@@ -4,7 +4,7 @@
 <!--[if IE 8]>         <html class="no-js lt-ie9"> <![endif]-->
 <!--[if gt IE 8]><!--> <html class="no-js"> <!--<![endif]-->
 <head>
-    <title>re:dash Login</title>
+    <title>{{name}} Login</title>
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
 
@@ -26,7 +26,7 @@
             <span class="icon-bar"></span>
             <span class="icon-bar"></span>
         </button>
-        <a class="navbar-brand" href="/"><strong>re:dash</strong></a>
+        <a class="navbar-brand" href="/"><strong>{{name}}</strong></a>
     </div>
     </div>
 </nav>

rd_ui/app/scripts/app.js

@@ -5,6 +5,7 @@ angular.module('redash', [
     'redash.filters',
     'redash.services',
     'redash.renderers',
+    'redash.visualization',
     'ui.codemirror',
     'highchart',
     'angular-growl',
@@ -16,6 +17,11 @@ angular.module('redash', [
 ]).config(['$routeProvider', '$locationProvider', '$compileProvider', 'growlProvider',
     function($routeProvider, $locationProvider, $compileProvider, growlProvider) {
 
+        function getQuery(Query, $route) {
+            var query = Query.get({'id': $route.current.params.queryId });
+            return query.$promise;
+        };
+
         $compileProvider.aHrefSanitizationWhitelist(/^\s*(https?|http|data):/);
         $locationProvider.html5Mode(true);
         growlProvider.globalTimeToLive(2000);
@@ -30,14 +36,30 @@ angular.module('redash', [
             reloadOnSearch: false
         });
         $routeProvider.when('/queries/new', {
-            templateUrl: '/views/queryfiddle.html',
-            controller: 'QueryFiddleCtrl',
-            reloadOnSearch: false
+            templateUrl: '/views/query.html',
+            controller: 'QuerySourceCtrl',
+            reloadOnSearch: false,
+            resolve: {
+                'query': ['Query', function newQuery(Query) {
+                    return Query.newQuery();
+                }]
+            }
         });
         $routeProvider.when('/queries/:queryId', {
-            templateUrl: '/views/queryfiddle.html',
-            controller: 'QueryFiddleCtrl',
-            reloadOnSearch: false
+            templateUrl: '/views/query.html',
+            controller: 'QueryViewCtrl',
+            reloadOnSearch: false,
+            resolve: {
+                'query': ['Query', '$route', getQuery]
+            }
+        });
+        $routeProvider.when('/queries/:queryId/source', {
+            templateUrl: '/views/query.html',
+            controller: 'QuerySourceCtrl',
+            reloadOnSearch: false,
+            resolve: {
+                'query': ['Query', '$route', getQuery]
+            }
         });
         $routeProvider.when('/admin/status', {
             templateUrl: '/views/admin_status.html',
@@ -51,9 +73,6 @@ angular.module('redash', [
             redirectTo: '/'
         });
 
-        Highcharts.setOptions({
-            colors: ["#4572A7", "#AA4643", "#89A54E", "#80699B", "#3D96AE",
-                        "#DB843D", "#92A8CD", "#A47D7C", "#B5CA92"]
-        });
+
     }
 ]);

rd_ui/app/scripts/controllers.js

@@ -1,438 +0,0 @@
-(function () {
-    var DashboardCtrl = function ($scope, $routeParams, $http, $timeout, Dashboard) {
-        $scope.refreshEnabled = false;
-        $scope.refreshRate = 60;
-        $scope.dashboard = Dashboard.get({slug: $routeParams.dashboardSlug}, function(dashboard) {
-            $scope.$parent.pageTitle = dashboard.name;
-        });
-
-        var autoRefresh = function() {
-            if ($scope.refreshEnabled) {
-                $timeout(function() {
-                    Dashboard.get({slug: $routeParams.dashboardSlug}, function(dashboard) {
-                        var newWidgets = _.groupBy(_.flatten(dashboard.widgets), 'id');
-
-                        _.each($scope.dashboard.widgets, function(row) {
-                            _.each(row, function(widget, i) {
-                                var newWidget = newWidgets[widget.id];
-                                if (newWidget && newWidget[0].visualization.query.latest_query_data_id != widget.visualization.query.latest_query_data_id ) {
-                                    row[i] = newWidget[0];
-                                }
-                            });
-                        });
-
-                        autoRefresh();
-                    });
-
-                }, $scope.refreshRate);
-            };
-        }
-
-        $scope.triggerRefresh = function(){
-            $scope.refreshEnabled = !$scope.refreshEnabled;
-
-            if ($scope.refreshEnabled) {
-                var refreshRate = _.min(_.flatten($scope.dashboard.widgets), function(widget) {
-                    return widget.visualization.query.ttl;
-                }).visualization.query.ttl;
-
-                $scope.refreshRate = _.max([120, refreshRate * 2])*1000;
-
-                autoRefresh();
-            }
-        };
-    };
-
-    var WidgetCtrl = function ($scope, $http, $location, Query) {
-        $scope.deleteWidget = function() {
-            if (!confirm('Are you sure you want to remove "' + $scope.widget.visualization.name + '" from the dashboard?')) {
-                return;
-            }
-
-            $http.delete('/api/widgets/' + $scope.widget.id).success(function() {
-                $scope.dashboard.widgets = _.map($scope.dashboard.widgets, function(row) {
-                    return _.filter(row, function(widget) {
-                        return widget.id != $scope.widget.id;
-                    })
-                });
-            });
-        };
-
-        $scope.open = function(query, visualization) {
-            $location.path('/queries/' + query.id);
-            $location.hash(visualization.id);
-        }
-
-        $scope.query = new Query($scope.widget.visualization.query);
-        $scope.queryResult = $scope.query.getQueryResult();
-
-        $scope.updateTime = (new Date($scope.queryResult.getUpdatedAt())).toISOString();
-        $scope.nextUpdateTime = moment(new Date(($scope.query.updated_at + $scope.query.ttl + $scope.query.runtime + 300) * 1000)).fromNow();
-
-        $scope.updateTime = '';
-    }
-
-    var QueryFiddleCtrl = function ($scope, $window, $location, $routeParams, $http, $location, growl, notifications, Query, Visualization) {
-        var DEFAULT_TAB = 'table';
-        var pristineHash = null;
-        var leavingPageText = "You will lose your changes if you leave";
-
-        $scope.dirty = undefined;
-        $scope.newVisualization = undefined;
-
-        $window.onbeforeunload = function(){
-            if (currentUser.canEdit($scope.query) && $scope.dirty) {
-                return leavingPageText;
-            }
-        }
-
-        Mousetrap.bindGlobal("meta+s", function(e) {
-            e.preventDefault();
-
-            if (currentUser.canEdit($scope.query)) {
-                $scope.saveQuery();
-            }
-        });
-
-        $scope.$on('$locationChangeStart', function(event, next, current) {
-            if (next.split("#")[0] == current.split("#")[0]) {
-                return;
-            }
-
-            if (!currentUser.canEdit($scope.query)) {
-                return;
-            }
-
-            if($scope.dirty &&
-                !confirm(leavingPageText + "\n\nAre you sure you want to leave this page?")) {
-                event.preventDefault();
-            } else {
-                Mousetrap.unbind("meta+s");
-            }
-        });
-
-        $scope.$parent.pageTitle = "Query Fiddle";
-
-        $scope.$watch(function() {return $location.hash()}, function(hash) {
-            $scope.selectedTab = hash || DEFAULT_TAB;
-        });
-
-        $scope.lockButton = function (lock) {
-            $scope.queryExecuting = lock;
-        };
-
-        $scope.formatQuery = function() {
-            $scope.editorOptions.readOnly = 'nocursor';
-
-            $http.post('/api/queries/format', {'query': $scope.query.query}).success(function(response) {
-                $scope.query.query = response;
-                $scope.editorOptions.readOnly = false;
-            })
-        }
-
-        $scope.saveQuery = function (duplicate, oldId) {
-            if (!oldId) {
-                oldId = $scope.query.id;
-            }
-            delete $scope.query.latest_query_data;
-            $scope.query.$save(function (q) {
-                pristineHash = q.getHash();
-                $scope.dirty = false;
-
-                if (duplicate) {
-                    growl.addInfoMessage("Query duplicated.", {ttl: 2000});
-                } else{
-                    growl.addSuccessMessage("Query saved.", {ttl: 2000});
-                }
-
-                if (oldId != q.id) {
-                    if (oldId == undefined) {
-                        $location.path($location.path().replace('new', q.id)).replace();
-                    } else {
-                        // TODO: replace this with a safer method
-                        $location.path($location.path().replace(oldId, q.id)).replace();
-
-                        // Reset visualizations tab to table after duplicating a query:
-                        $location.hash('table');
-                    }
-                }
-            }, function(httpResponse) {
-                growl.addErrorMessage("Query could not be saved");
-            });
-        };
-
-        $scope.duplicateQuery = function () {
-            var oldId = $scope.query.id;
-            $scope.query.id = null;
-            $scope.query.ttl = -1;
-
-            $scope.saveQuery(true, oldId);
-        };
-
-        // Query Editor:
-        $scope.editorOptions = {
-            mode: 'text/x-sql',
-            lineWrapping: true,
-            lineNumbers: true,
-            readOnly: false,
-            matchBrackets: true,
-            autoCloseBrackets: true
-        };
-
-        $scope.refreshOptions = [
-            {value: -1, name: 'No Refresh'},
-            {value: 60, name: 'Every minute'},
-        ]
-
-        _.each(_.range(1, 13), function(i) {
-            $scope.refreshOptions.push({value: i*3600, name: 'Every ' + i + 'h'});
-        })
-
-        $scope.refreshOptions.push({value: 24*3600, name: 'Every 24h'});
-        $scope.refreshOptions.push({value: 7*24*3600, name: 'Once a week'});
-
-        $scope.$watch('queryResult && queryResult.getError()', function (newError, oldError) {
-            if (newError == undefined) {
-                return;
-            }
-
-            if (oldError == undefined && newError != undefined) {
-                $scope.lockButton(false);
-            }
-        });
-
-        $scope.$watch('queryResult && queryResult.getData()', function (data, oldData) {
-            if (!data) {
-                return;
-            }
-
-            if ($scope.queryResult.getId() == null) {
-                $scope.dataUri = "";
-            } else {
-                $scope.dataUri = '/api/queries/' + $scope.query.id + '/results/' + $scope.queryResult.getId() + '.csv';
-                $scope.dataFilename = $scope.query.name.replace(" ", "_") + moment($scope.queryResult.getUpdatedAt()).format("_YYYY_MM_DD") + ".csv";
-            }
-        });
-
-        $scope.$watch("queryResult && queryResult.getStatus()", function (status) {
-            if (!status) {
-                return;
-            }
-
-            if (status == "done") {
-                if ($scope.query.id && $scope.query.latest_query_data_id != $scope.queryResult.getId() &&
-                    $scope.query.query_hash == $scope.queryResult.query_result.query_hash) {
-                    Query.save({'id': $scope.query.id, 'latest_query_data_id': $scope.queryResult.getId()})
-                }
-                $scope.query.latest_query_data_id = $scope.queryResult.getId();
-
-                notifications.showNotification("re:dash", $scope.query.name + " updated.");
-
-                $scope.lockButton(false);
-            }
-        });
-
-        if ($routeParams.queryId != undefined) {
-            $scope.query = Query.get({id: $routeParams.queryId}, function(q) {
-                pristineHash = q.getHash();
-                $scope.dirty = false;
-                $scope.queryResult = $scope.query.getQueryResult();
-            });
-        } else {
-            $scope.query = new Query({query: "", name: "New Query", ttl: -1, user: currentUser});
-            $scope.lockButton(false);
-        }
-
-        $scope.$watch('query.name', function() {
-            $scope.$parent.pageTitle = $scope.query.name;
-        });
-
-        $scope.$watch(function() {
-            return $scope.query.getHash();
-        }, function(newHash) {
-            $scope.dirty = (newHash !== pristineHash);
-        });
-
-        $scope.executeQuery = function() {
-            $scope.queryResult = $scope.query.getQueryResult(0);
-            $scope.lockButton(true);
-            $scope.cancelling = false;
-        };
-
-        $scope.cancelExecution = function() {
-            $scope.cancelling = true;
-            $scope.queryResult.cancelExecution();
-        };
-
-        $scope.deleteVisualization = function($e, vis) {
-            $e.preventDefault();
-            if (confirm('Are you sure you want to delete ' + vis.name + ' ?')) {
-                Visualization.delete(vis);
-                if ($scope.selectedTab == vis.id) {
-                    $scope.selectedTab = DEFAULT_TAB;
-                }
-                $scope.query.visualizations =
-                    $scope.query.visualizations.filter(function(v) {
-                        return vis.id !== v.id;
-                    });
-            }
-        };
-    }
-
-    var QueriesCtrl = function($scope, $http, $location, $filter, Query) {
-        $scope.$parent.pageTitle = "All Queries";
-        $scope.gridConfig = {
-            isPaginationEnabled: true,
-            itemsByPage: 50,
-            maxSize: 8,
-            isGlobalSearchActivated: true
-        }
-
-        $scope.allQueries = [];
-        $scope.queries = [];
-
-        var dateFormatter = function (value) {
-            if (!value) return "-";
-            return value.format("DD/MM/YY HH:mm");
-        }
-
-        var filterQueries = function() {
-            $scope.queries = _.filter($scope.allQueries, function(query) {
-                if (!$scope.selectedTab) {
-                    return false;
-                }
-
-                if ($scope.selectedTab.key == 'my') {
-                    return query.user.id == currentUser.id && query.name != 'New Query';
-                } else if ($scope.selectedTab.key == 'drafts') {
-                    return query.user.id == currentUser.id && query.name == 'New Query';
-                }
-
-                return query.name != 'New Query';
-            });
-        }
-
-        Query.query(function(queries) {
-            $scope.allQueries = _.map(queries, function(query) {
-                query.created_at = moment(query.created_at);
-                query.last_retrieved_at = moment(query.last_retrieved_at);
-                return query;
-            });
-
-            filterQueries();
-        });
-
-        $scope.gridColumns = [
-            {
-                "label": "Name",
-                "map": "name",
-                "cellTemplateUrl": "/views/queries_query_name_cell.html"
-            },
-            {
-                'label': 'Created By',
-                'map': 'user.name'
-            },
-            {
-                'label': 'Created At',
-                'map': 'created_at',
-                'formatFunction': dateFormatter
-            },
-            {
-                'label': 'Runtime (avg)',
-                'map': 'avg_runtime',
-                'formatFunction': function(value) {
-                    return $filter('durationHumanize')(value);
-                }
-            },
-            {
-                'label': 'Runtime (min)',
-                'map': 'min_runtime',
-                'formatFunction': function(value) {
-                    return $filter('durationHumanize')(value);
-                }
-            },
-            {
-                'label': 'Runtime (max)',
-                'map': 'max_runtime',
-                'formatFunction': function(value) {
-                    return $filter('durationHumanize')(value);
-                }
-            },
-            {
-                'label': 'Last Executed At',
-                'map': 'last_retrieved_at',
-                'formatFunction': dateFormatter
-            },
-            {
-                'label': 'Times Executed',
-                'map': 'times_retrieved'
-            },
-            {
-                'label': 'Update Schedule',
-                'map': 'ttl',
-                'formatFunction': function(value) {
-                    return $filter('refreshRateHumanize')(value);
-                }
-            }
-        ]
-        $scope.tabs = [{"name": "My Queries", "key": "my"}, {"key": "all", "name": "All Queries"}, {"key": "drafts", "name": "Drafts"}];
-
-        $scope.$watch('selectedTab', function(tab) {
-            if (tab) {
-                $scope.$parent.pageTitle =  tab.name;
-            }
-
-            filterQueries();
-        });
-    }
-
-    var MainCtrl = function ($scope, Dashboard, notifications) {
-        $scope.dashboards = [];
-        $scope.reloadDashboards = function() {
-            Dashboard.query(function (dashboards) {
-                $scope.dashboards = _.sortBy(dashboards, "name");
-                $scope.allDashboards = _.groupBy($scope.dashboards, function(d) {
-                    parts = d.name.split(":");
-                    if (parts.length == 1) {
-                        return "Other";
-                    }
-                    return parts[0];
-                });
-                $scope.otherDashboards = $scope.allDashboards['Other'] || [];
-                $scope.groupedDashboards = _.omit($scope.allDashboards, 'Other');
-            });
-        }
-
-        $scope.reloadDashboards();
-
-        $scope.currentUser = currentUser;
-        $scope.newDashboard = {
-            'name': null,
-            'layout': null
-        }
-
-        $(window).click(function () {
-            notifications.getPermissions();
-        });
-    }
-
-    var IndexCtrl = function($scope, Dashboard) {
-        $scope.$parent.pageTitle = "Home";
-
-        $scope.archiveDashboard = function(dashboard) {
-            if (confirm('Are you sure you want to delete "' + dashboard.name + '" dashboard?')) {
-                dashboard.$delete(function() {
-                    $scope.$parent.reloadDashboards();
-                });
-            }
-        }
-    }
-
-    angular.module('redash.controllers', [])
-        .controller('DashboardCtrl', ['$scope', '$routeParams', '$http', '$timeout', 'Dashboard', DashboardCtrl])
-        .controller('WidgetCtrl', ['$scope', '$http', '$location', 'Query', WidgetCtrl])
-        .controller('QueriesCtrl', ['$scope', '$http', '$location', '$filter', 'Query', QueriesCtrl])
-        .controller('QueryFiddleCtrl', ['$scope', '$window', '$location', '$routeParams', '$http', '$location', 'growl', 'notifications', 'Query', 'Visualization', QueryFiddleCtrl])
-        .controller('IndexCtrl', ['$scope', 'Dashboard', IndexCtrl])
-        .controller('MainCtrl', ['$scope', 'Dashboard', 'notifications', MainCtrl]);
-})();

rd_ui/app/scripts/controllers/controllers.js

@@ -0,0 +1,155 @@
+(function () {
+    var QueriesCtrl = function($scope, $http, $location, $filter, Query) {
+        $scope.$parent.pageTitle = "All Queries";
+        $scope.gridConfig = {
+            isPaginationEnabled: true,
+            itemsByPage: 50,
+            maxSize: 8,
+            isGlobalSearchActivated: true
+        }
+
+        $scope.allQueries = [];
+        $scope.queries = [];
+
+        var dateFormatter = function (value) {
+            if (!value) return "-";
+            return value.format("DD/MM/YY HH:mm");
+        }
+
+        var filterQueries = function() {
+            $scope.queries = _.filter($scope.allQueries, function(query) {
+                if (!$scope.selectedTab) {
+                    return false;
+                }
+
+                if ($scope.selectedTab.key == 'my') {
+                    return query.user.id == currentUser.id && query.name != 'New Query';
+                } else if ($scope.selectedTab.key == 'drafts') {
+                    return query.user.id == currentUser.id && query.name == 'New Query';
+                }
+
+                return query.name != 'New Query';
+            });
+        }
+
+        Query.query(function(queries) {
+            $scope.allQueries = _.map(queries, function(query) {
+                query.created_at = moment(query.created_at);
+                query.last_retrieved_at = moment(query.last_retrieved_at);
+                return query;
+            });
+
+            filterQueries();
+        });
+
+        $scope.gridColumns = [
+            {
+                "label": "Name",
+                "map": "name",
+                "cellTemplateUrl": "/views/queries_query_name_cell.html"
+            },
+            {
+                'label': 'Created By',
+                'map': 'user.name'
+            },
+            {
+                'label': 'Created At',
+                'map': 'created_at',
+                'formatFunction': dateFormatter
+            },
+            {
+                'label': 'Runtime (avg)',
+                'map': 'avg_runtime',
+                'formatFunction': function(value) {
+                    return $filter('durationHumanize')(value);
+                }
+            },
+            {
+                'label': 'Runtime (min)',
+                'map': 'min_runtime',
+                'formatFunction': function(value) {
+                    return $filter('durationHumanize')(value);
+                }
+            },
+            {
+                'label': 'Runtime (max)',
+                'map': 'max_runtime',
+                'formatFunction': function(value) {
+                    return $filter('durationHumanize')(value);
+                }
+            },
+            {
+                'label': 'Last Executed At',
+                'map': 'last_retrieved_at',
+                'formatFunction': dateFormatter
+            },
+            {
+                'label': 'Times Executed',
+                'map': 'times_retrieved'
+            },
+            {
+                'label': 'Update Schedule',
+                'map': 'ttl',
+                'formatFunction': function(value) {
+                    return $filter('refreshRateHumanize')(value);
+                }
+            }
+        ]
+        $scope.tabs = [{"name": "My Queries", "key": "my"}, {"key": "all", "name": "All Queries"}, {"key": "drafts", "name": "Drafts"}];
+
+        $scope.$watch('selectedTab', function(tab) {
+            if (tab) {
+                $scope.$parent.pageTitle =  tab.name;
+            }
+
+            filterQueries();
+        });
+    }
+
+    var MainCtrl = function ($scope, Dashboard, notifications) {
+        $scope.dashboards = [];
+        $scope.reloadDashboards = function() {
+            Dashboard.query(function (dashboards) {
+                $scope.dashboards = _.sortBy(dashboards, "name");
+                $scope.allDashboards = _.groupBy($scope.dashboards, function(d) {
+                    parts = d.name.split(":");
+                    if (parts.length == 1) {
+                        return "Other";
+                    }
+                    return parts[0];
+                });
+                $scope.otherDashboards = $scope.allDashboards['Other'] || [];
+                $scope.groupedDashboards = _.omit($scope.allDashboards, 'Other');
+            });
+        }
+
+        $scope.reloadDashboards();
+
+        $scope.currentUser = currentUser;
+        $scope.newDashboard = {
+            'name': null,
+            'layout': null
+        }
+
+        $(window).click(function () {
+            notifications.getPermissions();
+        });
+    }
+
+    var IndexCtrl = function($scope, Dashboard) {
+        $scope.$parent.pageTitle = "Home";
+
+        $scope.archiveDashboard = function(dashboard) {
+            if (confirm('Are you sure you want to delete "' + dashboard.name + '" dashboard?')) {
+                dashboard.$delete(function() {
+                    $scope.$parent.reloadDashboards();
+                });
+            }
+        }
+    }
+
+    angular.module('redash.controllers', [])
+        .controller('QueriesCtrl', ['$scope', '$http', '$location', '$filter', 'Query', QueriesCtrl])
+        .controller('IndexCtrl', ['$scope', 'Dashboard', IndexCtrl])
+        .controller('MainCtrl', ['$scope', 'Dashboard', 'notifications', MainCtrl]);
+})();

rd_ui/app/scripts/controllers/dashboard.js

@@ -0,0 +1,78 @@
+(function() {
+  var DashboardCtrl = function($scope, $routeParams, $http, $timeout, Dashboard) {
+    $scope.refreshEnabled = false;
+    $scope.refreshRate = 60;
+    $scope.dashboard = Dashboard.get({
+      slug: $routeParams.dashboardSlug
+    }, function(dashboard) {
+      $scope.$parent.pageTitle = dashboard.name;
+    });
+
+    var autoRefresh = function() {
+      if ($scope.refreshEnabled) {
+        $timeout(function() {
+          Dashboard.get({
+            slug: $routeParams.dashboardSlug
+          }, function(dashboard) {
+            var newWidgets = _.groupBy(_.flatten(dashboard.widgets), 'id');
+
+            _.each($scope.dashboard.widgets, function(row) {
+              _.each(row, function(widget, i) {
+                var newWidget = newWidgets[widget.id];
+                if (newWidget && newWidget[0].visualization.query.latest_query_data_id != widget.visualization.query.latest_query_data_id) {
+                  row[i] = newWidget[0];
+                }
+              });
+            });
+
+            autoRefresh();
+          });
+
+        }, $scope.refreshRate);
+      };
+    }
+
+    $scope.triggerRefresh = function() {
+      $scope.refreshEnabled = !$scope.refreshEnabled;
+
+      if ($scope.refreshEnabled) {
+        var refreshRate = _.min(_.flatten($scope.dashboard.widgets), function(widget) {
+          return widget.visualization.query.ttl;
+        }).visualization.query.ttl;
+
+        $scope.refreshRate = _.max([120, refreshRate * 2]) * 1000;
+
+        autoRefresh();
+      }
+    };
+  };
+
+  var WidgetCtrl = function($scope, $http, $location, Query) {
+    $scope.deleteWidget = function() {
+      if (!confirm('Are you sure you want to remove "' + $scope.widget.visualization.name + '" from the dashboard?')) {
+        return;
+      }
+
+      $http.delete('/api/widgets/' + $scope.widget.id).success(function() {
+        $scope.dashboard.widgets = _.map($scope.dashboard.widgets, function(row) {
+          return _.filter(row, function(widget) {
+            return widget.id != $scope.widget.id;
+          })
+        });
+      });
+    };
+
+    $scope.query = new Query($scope.widget.visualization.query);
+    $scope.queryResult = $scope.query.getQueryResult();
+
+    $scope.updateTime = (new Date($scope.queryResult.getUpdatedAt())).toISOString();
+    $scope.nextUpdateTime = moment(new Date(($scope.query.updated_at + $scope.query.ttl + $scope.query.runtime + 300) * 1000)).fromNow();
+
+    $scope.updateTime = '';
+  };
+
+  angular.module('redash.controllers')
+    .controller('DashboardCtrl', ['$scope', '$routeParams', '$http', '$timeout', 'Dashboard', DashboardCtrl])
+    .controller('WidgetCtrl', ['$scope', '$http', '$location', 'Query', WidgetCtrl])
+
+})();

rd_ui/app/scripts/controllers/query_source.js

@@ -0,0 +1,100 @@
+(function() {
+  'use strict';
+
+  function QuerySourceCtrl($controller, $scope, $location, growl, Query, Visualization, KeyboardShortcuts) {
+    // extends QueryViewCtrl
+    $controller('QueryViewCtrl', {$scope: $scope});
+
+    var
+    isNewQuery = !$scope.query.id,
+    queryText = $scope.query.query,
+
+    // ref to QueryViewCtrl.saveQuery
+    saveQuery = $scope.saveQuery,
+
+    shortcuts = {
+      'meta+s': function() {
+          if ($scope.canEdit) {
+            $scope.saveQuery();
+          }
+      }
+    };
+
+    $scope.sourceMode = true;
+    $scope.canEdit = currentUser.canEdit($scope.query);
+    $scope.isDirty = false;
+
+    $scope.newVisualization = undefined;
+
+    KeyboardShortcuts.bind(shortcuts);
+
+    // @override
+    $scope.saveQuery = function(options, data) {
+      var savePromise = saveQuery(options, data);
+
+      savePromise.then(function(savedQuery) {
+        queryText = savedQuery.query;
+        $scope.isDirty = $scope.query.query !== queryText;
+
+        if (isNewQuery) {
+          // redirect to new created query (keep hash)
+          $location.path(savedQuery.getSourceLink()).replace();
+        }
+      });
+
+      return savePromise;
+    };
+
+    $scope.duplicateQuery = function() {
+      $scope.query.id = null;
+      $scope.query.ttl = -1;
+
+      $scope.saveQuery({
+        successMessage: 'Query forked',
+        errorMessage: 'Query could not be forked'
+      }).then(function redirect(savedQuery) {
+        // redirect to forked query (clear hash)
+        $location.url(savedQuery.getSourceLink()).replace()
+      });
+    };
+
+    $scope.deleteVisualization = function($e, vis) {
+      $e.preventDefault();
+      if (confirm('Are you sure you want to delete ' + vis.name + ' ?')) {
+        Visualization.delete(vis);
+        if ($scope.selectedTab == vis.id) {
+          $scope.selectedTab = DEFAULT_TAB;
+          $location.hash($scope.selectedTab);
+        }
+        $scope.query.visualizations =
+          $scope.query.visualizations.filter(function(v) {
+            return vis.id !== v.id;
+          });
+      }
+    };
+
+    $scope.$watch('query.query', function(newQueryText) {
+      $scope.isDirty = (newQueryText !== queryText);
+    });
+
+    $scope.$on('$destroy', function destroy() {
+      KeyboardShortcuts.unbind(shortcuts);
+    });
+
+    if (isNewQuery) {
+      // save new query when creating a visualization
+      var unbind = $scope.$watch('selectedTab == "add"', function(triggerSave) {
+        if (triggerSave) {
+          unbind();
+          $scope.saveQuery();
+        }
+      });
+    }
+
+  };
+
+  angular.module('redash.controllers').controller('QuerySourceCtrl', [
+    '$controller', '$scope', '$location', 'growl', 'Query',
+    'Visualization', 'KeyboardShortcuts', QuerySourceCtrl
+    ]);
+})();

rd_ui/app/scripts/controllers/query_view.js

@@ -0,0 +1,149 @@
+(function() {
+  'use strict';
+
+  function QueryViewCtrl($scope, $route, $location, notifications, growl, Query, DataSource) {
+    var DEFAULT_TAB = 'table';
+
+    $scope.query = $route.current.locals.query;
+    $scope.queryResult = $scope.query.getQueryResult();
+    $scope.queryExecuting = false;
+
+    $scope.isQueryOwner = currentUser.id === $scope.query.user.id;
+    $scope.canViewSource = currentUser.hasPermission('view_source');
+
+    $scope.dataSources = DataSource.get(function(dataSources) {
+      $scope.query.data_source_id = $scope.query.data_source_id || dataSources[0].id;
+    });
+
+    $scope.lockButton = function(lock) {
+      $scope.queryExecuting = lock;
+    };
+
+    $scope.saveQuery = function(options, data) {
+      if (data) {
+        data.id = $scope.query.id;
+      } else {
+        data = $scope.query;
+      }
+
+      options = _.extend({}, {
+        successMessage: 'Query saved',
+        errorMessage: 'Query could not be saved'
+      }, options);
+
+      delete $scope.query.latest_query_data;
+
+      return Query.save(data, function() {
+        growl.addSuccessMessage(options.successMessage);
+      }, function(httpResponse) {
+        growl.addErrorMessage(options.errorMessage);
+      })
+      .$promise;
+    }
+
+    $scope.saveDescription = function() {
+      $scope.saveQuery(undefined, {'description': $scope.query.description});
+    };
+
+    $scope.saveName = function() {
+      $scope.saveQuery(undefined, {'name': $scope.query.name});
+    };
+
+    $scope.executeQuery = function() {
+      $scope.queryResult = $scope.query.getQueryResult(0);
+      $scope.lockButton(true);
+      $scope.cancelling = false;
+    };
+
+    $scope.cancelExecution = function() {
+      $scope.cancelling = true;
+      $scope.queryResult.cancelExecution();
+    };
+
+    $scope.updateDataSource = function() {
+        $scope.query.latest_query_data = null;
+        $scope.query.latest_query_data_id = null;
+        Query.save({
+            'id': $scope.query.id,
+            'data_source_id': $scope.query.data_source_id,
+            'latest_query_data_id': null
+        });
+
+        $scope.executeQuery();
+    };
+
+    $scope.setVisualizationTab = function (visualization) {
+      $scope.selectedTab = visualization.id;
+      $location.hash(visualization.id);
+    };
+
+    $scope.$watch('query.name', function() {
+      $scope.$parent.pageTitle = $scope.query.name;
+    });
+
+    $scope.$watch('queryResult && queryResult.getError()',
+      function(newError, oldError) {
+        if (newError == undefined) {
+          return;
+        }
+
+        if (oldError == undefined && newError != undefined) {
+          $scope.lockButton(false);
+        }
+      });
+
+    $scope.$watch('queryResult && queryResult.getData()',
+      function(data, oldData) {
+        if (!data) {
+          return;
+        }
+
+        $scope.filters = $scope.queryResult.getFilters();
+
+        if ($scope.queryResult.getId() == null) {
+          $scope.dataUri = "";
+        } else {
+          $scope.dataUri =
+            '/api/queries/' + $scope.query.id + '/results/' +
+            $scope.queryResult.getId() + '.csv';
+
+          $scope.dataFilename =
+            $scope.query.name.replace(" ", "_") +
+            moment($scope.queryResult.getUpdatedAt()).format("_YYYY_MM_DD") +
+            ".csv";
+        }
+      });
+
+    $scope.$watch("queryResult && queryResult.getStatus()", function(status) {
+      if (!status) {
+        return;
+      }
+
+      if (status == "done") {
+        if ($scope.query.id &&
+          $scope.query.latest_query_data_id != $scope.queryResult.getId() &&
+          $scope.query.query_hash == $scope.queryResult.query_result.query_hash) {
+          Query.save({
+            'id': $scope.query.id,
+            'latest_query_data_id': $scope.queryResult.getId()
+          })
+        }
+        $scope.query.latest_query_data_id = $scope.queryResult.getId();
+
+        notifications.showNotification("re:dash", $scope.query.name + " updated.");
+
+        $scope.lockButton(false);
+      }
+    });
+
+    $scope.$watch(function() {
+      return $location.hash()
+    }, function(hash) {
+      $scope.selectedTab = hash || DEFAULT_TAB;
+    });
+  };
+
+  angular.module('redash.controllers')
+    .controller('QueryViewCtrl',
+      ['$scope', '$route', '$location', 'notifications', 'growl', 'Query', 'DataSource', QueryViewCtrl]);
+})();

rd_ui/app/scripts/directives.js

@@ -1,436 +0,0 @@
-(function() {
-    'use strict';
-
-    var directives = angular.module('redash.directives', []);
-
-    directives.directive('rdTab', ['$location', function($location) {
-        return {
-            restrict: 'E',
-            scope: {
-                'id': '@',
-                'name': '@'
-            },
-            transclude: true,
-            template: '<li class="rd-tab" ng-class="{active: id==selectedTab}"><a href="#{{id}}">{{name}}<span ng-transclude></span></a></li>',
-            replace: true,
-            link: function(scope) {
-                scope.$watch(function(){return scope.$parent.selectedTab}, function(tab) {
-                    scope.selectedTab = tab;
-                });
-            }
-        }
-    }]);
-
-    directives.directive('rdTabs', ['$location', '$rootScope', function($location, $rootScope) {
-        return {
-            restrict: 'E',
-            scope: {
-                tabsCollection: '=',
-                selectedTab: '='
-            },
-            template: '<ul class="nav nav-tabs"><li ng-class="{active: tab==selectedTab}" ng-repeat="tab in tabsCollection"><a href="#{{tab.key}}">{{tab.name}}</a></li></ul>',
-            replace: true,
-            link: function($scope, element, attrs) {
-                $scope.selectTab = function(tabKey) {
-                    $scope.selectedTab  = _.find($scope.tabsCollection, function(tab) { return tab.key == tabKey; });
-                }
-
-                $scope.$watch(function() { return $location.hash()}, function(hash) {
-                    if (hash) {
-                        $scope.selectTab($location.hash());
-                    } else {
-                        $scope.selectTab($scope.tabsCollection[0].key);
-                    }
-                });
-            }
-        }
-    }]);
-
-    directives.directive('editVisulatizationForm', ['Visualization', 'growl', function(Visualization, growl) {
-        return {
-            restrict: 'E',
-            templateUrl: '/views/edit_visualization.html',
-            replace: true,
-            scope: {
-                query: '=',
-                vis: '=?'
-            },
-            link: function(scope, element, attrs) {
-                scope.advancedMode = false;
-                scope.visTypes = {
-                    'Chart': Visualization.prototype.TYPES.CHART,
-                    'Cohort': Visualization.prototype.TYPES.COHORT
-                };
-                scope.seriesTypes = {
-                    'Line': 'line',
-                    'Column': 'column',
-                    'Area': 'area',
-                    'Scatter': 'scatter',
-                    'Pie': 'pie'
-                };
-
-                scope.stackingOptions = {
-                    "None": "none",
-                    "Normal": "normal",
-                    "Percent": "percent"
-                };
-
-                scope.stacking = "none";
-
-                if (!scope.vis) {
-                    // create new visualization
-                    // wait for query to load to populate with defaults
-                    var unwatch = scope.$watch('query', function(q) {
-                        if (q && q.id) {
-                            unwatch();
-                            scope.vis = {
-                                'query_id': q.id,
-                                'type': Visualization.prototype.TYPES.CHART,
-                                'name': '',
-                                'description': q.description || '',
-                                'options': newOptions(Visualization.prototype.TYPES.CHART)
-                            };
-                        }
-                    }, true);
-                }
-
-                function newOptions(chartType) {
-                    if (chartType === Visualization.prototype.TYPES.CHART) {
-                        return {
-                            'series': {
-                                'type': 'column',
-                                'stacking': null
-                            }
-                        };
-                    };
-
-                    return {};
-                }
-
-                var chartOptionsUnwatch = null;
-
-                scope.$watch('vis.type', function(type) {
-                    // if not edited by user, set name to match type
-                    if (type && scope.vis && !scope.visForm.name.$dirty) {
-                        // poor man's titlecase
-                        scope.vis.name = scope.vis.type[0] + scope.vis.type.slice(1).toLowerCase();
-                    }
-
-                    if (type && type == Visualization.prototype.TYPES.CHART) {
-                        if (scope.vis.options.series.stacking === null) {
-                            scope.stacking = "none";
-                        } else if (scope.vis.options.series.stacking === undefined) {
-                            scope.stacking = "normal";
-                        } else {
-                            scope.stacking = scope.vis.options.series.stacking ;
-                        }
-
-                        chartOptionsUnwatch = scope.$watch("stacking", function(stacking) {
-                            if (stacking == "none") {
-                                scope.vis.options.series.stacking = null;
-                            } else {
-                                scope.vis.options.series.stacking = stacking;
-                            }
-                        });
-                    } else {
-                        if (chartOptionsUnwatch) {
-                            chartOptionsUnwatch();
-                            chartOptionsUnwatch = null;
-                        }
-                    }
-                });
-
-                scope.toggleAdvancedMode = function() {
-                    scope.advancedMode = !scope.advancedMode;
-                };
-
-                scope.typeChanged = function() {
-                    scope.vis.options = newOptions(scope.vis.type);
-                };
-
-                scope.submit = function() {
-                    Visualization.save(scope.vis, function success(result) {
-                        growl.addSuccessMessage("Visualization saved");
-
-                        scope.vis = result;
-
-                        var visIds = _.pluck(scope.query.visualizations, 'id');
-                        var index = visIds.indexOf(result.id);
-                        if (index > -1) {
-                            scope.query.visualizations[index] = result;
-                        } else {
-                            scope.query.visualizations.push(result);
-                        }
-                    }, function error() {
-                        growl.addErrorMessage("Visualization could not be saved");
-                    });
-                };
-            }
-        }
-    }]);
-
-    directives.directive('editDashboardForm', ['$http', '$location', '$timeout', 'Dashboard', function($http, $location, $timeout, Dashboard) {
-        return {
-            restrict: 'E',
-            scope: {
-                dashboard: '='
-            },
-            templateUrl: '/views/edit_dashboard.html',
-            replace: true,
-            link: function($scope, element, attrs) {
-                var gridster = element.find(".gridster ul").gridster({
-                    widget_margins: [5, 5],
-                    widget_base_dimensions: [260, 100],
-                    min_cols: 2,
-                    max_cols: 2,
-                    serialize_params: function($w, wgd) {
-                        return {
-                            col: wgd.col,
-                            row: wgd.row,
-                            id: $w.data('widget-id')
-                        }
-                    }
-                }).data('gridster');
-
-                var gsItemTemplate = '<li data-widget-id="{id}" class="widget panel panel-default gs-w">' +
-                                 '<div class="panel-heading">{name}' +
-                                 '</div></li>';
-
-                $scope.$watch('dashboard.widgets', function(widgets) {
-                    $timeout(function () {
-                        gridster.remove_all_widgets();
-
-                        if (widgets && widgets.length) {
-                            var layout = [];
-
-                            _.each(widgets, function(row, rowIndex) {
-                                _.each(row, function(widget, colIndex) {
-                                    layout.push({
-                                        id: widget.id,
-                                        col: colIndex+1,
-                                        row: rowIndex+1,
-                                        ySize: 1,
-                                        xSize: widget.width,
-                                        name: widget.visualization.query.name
-                                    });
-                                });
-                            });
-
-                            _.each(layout, function(item) {
-                                var el = gsItemTemplate.replace('{id}', item.id).replace('{name}', item.name);
-                                gridster.add_widget(el, item.xSize, item.ySize, item.col, item.row);
-
-                            });
-                        }
-                    });
-                }, true);
-
-                $scope.saveDashboard = function() {
-                    $scope.saveInProgress = true;
-                    // TODO: we should use the dashboard service here.
-                    if ($scope.dashboard.id) {
-                        var positions = $(element).find('.gridster ul').data('gridster').serialize();
-                        var layout = [];
-                        _.each(_.sortBy(positions, function (pos) {
-                            return pos.row * 10 + pos.col;
-                        }), function (pos) {
-                            var row = pos.row - 1;
-                            var col = pos.col - 1;
-                            layout[row] = layout[row] || [];
-                            if (col > 0 && layout[row][col - 1] == undefined) {
-                                layout[row][col - 1] = pos.id;
-                            } else {
-                                layout[row][col] = pos.id;
-                            }
-
-                        });
-                        $scope.dashboard.layout = layout;
-
-                        layout = JSON.stringify(layout);
-                        $http.post('/api/dashboards/' + $scope.dashboard.id, {'name': $scope.dashboard.name, 'layout': layout}).success(function(response) {
-                            $scope.dashboard = new Dashboard(response);
-                            $scope.saveInProgress = false;
-                            $(element).modal('hide');
-                        })
-                    } else {
-                        $http.post('/api/dashboards', {'name': $scope.dashboard.name}).success(function(response) {
-                            $(element).modal('hide');
-                            $location.path('/dashboard/' + response.slug).replace();
-                        })
-                    }
-                }
-
-            }
-        }
-    }]);
-
-    directives.directive('newWidgetForm', ['$http', 'Query', function($http, Query) {
-        return {
-            restrict: 'E',
-            scope: {
-                dashboard: '='
-            },
-            templateUrl: '/views/new_widget_form.html',
-            replace: true,
-            link: function($scope, element, attrs) {
-                $scope.widgetSizes = [{name: 'Regular', value: 1}, {name: 'Double', value: 2}];
-
-                var reset = function() {
-                    $scope.saveInProgress = false;
-                    $scope.widgetSize = 1;
-                    $scope.queryId = null;
-                    $scope.selectedVis = null;
-
-                }
-
-                reset();
-
-                $scope.loadVisualizations = function() {
-                    if (!$scope.queryId) {
-                        return;
-                    }
-
-                    Query.get({
-                        id: $scope.queryId
-                    }, function(query) {
-                        if (query) {
-                            $scope.query = query;
-                            if(query.visualizations.length) {
-                                $scope.selectedVis = query.visualizations[0];
-                            }
-                        }
-                    });
-                };
-
-                $scope.saveWidget = function() {
-                    $scope.saveInProgress = true;
-
-                    var widget = {
-                        'visualization_id': $scope.selectedVis.id,
-                        'dashboard_id': $scope.dashboard.id,
-                        'options': {},
-                        'width': $scope.widgetSize
-                    }
-
-                    $http.post('/api/widgets', widget).success(function(response) {
-                        // update dashboard layout
-                        $scope.dashboard.layout = response['layout'];
-                        if (response['new_row']) {
-                            $scope.dashboard.widgets.push([response['widget']]);
-                        } else {
-                            $scope.dashboard.widgets[$scope.dashboard.widgets.length-1].push(response['widget']);
-                        }
-
-                        // close the dialog
-                        $('#add_query_dialog').modal('hide');
-                        reset();
-                    })
-                }
-
-            }
-        }
-    }])
-
-    // From: http://jsfiddle.net/joshdmiller/NDFHg/
-    directives.directive('editInPlace', function () {
-        return {
-            restrict: 'E',
-            scope: {
-                value: '=',
-                ignoreBlanks: '=',
-                editable: '='
-            },
-            template: function(tElement, tAttrs) {
-                var elType = tAttrs.editor || 'input';
-                var placeholder = tAttrs.placeholder || 'Click to edit';
-                return '<span ng-click="editable && edit()" ng-bind="value" ng-class="{editable: editable}"></span>' +
-                       '<span ng-click="editable && edit()" ng-show="editable && !value" ng-class="{editable: editable}">' + placeholder + '</span>' +
-                       '<{elType} ng-model="value" class="form-control" rows="2"></{elType}>'.replace('{elType}', elType);
-            },
-            link: function ($scope, element, attrs) {
-                // Let's get a reference to the input element, as we'll want to reference it.
-                var inputElement = angular.element(element.children()[2]);
-
-                // This directive should have a set class so we can style it.
-                element.addClass('edit-in-place');
-
-                // Initially, we're not editing.
-                $scope.editing = false;
-
-                // ng-click handler to activate edit-in-place
-                $scope.edit = function () {
-                    if ($scope.ignoreBlanks) {
-                        $scope.oldValue = $scope.value;
-                    }
-
-                    $scope.editing = true;
-
-                    // We control display through a class on the directive itself. See the CSS.
-                    element.addClass('active');
-
-                    // And we must focus the element.
-                    // `angular.element()` provides a chainable array, like jQuery so to access a native DOM function,
-                    // we have to reference the first element in the array.
-                    inputElement[0].focus();
-                };
-
-                $(inputElement).blur(function() {
-                    if ($scope.ignoreBlanks && _.isEmpty($scope.value)) {
-                        $scope.value = $scope.oldValue;
-                    }
-                    $scope.editing = false;
-                    element.removeClass('active');
-                })
-            }
-        };
-    });
-
-    // http://stackoverflow.com/a/17904092/1559840
-    directives.directive('jsonText', function() {
-        return {
-            restrict: 'A',
-            require: 'ngModel',
-            link: function(scope, element, attr, ngModel) {
-              function into(input) {
-                return JSON.parse(input);
-              }
-              function out(data) {
-                return JSON.stringify(data, undefined, 2);
-              }
-              ngModel.$parsers.push(into);
-              ngModel.$formatters.push(out);
-
-            }
-        };
-    });
-
-    directives.directive('rdTimer', ['$timeout', function ($timeout) {
-        return {
-            restrict: 'E',
-            scope: { timestamp: '=' },
-            template: '{{currentTime}}',
-            controller: ['$scope' ,function ($scope) {
-                $scope.currentTime = "00:00:00";
-                var currentTimeout = null;
-
-                var updateTime = function() {
-                    $scope.currentTime = moment(moment() - moment($scope.timestamp)).utc().format("HH:mm:ss")
-                    currentTimeout = $timeout(updateTime, 1000);
-                }
-
-                var cancelTimer = function() {
-                    if (currentTimeout) {
-                        $timeout.cancel(currentTimeout);
-                        currentTimeout = null;
-                    }
-                }
-
-                updateTime();
-
-                $scope.$on('$destroy', function () {
-                    cancelTimer();
-                });
-            }]
-        };
-    }]);
-})();

rd_ui/app/scripts/directives/dashboard_directives.js

@@ -0,0 +1,187 @@
+(function() {
+  'use strict'
+
+  var directives = angular.module('redash.directives');
+
+  directives.directive('editDashboardForm', ['$http', '$location', '$timeout', 'Dashboard',
+    function($http, $location, $timeout, Dashboard) {
+      return {
+        restrict: 'E',
+        scope: {
+          dashboard: '='
+        },
+        templateUrl: '/views/edit_dashboard.html',
+        replace: true,
+        link: function($scope, element, attrs) {
+          var gridster = element.find(".gridster ul").gridster({
+            widget_margins: [5, 5],
+            widget_base_dimensions: [260, 100],
+            min_cols: 2,
+            max_cols: 2,
+            serialize_params: function($w, wgd) {
+              return {
+                col: wgd.col,
+                row: wgd.row,
+                id: $w.data('widget-id')
+              }
+            }
+          }).data('gridster');
+
+          var gsItemTemplate = '<li data-widget-id="{id}" class="widget panel panel-default gs-w">' +
+            '<div class="panel-heading">{name}' +
+            '</div></li>';
+
+          $scope.$watch('dashboard.widgets && dashboard.widgets.length', function(widgets_length) {
+            $timeout(function() {
+              gridster.remove_all_widgets();
+
+              if ($scope.dashboard.widgets && $scope.dashboard.widgets.length) {
+                var layout = [];
+
+                _.each($scope.dashboard.widgets, function(row, rowIndex) {
+                  _.each(row, function(widget, colIndex) {
+                    layout.push({
+                      id: widget.id,
+                      col: colIndex + 1,
+                      row: rowIndex + 1,
+                      ySize: 1,
+                      xSize: widget.width,
+                      name: widget.visualization.query.name
+                    });
+                  });
+                });
+
+                _.each(layout, function(item) {
+                  var el = gsItemTemplate.replace('{id}', item.id).replace('{name}', item.name);
+                  gridster.add_widget(el, item.xSize, item.ySize, item.col, item.row);
+
+                });
+              }
+            });
+          });
+
+          $scope.saveDashboard = function() {
+            $scope.saveInProgress = true;
+            // TODO: we should use the dashboard service here.
+            if ($scope.dashboard.id) {
+              var positions = $(element).find('.gridster ul').data('gridster').serialize();
+              var layout = [];
+              _.each(_.sortBy(positions, function(pos) {
+                return pos.row * 10 + pos.col;
+              }), function(pos) {
+                var row = pos.row - 1;
+                var col = pos.col - 1;
+                layout[row] = layout[row] || [];
+                if (col > 0 && layout[row][col - 1] == undefined) {
+                  layout[row][col - 1] = pos.id;
+                } else {
+                  layout[row][col] = pos.id;
+                }
+
+              });
+              $scope.dashboard.layout = layout;
+
+              layout = JSON.stringify(layout);
+              $http.post('/api/dashboards/' + $scope.dashboard.id, {
+                'name': $scope.dashboard.name,
+                'layout': layout
+              }).success(function(response) {
+                $scope.dashboard = new Dashboard(response);
+                $scope.saveInProgress = false;
+                $(element).modal('hide');
+              })
+            } else {
+              $http.post('/api/dashboards', {
+                'name': $scope.dashboard.name
+              }).success(function(response) {
+                $(element).modal('hide');
+                $location.path('/dashboard/' + response.slug).replace();
+              })
+            }
+          }
+
+        }
+      }
+    }
+  ]);
+
+  directives.directive('newWidgetForm', ['Query', 'Widget', 'growl',
+    function(Query, Widget, growl) {
+      return {
+        restrict: 'E',
+        scope: {
+          dashboard: '='
+        },
+        templateUrl: '/views/new_widget_form.html',
+        replace: true,
+        link: function($scope, element, attrs) {
+          $scope.widgetSizes = [{
+            name: 'Regular',
+            value: 1
+          }, {
+            name: 'Double',
+            value: 2
+          }];
+
+          var reset = function() {
+            $scope.saveInProgress = false;
+            $scope.widgetSize = 1;
+            $scope.queryId = null;
+            $scope.selectedVis = null;
+            $scope.query = null;
+
+          }
+
+          reset();
+
+          $scope.loadVisualizations = function() {
+            if (!$scope.queryId) {
+              return;
+            }
+
+            Query.get({
+              id: $scope.queryId
+            }, function(query) {
+              if (query) {
+                $scope.query = query;
+                if (query.visualizations.length) {
+                  $scope.selectedVis = query.visualizations[0];
+                }
+              }
+            });
+          };
+
+          $scope.saveWidget = function() {
+            $scope.saveInProgress = true;
+
+            var widget = new Widget({
+              'visualization_id': $scope.selectedVis.id,
+              'dashboard_id': $scope.dashboard.id,
+              'options': {},
+              'width': $scope.widgetSize
+            });
+
+            widget.$save().then(function(response) {
+              // update dashboard layout
+              $scope.dashboard.layout = response['layout'];
+              if (response['new_row']) {
+                $scope.dashboard.widgets.push([response['widget']]);
+              } else {
+                $scope.dashboard.widgets[$scope.dashboard.widgets.length - 1].push(response['widget']);
+              }
+
+              // close the dialog
+              $('#add_query_dialog').modal('hide');
+              reset();
+            }).catch(function() {
+              growl.addErrorMessage("Widget can not be added");
+            }).finally(function() {
+              $scope.saveInProgress = false;
+            });
+          }
+
+        }
+      }
+    }
+  ])
+})();

rd_ui/app/scripts/directives/directives.js

@@ -0,0 +1,219 @@
+(function() {
+    'use strict';
+
+    var directives = angular.module('redash.directives', []);
+
+    directives.directive('alertUnsavedChanges', ['$window', function($window) {
+        return {
+            restrict: 'E',
+            replace: true,
+            scope: {
+                'isDirty': '='
+            },
+            link: function($scope) {
+                var
+
+                unloadMessage = "You will lose your changes if you leave",
+                confirmMessage = unloadMessage + "\n\nAre you sure you want to leave this page?",
+
+                // store original handler (if any)
+                _onbeforeunload = $window.onbeforeunload;
+
+                $window.onbeforeunload = function() {
+                    return $scope.isDirty ? unloadMessage : null;
+                }
+
+                $scope.$on('$locationChangeStart', function(event, next, current) {
+                  if (next.split("#")[0] == current.split("#")[0]) {
+                    return;
+                  }
+
+                  if ($scope.isDirty && !confirm(confirmMessage)) {
+                    event.preventDefault();
+                  }
+                });
+
+                $scope.$on('$destroy', function() {
+                    $window.onbeforeunload = _onbeforeunload;
+                });
+            }
+        }
+    }]);
+
+    directives.directive('rdTab', function() {
+        return {
+            restrict: 'E',
+            scope: {
+                'tabId': '@',
+                'name': '@'
+            },
+            transclude: true,
+            template: '<li class="rd-tab" ng-class="{active: tabId==selectedTab}"><a href="#{{tabId}}">{{name}}<span ng-transclude></span></a></li>',
+            replace: true,
+            link: function(scope) {
+                scope.$watch(function(){return scope.$parent.selectedTab}, function(tab) {
+                    scope.selectedTab = tab;
+                });
+            }
+        }
+    });
+
+    directives.directive('rdTabs', ['$location', function($location) {
+        return {
+            restrict: 'E',
+            scope: {
+                tabsCollection: '=',
+                selectedTab: '='
+            },
+            template: '<ul class="nav nav-tabs"><li ng-class="{active: tab==selectedTab}" ng-repeat="tab in tabsCollection"><a href="#{{tab.key}}">{{tab.name}}</a></li></ul>',
+            replace: true,
+            link: function($scope, element, attrs) {
+                $scope.selectTab = function(tabKey) {
+                    $scope.selectedTab  = _.find($scope.tabsCollection, function(tab) { return tab.key == tabKey; });
+                }
+
+                $scope.$watch(function() { return $location.hash()}, function(hash) {
+                    if (hash) {
+                        $scope.selectTab($location.hash());
+                    } else {
+                        $scope.selectTab($scope.tabsCollection[0].key);
+                    }
+                });
+            }
+        }
+    }]);
+
+    // From: http://jsfiddle.net/joshdmiller/NDFHg/
+    directives.directive('editInPlace', function () {
+        return {
+            restrict: 'E',
+            scope: {
+                value: '=',
+                ignoreBlanks: '=',
+                editable: '=',
+                done: '='
+            },
+            template: function(tElement, tAttrs) {
+                var elType = tAttrs.editor || 'input';
+                var placeholder = tAttrs.placeholder || 'Click to edit';
+                return '<span ng-click="editable && edit()" ng-bind="value" ng-class="{editable: editable}"></span>' +
+                       '<span ng-click="editable && edit()" ng-show="editable && !value" ng-class="{editable: editable}">' + placeholder + '</span>' +
+                       '<{elType} ng-model="value" class="rd-form-control"></{elType}>'.replace('{elType}', elType);
+            },
+            link: function ($scope, element, attrs) {
+                // Let's get a reference to the input element, as we'll want to reference it.
+                var inputElement = angular.element(element.children()[2]);
+
+                // This directive should have a set class so we can style it.
+                element.addClass('edit-in-place');
+
+                // Initially, we're not editing.
+                $scope.editing = false;
+
+                // ng-click handler to activate edit-in-place
+                $scope.edit = function () {
+                    $scope.oldValue = $scope.value;
+
+                    $scope.editing = true;
+
+                    // We control display through a class on the directive itself. See the CSS.
+                    element.addClass('active');
+
+                    // And we must focus the element.
+                    // `angular.element()` provides a chainable array, like jQuery so to access a native DOM function,
+                    // we have to reference the first element in the array.
+                    inputElement[0].focus();
+                };
+
+                function save() {
+                    if ($scope.editing) {
+                        if ($scope.ignoreBlanks && _.isEmpty($scope.value)) {
+                            $scope.value = $scope.oldValue;
+                        }
+                        $scope.editing = false;
+                        element.removeClass('active');
+
+                        if ($scope.value !== $scope.oldValue) {
+                            $scope.done && $scope.done();
+                        }
+                    }
+                }
+
+                $(inputElement).keydown(function(e) {
+                    // 'return' or 'enter' key pressed
+                    // allow 'shift' to break lines
+                    if (e.which === 13 && !e.shiftKey) {
+                        save();
+                    } else if (e.which === 27) {
+                        $scope.value = $scope.oldValue;
+                        $scope.$apply(function() {
+                            $(inputElement[0]).blur();
+                        });
+                    }
+                }).blur(function() {
+                    save();
+                });
+            }
+        };
+    });
+
+    // http://stackoverflow.com/a/17904092/1559840
+    directives.directive('jsonText', function() {
+        return {
+            restrict: 'A',
+            require: 'ngModel',
+            link: function(scope, element, attr, ngModel) {
+              function into(input) {
+                return JSON.parse(input);
+              }
+              function out(data) {
+                return JSON.stringify(data, undefined, 2);
+              }
+              ngModel.$parsers.push(into);
+              ngModel.$formatters.push(out);
+
+              scope.$watch(attr.ngModel, function(newValue) {
+                element[0].value = out(newValue);
+              }, true);
+            }
+        };
+    });
+
+    directives.directive('rdTimer', [function () {
+        return {
+            restrict: 'E',
+            scope: { timestamp: '=' },
+            template: '{{currentTime}}',
+            controller: ['$scope' ,function ($scope) {
+                $scope.currentTime = "00:00:00";
+
+                // We're using setInterval directly instead of $timeout, to avoid using $apply, to
+                // prevent the digest loop being run every second.
+                var currentTimer = setInterval(function() {
+                    $scope.currentTime = moment(moment() - moment($scope.timestamp)).utc().format("HH:mm:ss");
+                    $scope.$digest();
+                }, 1000);
+
+                $scope.$on('$destroy', function () {
+                    if (currentTimer) {
+                        clearInterval(currentTimer);
+                        currentTimer = null;
+                    }
+                });
+            }]
+        };
+    }]);
+
+    directives.directive('rdTimeAgo', function() {
+        return {
+            restrict: 'E',
+            scope: {
+                value: '='
+            },
+            template: '<span>' +
+                        '<span ng-show="value" am-time-ago="value"></span>' +
+                        '<span ng-hide="value">-</span>' +
+                      '</span>'
+        }
+    });
+})();

rd_ui/app/scripts/directives/query_directives.js

@@ -0,0 +1,141 @@
+(function() {
+  'use strict'
+
+  function queryLink() {
+    return {
+      restrict: 'E',
+      scope: {
+        'query': '=',
+        'visualization': '=?'
+      },
+      template: '<a ng-href="{{link}}" class="query-link">{{query.name}}</a>',
+      link: function(scope, element) {
+        scope.link = '/queries/' + scope.query.id;
+        if (scope.visualization) {
+          if (scope.visualization.type === 'TABLE') {
+            // link to hard-coded table tab instead of the (hidden) visualization tab
+            scope.link += '#table';
+          } else {
+            scope.link += '#' + scope.visualization.id;
+          }
+        }
+        // element.find('a').attr('href', link);
+      }
+    }
+  }
+
+  function querySourceLink() {
+    return {
+      restrict: 'E',
+      template: '<span ng-show="query.id && canViewSource">\
+                    <a ng-show="!sourceMode"\
+                      ng-href="{{query.id}}/source#{{selectedTab}}">Show Source\
+                    </a>\
+                    <a ng-show="sourceMode"\
+                      ng-href="/queries/{{query.id}}#{{selectedTab}}">Hide Source\
+                    </a>\
+                </span>'
+    }
+  }
+
+  function queryEditor() {
+    return {
+      restrict: 'E',
+      scope: {
+        'query': '=',
+        'lock': '='
+      },
+      template: '<textarea\
+                  ui-codemirror="editorOptions"\
+                  ng-model="query.query">',
+      link: function($scope) {
+        $scope.editorOptions = {
+            mode: 'text/x-sql',
+            lineWrapping: true,
+            lineNumbers: true,
+            readOnly: false,
+            matchBrackets: true,
+            autoCloseBrackets: true
+        };
+
+        $scope.$watch('lock', function(locked) {
+          $scope.editorOptions.readOnly = locked ? 'nocursor' : false;
+        });
+      }
+    }
+  }
+
+  function queryFormatter($http) {
+    return {
+      restrict: 'E',
+      // don't create new scope to avoid ui-codemirror bug
+      // seehttps://github.com/angular-ui/ui-codemirror/pull/37
+      scope: false,
+      template: '<button type="button" class="btn btn-default btn-xs"\
+                   ng-click="formatQuery()">\
+                    <span class="glyphicon glyphicon-indent-left"></span>\
+                     Format SQL\
+                </button>',
+      link: function($scope) {
+        $scope.formatQuery = function formatQuery() {
+            $scope.queryExecuting = true;
+            $http.post('/api/queries/format', {
+                'query': $scope.query.query
+            }).success(function (response) {
+                $scope.query.query = response;
+            }).finally(function () {
+              $scope.queryExecuting = false;
+            });
+        };
+      }
+    }
+  }
+
+  function queryRefreshSelect() {
+    return {
+      restrict: 'E',
+      template: '<select\
+                  ng-disabled="!isQueryOwner"\
+                  ng-model="query.ttl"\
+                  ng-change="saveQuery()"\
+                  ng-options="c.value as c.name for c in refreshOptions">\
+                  </select>',
+      link: function($scope) {
+        $scope.refreshOptions = [
+            {
+                value: -1,
+                name: 'No Refresh'
+            },
+            {
+                value: 60,
+                name: 'Every minute'
+            },
+        ]
+
+        _.each(_.range(1, 13), function (i) {
+            $scope.refreshOptions.push({
+                value: i * 3600,
+                name: 'Every ' + i + 'h'
+            });
+        })
+
+        $scope.refreshOptions.push({
+            value: 24 * 3600,
+            name: 'Every 24h'
+        });
+        $scope.refreshOptions.push({
+            value: 7 * 24 * 3600,
+            name: 'Once a week'
+        });
+      }
+
+    }
+  }
+
+  angular.module('redash.directives')
+  .directive('queryLink', queryLink)
+  .directive('querySourceLink', querySourceLink)
+  .directive('queryEditor', queryEditor)
+  .directive('queryRefreshSelect', queryRefreshSelect)
+  .directive('queryFormatter', ['$http', queryFormatter]);
+})();

rd_ui/app/scripts/filters.js

@@ -47,4 +47,15 @@ angular.module('redash.filters', []).
             }
             return 12;
         }
+    })
+    
+    .filter('capitalize', function () {
+        return function (text) {
+            if (text) {
+                return text[0].toUpperCase() + text.slice(1).toLowerCase();
+            } else {
+                return null;
+            }
+            
+        }
     });

rd_ui/app/scripts/ng_highchart.js

@@ -1,6 +1,11 @@
 (function () {
     'use strict';
 
+    Highcharts.setOptions({
+        colors: ["#4572A7", "#AA4643", "#89A54E", "#80699B", "#3D96AE",
+            "#DB843D", "#92A8CD", "#A47D7C", "#B5CA92"]
+    });
+
     var defaultOptions = {
         title: {
             "text": null
@@ -124,7 +129,7 @@
                     enabled: true,
                     color: '#000000',
                     connectorColor: '#000000',
-                    format: '<b>{point.name}</b>: {point.percentage:.1f} %'
+                    format: '<b>{point.name}</b>: {point.y} ({point.percentage:.1f} %)'
                 }
             },
             scatter: {
@@ -181,17 +186,13 @@
                         }, true);
 
                         //Update when charts data changes
-                        scope.$watch(function () {
-                            // TODO: this might be an issue in case the series change, but they stay
-                            // with the same length
-                            return (scope.series && scope.series.length) || 0;
-                        }, function (length) {
-                            if (!length || length == 0) {
+                        scope.$watchCollection('series', function (series) {
+                            if (!series || series.length == 0) {
                                 scope.chart.showLoading();
                             } else {
                                 drawChart();
                             };
-                        }, true);
+                        });
                     });
 
                     function initChart(options) {
@@ -210,8 +211,7 @@
                             scope.chart.series[0].remove(false);
                         };
 
-
-                        if (_.some(scope.series[0].data, function (p) {
+                        if (scope.series.length > 0 && _.some(scope.series[0].data, function (p) {
                             return (angular.isString(p.x) || angular.isDefined(p.name));
                         })) {
                             scope.chart.xAxis[0].update({type: 'category'});
@@ -226,12 +226,16 @@
                                     // TODO: move this logic to Query#getChartData
                                     var yValues = _.groupBy(s.data, 'x');
 
-                                    var newData = _.sortBy(_.map(categories, function (category) {
+                                    var newData = _.map(categories, function (category) {
                                         return {
                                             name: category,
-                                            y: yValues[category] && yValues[category][0].y
+                                            y: (yValues[category] && yValues[category][0].y) || 0
                                         }
-                                    }), 'y').reverse();
+                                    });
+
+                                    if (categories.length == 1) {
+                                        newData = _.sortBy(newData, 'y').reverse();
+                                    };
 
                                     s.data = newData;
                                 });

rd_ui/app/scripts/query_fiddle/renderers.js

@@ -1,203 +0,0 @@
-var renderers = angular.module('redash.renderers', []);
-
-renderers.directive('visualizationRenderer', function() {
-    return {
-        restrict: 'E',
-        scope: {
-            visualization: '=',
-            queryResult: '='
-        },
-        template: '<div ng-switch on="visualization.type">' +
-                    '<grid-renderer ng-switch-when="TABLE" options="visualization.options" query-result="queryResult"></grid-renderer>' +
-                    '<chart-renderer ng-switch-when="CHART" options="visualization.options" query-result="queryResult"></chart-renderer>' +
-                    '<cohort-renderer ng-switch-when="COHORT" options="visualization.options" query-result="queryResult"></cohort-renderer>' +
-                   '</div>',
-        replace: false
-    }
-});
-
-renderers.directive('chartRenderer', function () {
-    return {
-        restrict: 'E',
-        scope: {
-            queryResult: '=',
-            options: '=?'
-        },
-        template: "<chart options='chartOptions' series='chartSeries' class='graph'></chart>",
-        replace: false,
-        controller: ['$scope', function ($scope) {
-            $scope.chartSeries = [];
-            $scope.chartOptions = {};
-
-            $scope.$watch('options', function(chartOptions) {
-                if (chartOptions) {
-                    $scope.chartOptions = chartOptions;
-                }
-            });
-            $scope.$watch('queryResult && queryResult.getData()', function (data) {
-                if (!data || $scope.queryResult.getData() == null) {
-                    $scope.chartSeries.splice(0, $scope.chartSeries.length);
-                } else {
-                    $scope.chartSeries.splice(0, $scope.chartSeries.length);
-
-                    _.each($scope.queryResult.getChartData(), function (s) {
-                        $scope.chartSeries.push(_.extend(s, {'stacking': 'normal'}));
-                    });
-                }
-            });
-        }]
-    }
-})
-
-renderers.directive('gridRenderer', function () {
-    return {
-        restrict: 'E',
-        scope: {
-            queryResult: '=',
-            itemsPerPage: '='
-        },
-        templateUrl: "/views/grid_renderer.html",
-        replace: false,
-        controller: ['$scope', function ($scope) {
-            $scope.gridColumns = [];
-            $scope.gridData = [];
-            $scope.gridConfig = {
-                isPaginationEnabled: true,
-                itemsByPage: $scope.itemsPerPage || 15,
-                maxSize: 8
-            };
-
-            $scope.$watch('queryResult && queryResult.getData()', function (data) {
-                if (!data) {
-                    return;
-                }
-
-                if ($scope.queryResult.getData() == null) {
-                    $scope.gridColumns = [];
-                    $scope.gridData = [];
-                    $scope.filters = [];
-                } else {
-
-
-                    $scope.filters = $scope.queryResult.getFilters();
-
-                    var gridData = _.map($scope.queryResult.getData(), function (row) {
-                        var newRow = {};
-                        _.each(row, function (val, key) {
-                            newRow[$scope.queryResult.getColumnCleanName(key)] = val;
-                        })
-                        return newRow;
-                    });
-
-                    $scope.gridColumns = _.map($scope.queryResult.getColumnCleanNames(), function (col, i) {
-                        var columnDefinition = {
-                            'label': $scope.queryResult.getColumnFriendlyNames()[i],
-                            'map': col
-                        };
-
-                        if (gridData.length > 0) {
-                            var exampleData = gridData[0][col];
-                            if (angular.isNumber(exampleData)) {
-                                columnDefinition['formatFunction'] = 'number';
-                                columnDefinition['formatParameter'] = 2;
-                            } else if (moment.isMoment(exampleData)) {
-                                columnDefinition['formatFunction'] = function(value) {
-                                    return value.format("DD/MM/YY HH:mm");
-                                }
-                            }
-                        }
-
-                        return columnDefinition;
-                    });
-
-                    $scope.gridData = _.clone(gridData);
-
-                    $scope.$watch('filters', function (filters) {
-                        $scope.gridData = _.filter(gridData, function (row) {
-                            return _.reduce(filters, function (memo, filter) {
-                                if (filter.current == 'All') {
-                                    return memo && true;
-                                }
-
-                                return (memo && row[$scope.queryResult.getColumnCleanName(filter.name)] == filter.current);
-                            }, true);
-                        });
-                    }, true);
-                }
-            });
-        }]
-    }
-})
-
-renderers.directive('pivotTableRenderer', function () {
-    return {
-        restrict: 'E',
-        scope: {
-            queryResult: '='
-        },
-        template: "",
-        replace: false,
-        link: function($scope, element, attrs) {
-            $scope.$watch('queryResult && queryResult.getData()', function (data) {
-                if (!data) {
-                    return;
-                }
-
-                if ($scope.queryResult.getData() == null) {
-                } else {
-                    $(element).pivotUI($scope.queryResult.getData(), {
-                         renderers: $.pivotUtilities.renderers
-                    }, true);
-                }
-            });
-        }
-    }
-})
-
-renderers.directive('cohortRenderer', function() {
-    return {
-        restrict: 'E',
-        scope: {
-            queryResult: '='
-        },
-        template: "",
-        replace: false,
-        link: function($scope, element, attrs) {
-            $scope.$watch('queryResult && queryResult.getData()', function (data) {
-                if (!data) {
-                    return;
-                }
-
-                if ($scope.queryResult.getData() == null) {
-
-                } else {
-                    var sortedData = _.sortBy($scope.queryResult.getData(), "date");
-                    var grouped = _.groupBy(sortedData, "date");
-                    var data = _.map(grouped, function(values, date) {
-                       var row = [values[0].total];
-                        _.each(values, function(value) { row.push(value.value); });
-                        return row;
-                    });
-
-                    var initialDate = moment(sortedData[0].date).toDate(),
-                        container = angular.element(element)[0];
-
-                    Cornelius.draw({
-                        initialDate: initialDate,
-                        container: container,
-                        cohort: data,
-                        title: null,
-                        timeInterval: 'daily',
-                        labels: {
-                            time: 'Activation Day',
-                            people: 'Users'
-                        },
-                        formatHeaderLabel: function (i) {
-                            return "Day " + (i - 1);
-                        }
-                    });
-                }
-            });
-        }
-    }
-})

rd_ui/app/scripts/services/dashboards.js

@@ -2,7 +2,7 @@
     var Dashboard = function($resource) {
         var resource = $resource('/api/dashboards/:slug', {slug: '@slug'});
         resource.prototype.canEdit = function() {
-            return currentUser.is_admin || currentUser.canEdit(this);
+            return currentUser.hasPermission('admin') || currentUser.canEdit(this);
         }
         return resource;
     }

rd_ui/app/scripts/services/resources.js

@@ -7,6 +7,8 @@
             angular.extend(this, props);
             if ('query_result' in props) {
                 this.status = "done";
+                this.filters = undefined;
+                this.filterFreeze = undefined;
 
                 _.each(this.query_result.data.rows, function (row) {
                     _.each(row, function (v, k) {
@@ -26,6 +28,8 @@
             this.job = {};
             this.query_result = {};
             this.status = "waiting";
+            this.filters = undefined;
+            this.filterFreeze = undefined;
 
             this.updatedAt = moment();
 
@@ -76,15 +80,51 @@
             return this.query_result.runtime;
         }
 
-        QueryResult.prototype.getData = function() {
+        QueryResult.prototype.getRawData = function() {
             if (!this.query_result.data) {
                 return null;
             }
 
             var data = this.query_result.data.rows;
+
             return data;
         }
 
+        QueryResult.prototype.getData = function() {
+            if (!this.query_result.data) {
+                return null;
+            }
+
+            var filterValues = function(filters) {
+                if (!filters) {
+                    return null;
+                }
+
+                return _.reduce(filters, function(str, filter) {
+                    return str + filter.current;
+                }, "")
+            }
+
+            var filters = this.getFilters();
+            var filterFreeze = filterValues(filters);
+
+            if (this.filterFreeze != filterFreeze) {
+                this.filterFreeze = filterFreeze;
+
+                if (filters) {
+                    this.filteredData = _.filter(this.query_result.data.rows, function (row) {
+                        return _.reduce(filters, function (memo, filter) {
+                            return (memo && row[filter.name] == filter.current);
+                        }, true);
+                    });
+                } else {
+                    this.filteredData = this.query_result.data.rows;
+                }
+            }
+
+            return this.filteredData;
+        }
+
         QueryResult.prototype.getChartData = function () {
             var series = {};
 
@@ -142,10 +182,10 @@
         };
 
         QueryResult.prototype.getColumns = function () {
-            if (this.columns == undefined) {
+            if (this.columns == undefined && this.query_result.data) {
                 this.columns = _.map(this.query_result.data.columns, function(v) {
                     return v.name;
-                })
+                });
             }
 
             return this.columns;
@@ -181,6 +221,14 @@
         }
 
         QueryResult.prototype.getFilters = function () {
+            if (!this.filters) {
+                this.prepareFilters();
+            }
+
+            return this.filters;
+        };
+
+        QueryResult.prototype.prepareFilters = function() {
             var filterNames = [];
             _.each(this.getColumns(), function (col) {
                 if (col.split('::')[1] == 'filter') {
@@ -189,7 +237,7 @@
             });
 
             var filterValues = [];
-            _.each(this.getData(), function (row) {
+            _.each(this.getRawData(), function (row) {
                 _.each(filterNames, function (filter, i) {
                     if (filterValues[i] == undefined) {
                         filterValues[i] = [];
@@ -198,7 +246,7 @@
                 })
             });
 
-            var filters = _.map(filterNames, function (filter, i) {
+            this.filters = _.map(filterNames, function (filter, i) {
                 var f = {
                     name: filter,
                     friendlyName: this.getColumnFriendlyName(filter),
@@ -208,9 +256,7 @@
                 f.current = f.values[0];
                 return f;
             }, this);
-
-            return filters;
-        };
+        }
 
         var refreshStatus = function(queryResult, query, ttl) {
             Job.get({'id': queryResult.job.id}, function(response) {
@@ -238,10 +284,10 @@
             return queryResult;
         }
 
-        QueryResult.get = function (query, ttl) {
+        QueryResult.get = function (data_source_id, query, ttl) {
             var queryResult = new QueryResult();
 
-            QueryResultResource.post({'query': query, 'ttl': ttl}, function (response) {
+            QueryResultResource.post({'data_source_id': data_source_id, 'query': query, 'ttl': ttl}, function (response) {
                 queryResult.update(response);
 
                 if ('job' in response) {
@@ -255,51 +301,57 @@
         return QueryResult;
     };
 
-    var Query = function ($resource, QueryResult) {
+    var Query = function ($resource, QueryResult, DataSource) {
         var Query = $resource('/api/queries/:id', {id: '@id'});
 
+        Query.newQuery = function() {
+            return new Query({
+                query: "",
+                name: "New Query",
+                ttl: -1,
+                user: currentUser
+            });
+        };
+
+        Query.prototype.getSourceLink = function() {
+            return '/queries/' + this.id + '/source';
+        };
+
         Query.prototype.getQueryResult = function(ttl) {
             if (ttl == undefined) {
                 ttl = this.ttl;
             }
 
-
             var queryResult = null;
             if (this.latest_query_data && ttl != 0) {
                 queryResult = new QueryResult({'query_result': this.latest_query_data});
             } else if (this.latest_query_data_id && ttl != 0) {
                 queryResult = QueryResult.getById(this.latest_query_data_id);
-            } else {
-                queryResult = QueryResult.get(this.query, ttl);
+            } else if (this.data_source_id) {
+                queryResult = QueryResult.get(this.data_source_id, this.query, ttl);
             }
 
             return queryResult;
         };
 
-        Query.prototype.getHash = function() {
-            return [this.name, this.description, this.query].join('!#');
-        };
-
         return Query;
     };
 
-    var Visualization = function($resource) {
-        var Visualization = $resource('/api/visualizations/:id', {id: '@id'});
+    var DataSource = function($resource) {
+        var DataSourceResource = $resource('/api/data_sources/:id', {id: '@id'}, {'get': {'method': 'GET', 'cache': true, 'isArray': true}});
 
-        Visualization.prototype = {
-            TYPES: {
-                'CHART': 'CHART',
-                'COHORT': 'COHORT',
-                'TABLE': 'TABLE'
-            }
-        };
+        return DataSourceResource;
+    }
 
-        return Visualization;
-    };
+    var Widget = function($resource) {
+        var WidgetResource = $resource('/api/widgets/:id', {id: '@id'});
 
-    angular.module('redash.services', [])
+        return WidgetResource;
+    }
+
+    angular.module('redash.services')
         .factory('QueryResult', ['$resource', '$timeout', QueryResult])
-        .factory('Query', ['$resource', 'QueryResult', Query])
-        .factory('Visualization', ['$resource', Visualization])
-
+        .factory('Query', ['$resource', 'QueryResult', 'DataSource', Query])
+        .factory('DataSource', ['$resource', DataSource])
+        .factory('Widget', ['$resource', Widget]);
 })();

rd_ui/app/scripts/services/services.js

@@ -0,0 +1,24 @@
+(function() {
+  'use strict'
+
+  function KeyboardShortcuts() {
+    this.bind = function bind(keymap) {
+      _.forEach(keymap, function(fn, key) {
+        Mousetrap.bindGlobal(key, function(e) {
+            e.preventDefault();
+            fn();
+        });
+      });
+
+    }
+
+    this.unbind = function unbind(keymap) {
+      _.forEach(keymap, function(fn, key) {
+        Mousetrap.unbind(key);
+      });
+    }
+  }
+
+  angular.module('redash.services', [])
+    .service('KeyboardShortcuts', [KeyboardShortcuts])
+})();

rd_ui/app/scripts/visualizations/base.js

@@ -0,0 +1,170 @@
+(function () {
+    var VisualizationProvider = function() {
+        this.visualizations = {};
+        this.visualizationTypes = {};
+        var defaultConfig = {
+            defaultOptions: {},
+            skipTypes: false,
+            editorTemplate: null
+        }
+
+        this.registerVisualization = function(config) {
+            var visualization = _.extend({}, defaultConfig, config);
+
+            // TODO: this is prone to errors; better refactor.
+            if (_.isEmpty(this.visualizations)) {
+                this.defaultVisualization = visualization;
+            }
+
+            this.visualizations[config.type] = visualization;
+
+            if (!config.skipTypes) {
+                this.visualizationTypes[config.name] = config.type;
+            };
+        };
+
+        this.getSwitchTemplate = function(property) {
+            var pattern = /(<[a-zA-Z0-9-]*?)( |>)/
+
+            var mergedTemplates = _.reduce(this.visualizations, function(templates, visualization) {
+                if (visualization[property]) {
+                    var ngSwitch = '$1 ng-switch-when="' + visualization.type + '" $2';
+                    var template = visualization[property].replace(pattern, ngSwitch);
+
+                    return templates + "\n" + template;
+                }
+
+                return templates;
+            }, "");
+
+            mergedTemplates = '<div ng-switch on="visualization.type">'+ mergedTemplates + "</div>";
+
+            return mergedTemplates;
+        }
+
+        this.$get = ['$resource', function($resource) {
+            var Visualization = $resource('/api/visualizations/:id', {id: '@id'});
+            Visualization.visualizations = this.visualizations;
+            Visualization.visualizationTypes = this.visualizationTypes;
+            Visualization.renderVisualizationsTemplate = this.getSwitchTemplate('renderTemplate');
+            Visualization.editorTemplate = this.getSwitchTemplate('editorTemplate');
+            Visualization.defaultVisualization = this.defaultVisualization;
+
+            return Visualization;
+        }];
+    };
+
+    var VisualizationRenderer = function(Visualization) {
+        return {
+            restrict: 'E',
+            scope: {
+                visualization: '=',
+                queryResult: '='
+            },
+            // TODO: using switch here (and in the options editor) might introduce errors and bad
+            // performance wise. It's better to eventually show the correct template based on the
+            // visualization type and not make the browser render all of them.
+            template: '<filters></filters>\n' + Visualization.renderVisualizationsTemplate,
+            replace: false,
+            link: function(scope) {
+                scope.$watch('queryResult && queryResult.getFilters()', function(filters) {
+                    if (filters) {
+                        scope.filters = filters;
+                    }
+                });
+            }
+        }
+    };
+
+    var VisualizationOptionsEditor = function(Visualization) {
+        return {
+            restrict: 'E',
+            template: Visualization.editorTemplate,
+            replace: false
+        }
+    };
+
+    var Filters = function() {
+        return {
+            restrict: 'E',
+            templateUrl: '/views/visualizations/filters.html'
+        }
+    }
+
+    var EditVisualizationForm = function(Visualization, growl) {
+        return {
+            restrict: 'E',
+            templateUrl: '/views/visualizations/edit_visualization.html',
+            replace: true,
+            scope: {
+                query: '=',
+                queryResult: '=',
+                visualization: '=?',
+                onNewSuccess: '=?'
+            },
+            link: function (scope, element, attrs) {
+                scope.editRawOptions = currentUser.hasPermission('edit_raw_chart');
+                scope.visTypes = Visualization.visualizationTypes;
+
+                scope.newVisualization = function(q) {
+                    return {
+                        'query_id': q.id,
+                        'type': Visualization.defaultVisualization.type,
+                        'name': Visualization.defaultVisualization.name,
+                        'description': q.description || '',
+                        'options': Visualization.defaultVisualization.defaultOptions
+                    };
+                }
+
+                if (!scope.visualization) {
+                    // create new visualization
+                    // wait for query to load to populate with defaults
+                    var unwatch = scope.$watch('query', function (q) {
+                        if (q && q.id) {
+                            unwatch();
+
+                            scope.visualization = scope.newVisualization(q);
+                        }
+                    }, true);
+                }
+
+                scope.$watch('visualization.type', function (type, oldType) {
+                    // if not edited by user, set name to match type
+                    if (type && oldType != type && scope.visualization && !scope.visForm.name.$dirty) {
+                        // poor man's titlecase
+                        scope.visualization.name = scope.visualization.type[0] + scope.visualization.type.slice(1).toLowerCase();
+                    }
+                });
+
+                scope.submit = function () {
+                    Visualization.save(scope.visualization, function success(result) {
+                        growl.addSuccessMessage("Visualization saved");
+
+                        scope.visualization = scope.newVisualization(scope.query);
+
+                        var visIds = _.pluck(scope.query.visualizations, 'id');
+                        var index = visIds.indexOf(result.id);
+                        if (index > -1) {
+                            scope.query.visualizations[index] = result;
+                        } else {
+                            // new visualization
+                            scope.query.visualizations.push(result);
+                            scope.onNewSuccess && scope.onNewSuccess(result);
+                        }
+                    }, function error() {
+                        growl.addErrorMessage("Visualization could not be saved");
+                    });
+                };
+            }
+        }
+    };
+
+
+
+    angular.module('redash.visualization', [])
+        .provider('Visualization', VisualizationProvider)
+        .directive('visualizationRenderer', ['Visualization', VisualizationRenderer])
+        .directive('visualizationOptionsEditor', ['Visualization', VisualizationOptionsEditor])
+        .directive('filters', Filters)
+        .directive('editVisulatizationForm', ['Visualization', 'growl', EditVisualizationForm])
+})();

rd_ui/app/scripts/visualizations/chart.js

@@ -0,0 +1,106 @@
+(function () {
+    var chartVisualization = angular.module('redash.visualization');
+
+    chartVisualization.config(['VisualizationProvider', function(VisualizationProvider) {
+        var renderTemplate = '<chart-renderer options="visualization.options" query-result="queryResult"></chart-renderer>';
+        var editTemplate = '<chart-editor></chart-editor>';
+        var defaultOptions = {
+            'series': {
+                'type': 'column',
+                'stacking': null
+            }
+        };
+
+        VisualizationProvider.registerVisualization({
+            type: 'CHART',
+            name: 'Chart',
+            renderTemplate: renderTemplate,
+            editorTemplate: editTemplate,
+            defaultOptions: defaultOptions
+        });
+    }]);
+
+    chartVisualization.directive('chartRenderer', function () {
+        return {
+            restrict: 'E',
+            scope: {
+                queryResult: '=',
+                options: '=?'
+            },
+            template: "<chart options='chartOptions' series='chartSeries' class='graph'></chart>",
+            replace: false,
+            controller: ['$scope', function ($scope) {
+                $scope.chartSeries = [];
+                $scope.chartOptions = {};
+
+                $scope.$watch('options', function(chartOptions) {
+                    if (chartOptions) {
+                        $scope.chartOptions = chartOptions;
+                    }
+                });
+                $scope.$watch('queryResult && queryResult.getData()', function (data) {
+                    if (!data || $scope.queryResult.getData() == null) {
+                        $scope.chartSeries.splice(0, $scope.chartSeries.length);
+                    } else {
+                        $scope.chartSeries.splice(0, $scope.chartSeries.length);
+
+                        _.each($scope.queryResult.getChartData(), function (s) {
+                            $scope.chartSeries.push(_.extend(s, {'stacking': 'normal'}));
+                        });
+                    }
+                });
+            }]
+        }
+    });
+
+    chartVisualization.directive('chartEditor', function () {
+        return {
+            restrict: 'E',
+            templateUrl: '/views/visualizations/chart_editor.html',
+            link: function (scope, element, attrs) {
+                scope.seriesTypes = {
+                    'Line': 'line',
+                    'Column': 'column',
+                    'Area': 'area',
+                    'Scatter': 'scatter',
+                    'Pie': 'pie'
+                };
+
+                scope.stackingOptions = {
+                    "None": "none",
+                    "Normal": "normal",
+                    "Percent": "percent"
+                };
+
+                scope.stacking = "none";
+
+                var chartOptionsUnwatch = null;
+
+                scope.$watch('visualization', function (visualization) {
+                    if (visualization && visualization.type == 'CHART') {
+                        if (scope.visualization.options.series.stacking === null) {
+                            scope.stacking = "none";
+                        } else if (scope.visualization.options.series.stacking === undefined) {
+                            scope.stacking = "normal";
+                        } else {
+                            scope.stacking = scope.visualization.options.series.stacking;
+                        }
+
+                        chartOptionsUnwatch = scope.$watch("stacking", function (stacking) {
+                            if (stacking == "none") {
+                                scope.visualization.options.series.stacking = null;
+                            } else {
+                                scope.visualization.options.series.stacking = stacking;
+                            }
+                        });
+                    } else {
+                        if (chartOptionsUnwatch) {
+                            chartOptionsUnwatch();
+                            chartOptionsUnwatch = null;
+                        }
+                    }
+                });
+            }
+        }
+    });
+}());

rd_ui/app/scripts/visualizations/cohort.js

@@ -0,0 +1,60 @@
+(function () {
+    var cohortVisualization = angular.module('redash.visualization');
+
+    cohortVisualization.config(['VisualizationProvider', function(VisualizationProvider) {
+        VisualizationProvider.registerVisualization({
+            type: 'COHORT',
+            name: 'Cohort',
+            renderTemplate: '<cohort-renderer options="visualization.options" query-result="queryResult"></cohort-renderer>'
+        });
+    }]);
+
+    cohortVisualization.directive('cohortRenderer', function() {
+        return {
+            restrict: 'E',
+            scope: {
+                queryResult: '='
+            },
+            template: "",
+            replace: false,
+            link: function($scope, element, attrs) {
+                $scope.$watch('queryResult && queryResult.getData()', function (data) {
+                    if (!data) {
+                        return;
+                    }
+
+                    if ($scope.queryResult.getData() == null) {
+
+                    } else {
+                        var sortedData = _.sortBy($scope.queryResult.getData(), "date");
+                        var grouped = _.groupBy(sortedData, "date");
+                        var data = _.map(grouped, function(values, date) {
+                           var row = [values[0].total];
+                            _.each(values, function(value) { row.push(value.value); });
+                            return row;
+                        });
+
+                        var initialDate = moment(sortedData[0].date).toDate(),
+                            container = angular.element(element)[0];
+
+                        Cornelius.draw({
+                            initialDate: initialDate,
+                            container: container,
+                            cohort: data,
+                            title: null,
+                            timeInterval: 'daily',
+                            labels: {
+                                time: 'Activation Day',
+                                people: 'Users'
+                            },
+                            formatHeaderLabel: function (i) {
+                                return "Day " + (i - 1);
+                            }
+                        });
+                    }
+                });
+            }
+        }
+    });
+
+}());

rd_ui/app/scripts/visualizations/pivot.js

@@ -0,0 +1,29 @@
+var renderers = angular.module('redash.renderers', []);
+
+renderers.directive('pivotTableRenderer', function () {
+    return {
+        restrict: 'E',
+        scope: {
+            queryResult: '='
+        },
+        template: "",
+        replace: false,
+        link: function($scope, element, attrs) {
+            $scope.$watch('queryResult && queryResult.getData()', function (data) {
+                if (!data) {
+                    return;
+                }
+
+                if ($scope.queryResult.getData() == null) {
+                } else {
+                    // We need to give the pivot table its own copy of the data, because its change
+                    // it which interferes with other visualizations.
+                    var data = $.extend(true, [], $scope.queryResult.getData());
+                    $(element).pivotUI(data, {
+                         renderers: $.pivotUtilities.renderers
+                    }, true);
+                }
+            });
+        }
+    }
+});

rd_ui/app/scripts/visualizations/table.js

@@ -0,0 +1,90 @@
+(function () {
+    var tableVisualization = angular.module('redash.visualization');
+
+    tableVisualization.config(['VisualizationProvider', function(VisualizationProvider) {
+        VisualizationProvider.registerVisualization({
+            type: 'TABLE',
+            name: 'Table',
+            renderTemplate: '<grid-renderer options="visualization.options" query-result="queryResult"></grid-renderer>',
+            skipTypes: true
+        });
+    }]);
+
+    tableVisualization.directive('gridRenderer', function () {
+        return {
+            restrict: 'E',
+            scope: {
+                queryResult: '=',
+                itemsPerPage: '='
+            },
+            templateUrl: "/views/grid_renderer.html",
+            replace: false,
+            controller: ['$scope', function ($scope) {
+                $scope.gridColumns = [];
+                $scope.gridData = [];
+                $scope.gridConfig = {
+                    isPaginationEnabled: true,
+                    itemsByPage: $scope.itemsPerPage || 15,
+                    maxSize: 8
+                };
+
+                $scope.$watch('queryResult && queryResult.getData()', function (data) {
+                    if (!data) {
+                        return;
+                    }
+
+                    if ($scope.queryResult.getData() == null) {
+                        $scope.gridColumns = [];
+                        $scope.gridData = [];
+                        $scope.filters = [];
+                    } else {
+                        $scope.filters = $scope.queryResult.getFilters();
+
+                        var prepareGridData = function(data) {
+                            var gridData = _.map(data, function (row) {
+                                var newRow = {};
+                                _.each(row, function (val, key) {
+                                    newRow[$scope.queryResult.getColumnCleanName(key)] = val;
+                                })
+                                return newRow;
+                            });
+
+                            return gridData;
+                        };
+
+                        $scope.gridData = prepareGridData($scope.queryResult.getData());
+
+                        $scope.gridColumns = _.map($scope.queryResult.getColumnCleanNames(), function (col, i) {
+                            var columnDefinition = {
+                                'label': $scope.queryResult.getColumnFriendlyNames()[i],
+                                'map': col
+                            };
+
+                            var rawData = $scope.queryResult.getRawData();
+
+                            if (rawData.length > 0) {
+                                var exampleData = rawData[0][col];
+                                if (angular.isNumber(exampleData)) {
+                                    columnDefinition['formatFunction'] = 'number';
+                                    columnDefinition['formatParameter'] = 2;
+                                } else if (moment.isMoment(exampleData)) {
+                                    columnDefinition['formatFunction'] = function(value) {
+                                        // TODO: this is very hackish way to determine if we need
+                                        // to show the value as a time or date only. Better solution
+                                        // is to complete #70 and use the information it returns.
+                                        if (value._i.match(/^\d{4}-\d{2}-\d{2}T/)) {
+                                            return value.format("DD/MM/YY HH:mm");
+                                        }
+                                        return value.format("DD/MM/YY");
+                                    }
+                                }
+                            }
+
+                            return columnDefinition;
+                        });
+                    }
+                });
+            }]
+        }
+    })
+}());

rd_ui/app/styles/redash.css

@@ -25,7 +25,33 @@ a.navbar-brand {
     margin-top: 5px;
     margin-bottom: 5px;
 }
+.avatar img {
+    width: 40px;
+    height: 40px;
+}
 
+#logout {
+    color: white;
+    position: relative;
+    left: -9px;
+    bottom: -11px;
+}
+
+.details-toggle {
+    cursor: pointer;
+}
+.details-toggle::before {
+    content: '▸';
+    margin-right: 5px;
+}
+.details-toggle.open::before {
+    content: '▾';
+    margin-right: 5px;
+}
+
+.edit-in-place span {
+    white-space: pre-line;
+}
 .edit-in-place span.editable {
     cursor: pointer;
 }
@@ -60,10 +86,15 @@ a.navbar-brand {
     margin-bottom: 0px;
 }
 
-.panel-heading > a {
+.panel-heading > a,
+.panel-heading .query-link {
     color: inherit;
 }
 
+.panel-heading .query-link:hover {
+    text-decoration: none;
+}
+
 /* angular-growl */
 .growl {
     position: fixed;
@@ -215,6 +246,34 @@ to add those CSS styles here. */
     border-radius: 50%;
 }
 
+/* light version of bootstrap's form-control */
+.rd-form-control {
+    display: block;
+    padding: 6px 12px;
+    line-height: 1.428571429;
+    color: #555555;
+    vertical-align: middle;
+    background-color: #ffffff;
+    border: 1px solid #cccccc;
+    border-radius: 4px;
+    -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+    box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+    -webkit-transition: border-color ease-in-out 0.15s, box-shadow ease-in-out 0.15s;
+    transition: border-color ease-in-out 0.15s, box-shadow ease-in-out 0.15s;
+}
+.rd-form-control {
+     width: 100%;
+}
 visualization-renderer > div {
-    overflow: scroll;
+    overflow: auto;
+}
+
+/*
+bootstrap's hidden-xs class adds display:block when not hidden
+use this class when you need to keep the original display value
+*/
+@media (max-width: 767px) {
+    .rd-hidden-xs {
+        display: none !important;
+    }
 }

rd_ui/app/views/dashboard.html

@@ -23,9 +23,10 @@
 
             <div class="panel panel-default">
                 <div class="panel-heading">
-                    <h3 class="panel-title" style="cursor: pointer;" ng-click="open(query, widget.visualization)">
+                    <h3 class="panel-title">
                         <p>
-                            <span ng-bind="query.name"></span>
+                            <span ng-hide="currentUser.hasPermission('view_query')">{{query.name}}</span>
+                            <query-link query="query" visualization="widget.visualization" ng-show="currentUser.hasPermission('view_query')"></query-link>
                         </p>
                         <div class="text-muted" ng-bind="query.description"></div>
                     </h3>
@@ -39,7 +40,7 @@
                           tooltip-placement="bottom">Updated: <span am-time-ago="queryResult.getUpdatedAt()"></span></span>
 
                     <span class="pull-right">
-                        <a class="btn btn-default btn-xs" ng-href="/queries/{{query.id}}#{{widget.visualization.id}}"><span class="glyphicon glyphicon-link"></span></a>
+                        <a class="btn btn-default btn-xs" ng-href="/queries/{{query.id}}#{{widget.visualization.id}}" ng-show="currentUser.hasPermission('view_query')"><span class="glyphicon glyphicon-link"></span></a>
                         <button type="button" class="btn btn-default btn-xs" ng-show="dashboard.canEdit()" ng-click="deleteWidget()" title="Remove Widget"><span class="glyphicon glyphicon-trash"></span></button>
                     </span>
 

rd_ui/app/views/edit_visualization.html

@@ -1,24 +0,0 @@
-<form role="form" name="visForm" ng-submit="submit()">
-  <div class="form-group">
-    <label class="control-label">Name</label>
-    <input name="name" type="text" class="form-control" ng-model="vis.name" placeholder="{{vis.type}}">
-  </div>
-
-  <div class="form-group">
-    <label class="control-label">Visualization Type</label>
-    <select required ng-model="vis.type" ng-options="value as key for (key, value) in visTypes" class="form-control" ng-change="typeChanged()"></select>
-  </div>
-
-  <div class="form-group" ng-show="vis.type == visTypes.Chart">
-    <label class="control-label">Chart Type</label>
-    <select required ng-model="vis.options.series.type" ng-options="value as key for (key, value) in seriesTypes" class="form-control"></select>
-
-    <label class="control-label">Stacking</label>
-    <select required ng-model="stacking" ng-options="value as key for (key, value) in stackingOptions" class="form-control"></select>
-  </div>
-
-  <div class="form-group">
-    <button type="submit" class="btn btn-primary">Save</button>
-  </div>
-
-</form>

rd_ui/app/views/grid_renderer.html

@@ -1,17 +1,4 @@
 <div>
-    <div class="btn-group pull-right" ng-repeat="filter in filters">
-        <button type="button" class="btn btn-default dropdown-toggle" data-toggle="dropdown">
-            {{filter.friendlyName}}: {{filter.current}}<span class="caret"></span>
-        </button>
-        <ul class="dropdown-menu" role="menu">
-            <li ng-repeat="value in filter.values">
-                <a href="#" ng-click="filter.current = value">{{value}}</a>
-            </li>
-            <li class="divider"></li>
-            <li><a href="#" ng-click="filter.current = 'All'">All</a></li>
-        </ul>
-    </div>
-
     <smart-table rows="gridData" columns="gridColumns"
                  config="gridConfig"
                  class="table table-condensed table-hover"></smart-table>

rd_ui/app/views/index.html

@@ -3,7 +3,7 @@
     <div class="list-group" ng-repeat="(name, dashboards) in allDashboards">
         <div class="list-group-item active">
             {{name}}
-            <button type="button" class="btn btn-sm btn-link" data-toggle="modal" href="#new_dashboard_dialog" tooltip="New Dashboard"><span class="glyphicon glyphicon-plus-sign"></span></button>
+            <button ng-show="currentUser.hasPermission('create_dashboard')" type="button" class="btn btn-sm btn-link" data-toggle="modal" href="#new_dashboard_dialog" tooltip="New Dashboard"><span class="glyphicon glyphicon-plus-sign"></span></button>
         </div>
         <div class="list-group-item" ng-repeat="dashboard in dashboards" >
             <button type="button" class="close delete-button" aria-hidden="true" ng-show="dashboard.canEdit()" ng-click="archiveDashboard(dashboard)" tooltip="Delete Dashboard">&times;</button>
@@ -11,7 +11,7 @@
         </div>
     </div>
 
-    <div ng-show="currentUser.is_admin">
+    <div ng-show="currentUser.hasPermission('admin')">
         <div class="list-group">
             <div class="list-group-item active">Admin</div>
             <a href="/admin/status" class="list-group-item">Status</a>

rd_ui/app/views/new_widget_form.html

@@ -12,7 +12,7 @@
                         <input class="form-control" placeholder="Query Id" ng-model="queryId">
                       </div>
                       <button type="submit" class="btn btn-primary" ng-disabled="!queryId">
-                        <span class="glyphicon glyphicon-refresh"></span> Load
+                        Load visualizations
                       </button>
                     </form>
                 </p>
@@ -29,7 +29,7 @@
                   </div>
                 </div>
             </div>
-            <div class="modal-footer">
+            <div class="modal-footer" ng-if="selectedVis">
                 <button type="button" class="btn btn-default" ng-disabled="saveInProgress" data-dismiss="modal">Close</button>
                 <button type="button" class="btn btn-primary" ng-disabled="saveInProgress" ng-click="saveWidget()">Add to Dashboard</button>
             </div>

rd_ui/app/views/query.html

@@ -0,0 +1,182 @@
+
+<div class="container">
+
+    <alert-unsaved-changes ng-if="canEdit" is-dirty="isDirty"></alert-unsaved-changes>
+
+    <div class="row">
+        <div class="col-lg-12">
+            <div class="row">
+                <div class="col-lg-10">
+                    <h2>
+                        <edit-in-place editable="isQueryOwner" done="saveName" ignore-blanks='true' value="query.name"></edit-in-place>
+                    </h2>
+                    <p>
+                        <em>
+                            <edit-in-place editable="isQueryOwner" done="saveDescription" editor="textarea" placeholder="No description" ignore-blanks='false' value="query.description"></edit-in-place>
+                        </em>
+                    </p>
+                </div>
+
+                <div class="col-lg-2">
+                    <div class="rd-hidden-xs pull-right">
+                        <query-source-link></query-source-link>
+                    </div>
+                </div>
+            </div>
+
+            <div class="visible-xs">
+                <p>
+                    <span class="text-muted">Last update </span>
+                    <strong>
+                        <rd-time-ago value="queryResult.query_result.retrieved_at"></rd-time-ago>
+                    </strong>
+                    &nbsp;
+                    <span class="text-muted">Created By </span>
+                    <strong ng-hide="isQueryOwner">{{query.user.name}}</strong>
+                    <strong ng-show="isQueryOwner">You</strong>
+                    &nbsp;
+                    <span class="text-muted">Runtime </span>
+                    <strong ng-show="!queryExecuting">{{queryResult.getRuntime() | durationHumanize}}</strong>
+                    <span ng-show="queryExecuting">Running&hellip;</span>
+                    &nbsp;
+                    <span class="text-muted">Rows </span>
+                    <strong>{{queryResult.getData().length}}</strong>
+                </p>
+                <p>
+                    <query-source-link></query-source-link>
+                </p>
+            </div>
+        </div>
+    </div>
+
+    <hr>
+
+    <div class="row">
+        <div class="col-lg-12">
+            <div ng-show="sourceMode">
+                <p>
+                    <button type="button" class="btn btn-primary btn-xs" ng-disabled="queryExecuting" ng-click="executeQuery()">
+                        <span class="glyphicon glyphicon-play"></span> Execute
+                    </button>
+                    <query-formatter></query-formatter>
+                    <span class="pull-right">
+                        <button class="btn btn-xs btn-default rd-hidden-xs" ng-click="duplicateQuery()">
+                            <span class="glyphicon glyphicon-share-alt"></span> Fork
+                        </button>
+
+                        <button class="btn btn-success btn-xs" ng-show="canEdit" ng-click="saveQuery()">
+                            <span class="glyphicon glyphicon-floppy-disk"> </span> Save<span ng-show="isDirty">&#42;</span>
+                        </button>
+                    </span>
+                </p>
+            </div>
+
+            <!-- code editor -->
+            <div ng-show="sourceMode">
+                <p>
+                    <query-editor query="query" lock="queryExecuting"></query-editor>
+                </p>
+                <hr>
+            </div>
+        </div>
+
+    </div>
+
+    <div class="row">
+        <div class="col-lg-3 rd-hidden-xs">
+            <p>
+                <span class="glyphicon glyphicon-time"></span>
+                <span class="text-muted">Last update </span>
+                <strong>
+                    <rd-time-ago value="queryResult.query_result.retrieved_at"></rd-time-ago>
+                </strong>
+            </p>
+            <p>
+                <span class="glyphicon glyphicon-user"></span>
+                <span class="text-muted">Created By </span>
+                <strong ng-hide="isQueryOwner">{{query.user.name}}</strong>
+                <strong ng-show="isQueryOwner">You</strong>
+            </p>
+            <p>
+                <span class="glyphicon glyphicon-play"></span>
+                <span class="text-muted">Runtime </span>
+                <strong ng-show="!queryExecuting">{{queryResult.getRuntime() | durationHumanize}}</strong>
+                <span ng-show="queryExecuting">Running&hellip;</span>
+            </p>
+            <p>
+                <span class="glyphicon glyphicon-align-justify"></span>
+                <span class="text-muted">Rows </span><strong>{{queryResult.getData().length}}</strong>
+            </p>
+            <p>
+                <span class="glyphicon glyphicon-refresh"></span>
+                <span class="text-muted">Refresh Interval</span>
+                <query-refresh-select></query-refresh-select>
+            </p>
+
+            <p>
+                <span class="glyphicon glyphicon-hdd"></span>
+                <span class="text-muted">Data Source</span>
+                <select ng-disabled="!isQueryOwner" ng-model="query.data_source_id" ng-change="updateDataSource()" ng-options="ds.id as ds.name for ds in dataSources"></select>
+            </p>
+
+            <hr>
+
+            <p>
+                <a class="btn btn-primary btn-sm" ng-disabled="queryExecuting || !queryResult.getData()" ng-href="{{dataUri}}" download="{{dataFilename}}" target="_self">
+                    <span class="glyphicon glyphicon-cloud-download"></span>
+                    <span class="rd-hidden-xs">Download Dataset</span>
+                </a>
+            </p>
+        </div>
+
+        <div class="col-lg-9">
+            <!-- alerts -->
+            <div class="alert alert-info" ng-show="queryResult.getStatus() == 'processing'">
+                Executing query&hellip; <rd-timer timestamp="queryResult.getUpdatedAt()"></rd-timer>
+                <button type="button" class="btn btn-warning btn-xs pull-right" ng-disabled="cancelling" ng-click="cancelExecution()">Cancel</button>
+            </div>
+            <div class="alert alert-info" ng-show="queryResult.getStatus() == 'waiting'">
+                Query in queue&hellip; <rd-timer timestamp="queryResult.getUpdatedAt()"></rd-timer>
+                <button type="button" class="btn btn-warning btn-xs pull-right" ng-disabled="cancelling" ng-click="cancelExecution()">Cancel</button>
+            </div>
+            <div class="alert alert-danger" ng-show="queryResult.getError()">Error running query: <strong>{{queryResult.getError()}}</strong></div>
+
+            <!-- tabs and data -->
+            <div ng-show="queryResult.getStatus() == 'done'">
+                <div class="row">
+                    <div class="col-lg-12">
+                        <ul class="nav nav-tabs">
+                            <rd-tab tab-id="table" name="Table"></rd-tab>
+                            <rd-tab tab-id="pivot" name="Pivot Table"></rd-tab>
+                            <rd-tab tab-id="{{vis.id}}" name="{{vis.name}}" ng-if="vis.type!='TABLE'" ng-repeat="vis in query.visualizations">
+                                <span class="remove" ng-click="deleteVisualization($event, vis)" ng-show="canEdit"> &times;</span>
+                            </rd-tab>
+                            <rd-tab tab-id="add" name="&plus; New" removeable="true" ng-show="canEdit"></rd-tab>
+                        </ul>
+                    </div>
+                </div>
+                <div class="row">
+                    <div class="col-lg-12">
+                        <div ng-show="selectedTab == 'table'" >
+                            <filters></filters>
+                            <grid-renderer query-result="queryResult" items-per-page="50"></grid-renderer>
+                        </div>
+
+                        <pivot-table-renderer ng-show="selectedTab == 'pivot'" query-result="queryResult"></pivot-table-renderer>
+
+                        <div ng-show="selectedTab == vis.id" ng-repeat="vis in query.visualizations">
+                            <visualization-renderer visualization="vis" query-result="queryResult"></visualization-renderer>
+
+                            <edit-visulatization-form visualization="vis" query="query" query-result="queryResult" ng-show="canEdit"></edit-visulatization-form>
+                        </div>
+
+                        <div ng-show="selectedTab == 'add'">
+                            <visualization-renderer visualization="newVisualization" query-result="queryResult"></visualization-renderer>
+                            <edit-visulatization-form visualization="newVisualization" query="query" ng-show="canEdit" on-new-success="setVisualizationTab"></edit-visulatization-form>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+    </div>
+</div>

rd_ui/app/views/queryfiddle.html

@@ -1,102 +0,0 @@
-<div class="container">
-    <div class="row">
-        <div class="panel panel-default">
-            <div class="panel-heading">
-                <h3 class="panel-title">
-                    <p>
-                     <edit-in-place editable="currentUser.canEdit(query)" ignore-blanks='true' value="query.name"></edit-in-place>
-                 </p>
-                </h3>
-                <p>
-                    <edit-in-place editable="currentUser.canEdit(query)" editor="textarea" placeholder="No description" ignore-blanks='false' value="query.description" class="text-muted"></edit-in-place>
-                </p>
-            </div>
-            <div class="panel-body">
-                <textarea ui-codemirror="editorOptions" ng-model="query.query"></textarea>
-
-                <div>
-                    <a class="btn btn-default" ng-disabled="queryExecuting || !queryResult.getData()" ng-href="{{dataUri}}" download="{{dataFilename}}" target="_self">
-                        <span class="glyphicon glyphicon-floppy-disk"></span> Download Data Set
-                    </a>
-                    <button type="button" class="btn btn-default center-x" ng-click="formatQuery()"><span class="glyphicon glyphicon-ok"></span> Format SQL</button>
-
-                    <div class="btn-group pull-right">
-                        <button type="button" class="btn btn-default" ng-click="duplicateQuery()">Duplicate</button>
-                        <button type="button" class="btn btn-default" ng-disabled="!currentUser.canEdit(query)" ng-click="saveQuery()">Save
-                            <span ng-show="dirty">&#42;</span>
-                        </button>
-                        <button type="button" class="btn btn-primary" ng-disabled="queryExecuting" ng-click="executeQuery()">Execute</button>
-                    </div>
-                </div>
-
-            </div>
-            <div class="panel-footer">
-                <span ng-show="queryResult.getRuntime()>=0">Query runtime: {{queryResult.getRuntime() | durationHumanize}} | </span>
-                <span ng-show="queryResult.query_result.retrieved_at">Last update time: <span am-time-ago="queryResult.query_result.retrieved_at"></span> | </span>
-                <span ng-show="queryResult.getStatus() == 'done'">Rows: {{queryResult.getData().length}} | </span>
-                Created by: {{query.user.name}}
-                <div class="pull-right">Refresh query: <select ng-model="query.ttl" ng-options="c.value as c.name for c in refreshOptions"></select><br></div>
-            </div>
-        </div>
-
-        <div class="alert alert-info" ng-show="queryResult.getStatus() == 'processing'">
-            Executing query... <rd-timer timestamp="queryResult.getUpdatedAt()"></rd-timer>
-            <button type="button" class="btn btn-warning btn-xs pull-right" ng-disabled="cancelling" ng-click="cancelExecution()">Cancel</button>
-        </div>
-        <div class="alert alert-info" ng-show="queryResult.getStatus() == 'waiting'">
-            Query in queue... <rd-timer timestamp="queryResult.getUpdatedAt()"></rd-timer>
-            <button type="button" class="btn btn-warning btn-xs pull-right" ng-disabled="cancelling" ng-click="cancelExecution()">Cancel</button>
-        </div>
-        <div class="alert alert-danger" ng-show="queryResult.getError()">Error running query: <strong>{{queryResult.getError()}}</strong></div>
-    </div>
-
-    <div class="row" ng-show="queryResult.getStatus() == 'done'">
-        <ul class="nav nav-tabs">
-            <rd-tab id="table" name="Table"></rd-tab>
-            <rd-tab id="pivot" name="Pivot Table"></rd-tab>
-            <!-- hide the table visualization -->
-            <rd-tab id="{{vis.id}}" name="{{vis.name}}" ng-hide="vis.type=='TABLE'" ng-repeat="vis in query.visualizations">
-                <span class="remove" ng-click="deleteVisualization($event, vis)" ng-show="currentUser.canEdit(query)"> &times;</span>
-            </rd-tab>
-            <rd-tab id="add" name="&plus;New" removeable="true" ng-show="currentUser.canEdit(query)"></rd-tab>
-        </ul>
-
-        <div class="col-lg-12" ng-show="selectedTab == 'table'">
-            <grid-renderer query-result="queryResult" items-per-page="50"></grid-renderer>
-        </div>
-
-        <div class="col-lg-12" ng-show="selectedTab == 'pivot'">
-            <pivot-table-renderer query-result="queryResult"></pivot-table-renderer>
-        </div>
-
-        <div class="col-lg-12" ng-show="selectedTab == vis.id" ng-repeat="vis in query.visualizations">
-            <div class="row" ng-show="currentUser.canEdit(query)">
-                <p>
-                    <div class="col-lg-12">
-                        <edit-visulatization-form vis="vis" query="query"></edit-visulatization-form>
-                    </div>
-                </p>
-            </div>
-            <div class="row">
-                <p>
-                    <div class="col-lg-12">
-                        <visualization-renderer visualization="vis" query-result="queryResult"></visualization-renderer>
-                    </div>
-                </p>
-            </div>
-        </div>
-
-        <div class="col-lg-12" ng-show="selectedTab == 'add'">
-            <div class="row">
-                 <p>
-                    <div class="col-lg-6">
-                        <edit-visulatization-form vis="newVisualization" query="query"></edit-visulatization-form>
-                    </div>
-                    <div class="col-lg-6">
-                        <visualization-renderer visualization="newVisualization" query-result="queryResult"></visualization-renderer>
-                    </div>
-                </p>
-            </div>
-        </div>
-    </div>
-</div>

rd_ui/app/views/visualizations/chart_editor.html

@@ -0,0 +1,11 @@
+<div>
+  <div class="form-group">
+      <label class="control-label">Chart Type</label>
+      <select required ng-model="visualization.options.series.type" ng-options="value as key for (key, value) in seriesTypes" class="form-control"></select>
+  </div>
+
+  <div class="form-group">
+      <label class="control-label">Stacking</label>
+      <select required ng-model="stacking" ng-options="value as key for (key, value) in stackingOptions" class="form-control"></select>
+  </div>
+</div>

rd_ui/app/views/visualizations/cohort_editor.html

@@ -0,0 +1,15 @@
+<div class="form-group">
+    <label class="control-label">Time Label</label>
+    <input type="text" class="form-control" ng-model="cohortOptions.timeLabel">
+    <label class="control-label">People Label</label>
+    <input type="text" class="form-control" ng-model="cohortOptions.peopleLabel">
+
+    <label class="control-label">Bucket Column</label>
+    <select ng-model="bucket_column" ng-options="value as key for (key, value) in columns" class="form-control"></select>
+    <label class="control-label">Bucket Total Value Column</label>
+    <select ng-model="total_column" ng-options="value as key for (key, value) in columns" class="form-control"></select>
+    <label class="control-label">Day Number Column</label>
+    <select ng-model="value_column" ng-options="value as key for (key, value) in columns" class="form-control"></select>
+    <label class="control-label">Day Value Column</label>
+    <select ng-model="day_column" ng-options="value as key for (key, value) in columns" class="form-control"></select>
+</div>

rd_ui/app/views/visualizations/edit_visualization.html

@@ -0,0 +1,27 @@
+<div>
+  <span ng-click="visEdit=!visEdit" class="details-toggle" ng-class="{open: visEdit}">Edit</span>
+
+  <form ng-if="visEdit" role="form" name="visForm" ng-submit="submit()">
+    <div class="form-group">
+      <label class="control-label">Name</label>
+      <input name="name" type="text" class="form-control" ng-model="visualization.name" placeholder="{{visualization.type | capitalize}}">
+    </div>
+
+    <div class="form-group">
+      <label class="control-label">Visualization Type</label>
+      <select required ng-model="visualization.type" ng-options="value as key for (key, value) in visTypes" class="form-control" ng-change="typeChanged()"></select>
+    </div>
+
+    <visualization-options-editor></visualization-options-editor>
+
+    <div class="form-group" ng-if="editRawOptions">
+      <label class="control-label">Advanced</label>
+      <textarea json-text ng-model="visualization.options" class="form-control" rows="10"></textarea>
+    </div>
+
+    <div class="form-group">
+      <button type="submit" class="btn btn-primary">Save</button>
+    </div>
+
+  </form>
+</div>

rd_ui/app/views/visualizations/filters.html

@@ -0,0 +1,12 @@
+<div class="well well-sm" ng-show="filters">
+    <div class="btn-group" ng-repeat="filter in filters">
+        <button type="button" class="btn btn-default dropdown-toggle" data-toggle="dropdown">
+            {{filter.friendlyName}}: {{filter.current}}<span class="caret"></span>
+        </button>
+        <ul class="dropdown-menu" role="menu">
+            <li ng-repeat="value in filter.values">
+                <a href="" ng-click="filter.current = value">{{value}}</a>
+            </li>
+        </ul>
+    </div>
+</div>

rd_ui/bower.json

@@ -13,11 +13,11 @@
     "angular-ui-codemirror": "0.0.5",
     "highcharts": "3.0.1",
     "underscore": "1.5.1",
-    "angular-resource": "1.0.7",
+    "angular-resource": "1.2.15",
     "angular-growl": "0.3.1",
     "angular-route": "1.2.7",
-    "pivottable": "git@github.com:arikfr/pivottable.git#master",
-    "cornelius": "git@github.com:restorando/cornelius.git",
+    "pivottable": "https://github.com/arikfr/pivottable.git",
+    "cornelius": "https://github.com/restorando/cornelius.git",
     "gridster": "0.2.0",
     "mousetrap": "~1.4.6"
   },
@@ -26,7 +26,6 @@
     "angular-scenario": "~1.0.7"
   },
   "resolutions": {
-    "angular": "~1.2.7",
-    "jquery": "~1.9.1"
+    "angular": "1.2.7"
   }
 }

redash/__init__.py

@@ -1,13 +1,18 @@
 import json
 import urlparse
+import logging
 from flask import Flask, make_response
 from flask.ext.restful import Api
 from flask_peewee.db import Database
 
 import redis
+from statsd import StatsClient
 from redash import settings, utils
 
-__version__ = '0.3.3'
+__version__ = '0.3.5'
+
+logging.getLogger().addHandler(logging.StreamHandler())
+logging.getLogger().setLevel(settings.LOG_LEVEL)
 
 app = Flask(__name__,
             template_folder=settings.STATIC_ASSETS_PATH,
@@ -36,9 +41,11 @@ if redis_url.path:
     redis_db = redis_url.path[1]
 else:
     redis_db = 0
+
 redis_connection = redis.StrictRedis(host=redis_url.hostname, port=redis_url.port, db=redis_db, password=redis_url.password)
+statsd_client = StatsClient(host=settings.STATSD_HOST, port=settings.STATSD_PORT, prefix=settings.STATSD_PREFIX)
 
 from redash import data
-data_manager = data.Manager(redis_connection, db)
+data_manager = data.Manager(redis_connection, statsd_client)
 
 from redash import controllers

redash/authentication.py

@@ -1,17 +1,22 @@
 import functools
 import hashlib
 import hmac
-from flask import current_app, request, make_response, g, redirect, url_for
-from flask.ext.googleauth import GoogleAuth, login
-from flask.ext.login import LoginManager, login_user, current_user
 import time
 import logging
+
+from flask import request, make_response, redirect, url_for
+from flask.ext.googleauth import GoogleAuth, login
+from flask.ext.login import LoginManager, login_user, current_user
 from werkzeug.contrib.fixers import ProxyFix
+
+from models import AnonymousUser
 from redash import models, settings
 
+
 login_manager = LoginManager()
 logger = logging.getLogger('authentication')
 
+
 def sign(key, path, expires):
     if not key:
         return None
@@ -35,18 +40,15 @@ class HMACAuthentication(object):
             calculated_signature = sign(query.api_key, request.path, expires)
 
             if query.api_key and signature == calculated_signature:
+                login_user(models.ApiUser(query.api_key), remember=False)
                 return True
 
         return False
 
-    @staticmethod
-    def is_user_logged_in():
-        return current_user.is_authenticated()
-
     def required(self, fn):
         @functools.wraps(fn)
         def decorated(*args, **kwargs):
-            if self.is_user_logged_in():
+            if current_user.is_authenticated():
                 return fn(*args, **kwargs)
 
             if self.api_key_authentication():
@@ -98,6 +100,7 @@ def setup_authentication(app):
             openid_auth._OPENID_ENDPOINT = "https://www.google.com/a/%s/o8/ud?be=o8" % settings.GOOGLE_APPS_DOMAIN
 
     login_manager.init_app(app)
+    login_manager.anonymous_user = AnonymousUser
     app.wsgi_app = ProxyFix(app.wsgi_app)
     app.secret_key = settings.COOKIE_SECRET
 

redash/controllers.py

@@ -17,6 +17,7 @@ from flask.ext.restful import Resource, abort
 from flask_login import current_user, login_user, logout_user
 
 import sqlparse
+from permissions import require_permission
 from redash import settings, utils
 from redash import data
 
@@ -32,10 +33,11 @@ def ping():
 @app.route('/admin/<anything>')
 @app.route('/dashboard/<anything>')
 @app.route('/queries')
-@app.route('/queries/<anything>')
+@app.route('/queries/<query_id>')
+@app.route('/queries/<query_id>/<anything>')
 @app.route('/')
 @auth.required
-def index(anything=None):
+def index(**kwargs):
     email_md5 = hashlib.md5(current_user.email.lower()).hexdigest()
     gravatar_url = "https://www.gravatar.com/avatar/%s?s=40" % email_md5
 
@@ -44,10 +46,12 @@ def index(anything=None):
         'is_admin': current_user.is_admin,
         'id': current_user.id,
         'name': current_user.name,
-        'email': current_user.email
+        'email': current_user.email,
+        'permissions': current_user.permissions
     }
 
-    return render_template("index.html", user=json.dumps(user), analytics=settings.ANALYTICS)
+    return render_template("index.html", user=json.dumps(user), name=settings.NAME,
+                           analytics=settings.ANALYTICS)
 
 
 @app.route('/login', methods=['GET', 'POST'])
@@ -67,6 +71,7 @@ def login():
             return redirect(request.args.get('next') or '/')
 
     return render_template("login.html",
+                           name=settings.NAME,
                            analytics=settings.ANALYTICS,
                            next=request.args.get('next'),
                            username=request.form.get('username', ''),
@@ -82,6 +87,7 @@ def logout():
 
 @app.route('/status.json')
 @auth.required
+@require_permission('admin')
 def status_api():
     status = {}
     info = redis_connection.info()
@@ -123,6 +129,14 @@ class BaseResource(Resource):
         return current_user._get_current_object()
 
 
+class DataSourceListAPI(BaseResource):
+    def get(self):
+        data_sources = [ds.to_dict() for ds in models.DataSource.select()]
+        return data_sources
+
+api.add_resource(DataSourceListAPI, '/api/data_sources', endpoint='data_sources')
+
+
 class DashboardListAPI(BaseResource):
     def get(self):
         dashboards = [d.to_dict() for d in
@@ -130,6 +144,7 @@ class DashboardListAPI(BaseResource):
 
         return dashboards
 
+    @require_permission('create_dashboard')
     def post(self):
         dashboard_properties = request.get_json(force=True)
         dashboard = models.Dashboard(name=dashboard_properties['name'],
@@ -148,6 +163,7 @@ class DashboardAPI(BaseResource):
 
         return dashboard.to_dict(with_widgets=True)
 
+    @require_permission('edit_dashboard')
     def post(self, dashboard_slug):
         dashboard_properties = request.get_json(force=True)
         # TODO: either convert all requests to use slugs or ids
@@ -158,6 +174,7 @@ class DashboardAPI(BaseResource):
 
         return dashboard.to_dict(with_widgets=True)
 
+    @require_permission('edit_dashboard')
     def delete(self, dashboard_slug):
         dashboard = models.Dashboard.get_by_slug(dashboard_slug)
         dashboard.is_archived = True
@@ -168,6 +185,7 @@ api.add_resource(DashboardAPI, '/api/dashboards/<dashboard_slug>', endpoint='das
 
 
 class WidgetListAPI(BaseResource):
+    @require_permission('edit_dashboard')
     def post(self):
         widget_properties = request.get_json(force=True)
         widget_properties['options'] = json.dumps(widget_properties['options'])
@@ -199,6 +217,7 @@ class WidgetListAPI(BaseResource):
 
 
 class WidgetAPI(BaseResource):
+    @require_permission('edit_dashboard')
     def delete(self, widget_id):
         widget = models.Widget.get(models.Widget.id == widget_id)
         # TODO: reposition existing ones
@@ -215,13 +234,14 @@ api.add_resource(WidgetAPI, '/api/widgets/<int:widget_id>', endpoint='widget')
 
 
 class QueryListAPI(BaseResource):
+    @require_permission('create_query')
     def post(self):
         query_def = request.get_json(force=True)
-        # id, created_at, api_key
         for field in ['id', 'created_at', 'api_key', 'visualizations', 'latest_query_data']:
             query_def.pop(field, None)
 
         query_def['user'] = self.current_user
+        query_def['data_source'] = query_def.pop('data_source_id')
         query = models.Query(**query_def)
         query.save()
 
@@ -229,11 +249,13 @@ class QueryListAPI(BaseResource):
 
         return query.to_dict(with_result=False)
 
+    @require_permission('view_query')
     def get(self):
         return [q.to_dict(with_result=False, with_stats=True) for q in models.Query.all_queries()]
 
 
 class QueryAPI(BaseResource):
+    @require_permission('edit_query')
     def post(self, query_id):
         query_def = request.get_json(force=True)
         for field in ['id', 'created_at', 'api_key', 'visualizations', 'latest_query_data', 'user']:
@@ -242,12 +264,16 @@ class QueryAPI(BaseResource):
         if 'latest_query_data_id' in query_def:
             query_def['latest_query_data'] = query_def.pop('latest_query_data_id')
 
+        if 'data_source_id' in query_def:
+            query_def['data_source'] = query_def.pop('data_source_id')
+
         models.Query.update_instance(query_id, **query_def)
-        
+
         query = models.Query.get_by_id(query_id)
 
         return query.to_dict(with_result=False, with_visualizations=True)
 
+    @require_permission('view_query')
     def get(self, query_id):
         q = models.Query.get(models.Query.id == query_id)
         if q:
@@ -260,6 +286,7 @@ api.add_resource(QueryAPI, '/api/queries/<query_id>', endpoint='query')
 
 
 class VisualizationListAPI(BaseResource):
+    @require_permission('edit_query')
     def post(self):
         kwargs = request.get_json(force=True)
         kwargs['options'] = json.dumps(kwargs['options'])
@@ -272,6 +299,7 @@ class VisualizationListAPI(BaseResource):
 
 
 class VisualizationAPI(BaseResource):
+    @require_permission('edit_query')
     def post(self, visualization_id):
         kwargs = request.get_json(force=True)
         if 'options' in kwargs:
@@ -285,6 +313,7 @@ class VisualizationAPI(BaseResource):
 
         return vis.to_dict(with_query=False)
 
+    @require_permission('edit_query')
     def delete(self, visualization_id):
         vis = models.Visualization.get(models.Visualization.id == visualization_id)
         vis.delete_instance()
@@ -294,38 +323,48 @@ api.add_resource(VisualizationAPI, '/api/visualizations/<visualization_id>', end
 
 
 class QueryResultListAPI(BaseResource):
+    @require_permission('execute_query')
     def post(self):
         params = request.json
 
+        models.ActivityLog(
+            user=self.current_user,
+            type=models.ActivityLog.QUERY_EXECUTION,
+            activity=params['query']
+        ).save()
+
         if params['ttl'] == 0:
             query_result = None
         else:
-            query_result = data_manager.get_query_result(params['query'], int(params['ttl']))
+            query_result = models.QueryResult.get_latest(params['data_source_id'], params['query'], int(params['ttl']))
 
         if query_result:
-            return {'query_result': query_result.to_dict(parse_data=True)}
+            return {'query_result': query_result.to_dict()}
         else:
-            job = data_manager.add_job(params['query'], data.Job.HIGH_PRIORITY)
+            data_source = models.DataSource.get_by_id(params['data_source_id'])
+            job = data_manager.add_job(params['query'], data.Job.HIGH_PRIORITY, data_source)
             return {'job': job.to_dict()}
 
 
 class QueryResultAPI(BaseResource):
+    @require_permission('view_query')
     def get(self, query_result_id):
-        query_result = data_manager.get_query_result_by_id(query_result_id)
+        query_result = models.QueryResult.get_by_id(query_result_id)
         if query_result:
-            return {'query_result': query_result.to_dict(parse_data=True)}
+            return {'query_result': query_result.to_dict()}
         else:
             abort(404)
 
 
 class CsvQueryResultsAPI(BaseResource):
+    @require_permission('view_query')
     def get(self, query_id, query_result_id=None):
         if not query_result_id:
             query = models.Query.get(models.Query.id == query_id)
             if query:
                 query_result_id = query._data['latest_query_data']
 
-        query_result = query_result_id and data_manager.get_query_result_by_id(query_result_id)
+        query_result = query_result_id and models.QueryResult.get_by_id(query_result_id)
         if query_result:
             s = cStringIO.StringIO()
 

redash/data/manager.py

@@ -1,34 +1,30 @@
 """
 Data manager. Used to manage and coordinate execution of queries.
 """
-from contextlib import contextmanager
-import collections
-import json
 import time
 import logging
-import psycopg2
+import peewee
 import qr
 import redis
-from redash import settings
+import json
+from redash import models
 from redash.data import worker
 from redash.utils import gen_query_hash
 
-class QueryResult(collections.namedtuple('QueryData', 'id query data runtime retrieved_at query_hash')):
-    def to_dict(self, parse_data=False):
-        d = self._asdict()
 
-        if parse_data and d['data']:
-            d['data'] = json.loads(d['data'])
-
-        return d
+class JSONPriorityQueue(qr.PriorityQueue):
+    """ Use a JSON serializer to help with cross language support """
+    def __init__(self, key, **kwargs):
+        super(qr.PriorityQueue, self).__init__(key, **kwargs)
+        self.serializer = json
 
 
 class Manager(object):
-    def __init__(self, redis_connection, db):
+    def __init__(self, redis_connection, statsd_client):
+        self.statsd_client = statsd_client
         self.redis_connection = redis_connection
-        self.db = db
         self.workers = []
-        self.queue = qr.PriorityQueue("jobs", **self.redis_connection.connection_pool.connection_kwargs)
+        self.queue = JSONPriorityQueue("jobs", **self.redis_connection.connection_pool.connection_kwargs)
         self.max_retries = 5
         self.status = {
             'last_refresh_at': 0,
@@ -37,36 +33,7 @@ class Manager(object):
 
         self._save_status()
 
-    # TODO: Use our Django Models
-    def get_query_result_by_id(self, query_result_id):
-        with self.db_transaction() as cursor:
-            sql = "SELECT id, query, data, runtime, retrieved_at, query_hash FROM query_results " \
-                  "WHERE id=%s LIMIT 1"
-            cursor.execute(sql, (query_result_id,))
-            query_result = cursor.fetchone()
-
-        if query_result:
-            query_result = QueryResult(*query_result)
-
-        return query_result
-
-    def get_query_result(self, query, ttl=0):
-        query_hash = gen_query_hash(query)
-
-        with self.db_transaction() as cursor:
-            sql = "SELECT id, query, data, runtime, retrieved_at, query_hash FROM query_results " \
-                  "WHERE query_hash=%s " \
-                  "AND retrieved_at < now() at time zone 'utc' - interval '%s second'" \
-                  "ORDER BY retrieved_at DESC LIMIT 1"
-            cursor.execute(sql, (query_hash, psycopg2.extensions.AsIs(ttl)))
-            query_result = cursor.fetchone()
-
-        if query_result:
-            query_result = QueryResult(*query_result)
-
-        return query_result
-
-    def add_job(self, query, priority):
+    def add_job(self, query, priority, data_source):
         query_hash = gen_query_hash(query)
         logging.info("[Manager][%s] Inserting job with priority=%s", query_hash, priority)
         try_count = 0
@@ -83,7 +50,11 @@ class Manager(object):
                     logging.info("[Manager][%s] Found existing job: %s", query_hash, job_id)
                     job = worker.Job.load(self.redis_connection, job_id)
                 else:
-                    job = worker.Job(self.redis_connection, query, priority)
+                    job = worker.Job(self.redis_connection, query=query, priority=priority,
+                                     data_source_id=data_source.id,
+                                     data_source_name=data_source.name,
+                                     data_source_type=data_source.type,
+                                     data_source_options=data_source.options)
                     pipe.multi()
                     job.save(pipe)
                     logging.info("[Manager][%s] Created new job: %s", query_hash, job.id)
@@ -98,79 +69,79 @@ class Manager(object):
 
         return job
 
+    def report_status(self):
+        workers = [self.redis_connection.hgetall(w)
+                   for w in self.redis_connection.smembers('workers')]
+
+        for w in workers:
+            self.statsd_client.gauge('worker_{}.seconds_since_update'.format(w['id']),
+                                     time.time() - float(w['updated_at']))
+            self.statsd_client.gauge('worker_{}.jobs_received'.format(w['id']), int(w['jobs_count']))
+            self.statsd_client.gauge('worker_{}.jobs_done'.format(w['id']), int(w['done_jobs_count']))
+
+        manager_status = self.redis_connection.hgetall('manager:status')
+        self.statsd_client.gauge('manager.seconds_since_refresh',
+                                 time.time() - float(manager_status['last_refresh_at']))
+
     def refresh_queries(self):
-        sql = """SELECT queries.query, queries.ttl, retrieved_at
-        FROM (SELECT query, min(ttl) as ttl FROM queries  WHERE ttl > 0 GROUP by query) queries
-        JOIN (SELECT query, max(retrieved_at) as retrieved_at
-        FROM query_results
-        GROUP BY query) query_results on query_results.query=queries.query
-        WHERE queries.ttl > 0
-          AND query_results.retrieved_at + ttl * interval '1 second' < now() at time zone 'utc';"""
+        # TODO: this will only execute scheduled queries that were executed before. I think this is
+        # a reasonable assumption, but worth revisiting.
+
+        # TODO: move this logic to the model.
+        outdated_queries = models.Query.select(peewee.Func('first_value', models.Query.id)\
+            .over(partition_by=[models.Query.query_hash, models.Query.data_source]))\
+            .join(models.QueryResult)\
+            .where(models.Query.ttl > 0,
+                   (models.QueryResult.retrieved_at +
+                    (models.Query.ttl * peewee.SQL("interval '1 second'"))) <
+                   peewee.SQL("(now() at time zone 'utc')"))
+
+        queries = models.Query.select(models.Query, models.DataSource).join(models.DataSource)\
+            .where(models.Query.id << outdated_queries)
 
         self.status['last_refresh_at'] = time.time()
         self._save_status()
 
         logging.info("Refreshing queries...")
-        queries = self.run_query(sql)
-        for query, ttl, retrieved_at in queries:
-            self.add_job(query, worker.Job.LOW_PRIORITY)
 
-        logging.info("Done refreshing queries... %d" % len(queries))
+        outdated_queries_count = 0
+        for query in queries:
+            self.add_job(query.query, worker.Job.LOW_PRIORITY, query.data_source)
+            outdated_queries_count += 1
 
-    def store_query_result(self, query, data, run_time, retrieved_at):
-        query_result_id = None
+        self.statsd_client.gauge('manager.outdated_queries', outdated_queries_count)
+        self.statsd_client.gauge('manager.queue_size', self.redis_connection.zcard('jobs'))
+
+        logging.info("Done refreshing queries... %d" % outdated_queries_count)
+
+    def store_query_result(self, data_source_id, query, data, run_time, retrieved_at):
         query_hash = gen_query_hash(query)
-        sql = "INSERT INTO query_results (query_hash, query, data, runtime, retrieved_at) " \
-              "VALUES (%s, %s, %s, %s, %s) RETURNING id"
-        with self.db_transaction() as cursor:
-            cursor.execute(sql, (query_hash, query, data, run_time, retrieved_at))
-            if cursor.rowcount == 1:
-                query_result_id = cursor.fetchone()[0]
-                logging.info("[Manager][%s] Inserted query data; id=%s", query_hash, query_result_id)
 
-                sql = "UPDATE queries SET latest_query_data_id=%s WHERE query_hash=%s"
-                cursor.execute(sql, (query_result_id, query_hash))
+        query_result = models.QueryResult.create(query_hash=query_hash,
+                                                 query=query,
+                                                 runtime=run_time,
+                                                 data_source=data_source_id,
+                                                 retrieved_at=retrieved_at,
+                                                 data=data)
 
-                logging.info("[Manager][%s] Updated %s queries.", query_hash, cursor.rowcount)
-            else:
-                logging.error("[Manager][%s] Failed inserting query data.", query_hash)
-        return query_result_id
+        logging.info("[Manager][%s] Inserted query data; id=%s", query_hash, query_result.id)
 
-    def run_query(self, *args):
-        sql = args[0]
-        logging.debug("running query: %s %s", sql, args[1:])
+        # TODO: move this logic to the model?
+        updated_count = models.Query.update(latest_query_data=query_result).\
+            where(models.Query.query_hash==query_hash, models.Query.data_source==data_source_id).\
+            execute()
 
-        with self.db_transaction() as cursor:
-            cursor.execute(sql, args[1:])
-            if cursor.description:
-                data = list(cursor)
-            else:
-                data = cursor.rowcount
+        logging.info("[Manager][%s] Updated %s queries.", query_hash, updated_count)
 
-        return data
+        return query_result.id
 
-    def start_workers(self, workers_count, connection_type, connection_string):
+    def start_workers(self, workers_count):
         if self.workers:
             return self.workers
-        
-        if connection_type == 'mysql':
-            from redash.data import query_runner_mysql
-            runner = query_runner_mysql.mysql(connection_string)
-        elif connection_type == 'graphite':
-            from redash.data import query_runner_graphite
-            connection_params = json.loads(connection_string)
-            if connection_params['auth']:
-                connection_params['auth'] = tuple(connection_params['auth'])
-            else:
-                connection_params['auth'] = None
-            runner = query_runner_graphite.graphite(connection_params)
-        else:
-            from redash.data import query_runner
-            runner = query_runner.redshift(connection_string)
 
         redis_connection_params = self.redis_connection.connection_pool.connection_kwargs
-        self.workers = [worker.Worker(self, redis_connection_params, runner)
-                        for _ in range(workers_count)]
+        self.workers = [worker.Worker(worker_id, self, redis_connection_params)
+                        for worker_id in xrange(workers_count)]
         for w in self.workers:
             w.start()
 
@@ -181,20 +152,5 @@ class Manager(object):
             w.continue_working = False
             w.join()
 
-    @contextmanager
-    def db_transaction(self):
-        self.db.connect_db()
-
-        cursor = self.db.database.get_cursor()
-        try:
-            yield cursor
-        except:
-            self.db.database.rollback()
-            raise
-        else:
-            self.db.database.commit()
-        finally:
-            self.db.close_db(None)
-
     def _save_status(self):
         self.redis_connection.hmset('manager:status', self.status)

redash/data/query_runner.py

@@ -1,69 +1,30 @@
-"""
-QueryRunner is the function that the workers use, to execute queries. This is the Redshift
-(PostgreSQL in fact) version, but easily we can write another to support additional databases
-(MySQL and others).
-
-Because the worker just pass the query, this can be used with any data store that has some sort of
-query language (for example: HiveQL).
-"""
-import logging
 import json
-import sys
-import select
 
-import psycopg2
 
-from redash.utils import JSONEncoder
+def get_query_runner(connection_type, connection_string):
+    if connection_type == 'mysql':
+        from redash.data import query_runner_mysql
+        runner = query_runner_mysql.mysql(connection_string)
+    elif connection_type == 'graphite':
+        from redash.data import query_runner_graphite
+        connection_params = json.loads(connection_string)
+        if connection_params['auth']:
+            connection_params['auth'] = tuple(connection_params['auth'])
+        else:
+            connection_params['auth'] = None
+        runner = query_runner_graphite.graphite(connection_params)
+    elif connection_type == 'bigquery':
+        from redash.data import query_runner_bigquery
+        connection_params = json.loads(connection_string)
+        runner = query_runner_bigquery.bigquery(connection_params)
+    elif connection_type == 'script':
+        from redash.data import query_runner_script
+        runner = query_runner_script.script(connection_string)
+    elif connection_type == 'url':
+        from redash.data import query_runner_url
+        runner = query_runner_url.url(connection_string)
+    else:
+        from redash.data import query_runner_pg
+        runner = query_runner_pg.pg(connection_string)
 
-def redshift(connection_string):
-    def column_friendly_name(column_name):
-        return column_name
- 
-    def wait(conn):
-        while 1:
-            state = conn.poll()
-            if state == psycopg2.extensions.POLL_OK:
-                break
-            elif state == psycopg2.extensions.POLL_WRITE:
-                select.select([], [conn.fileno()], [])
-            elif state == psycopg2.extensions.POLL_READ:
-                select.select([conn.fileno()], [], [])
-            else:
-                raise psycopg2.OperationalError("poll() returned %s" % state)
- 
-    def query_runner(query):
-        connection = psycopg2.connect(connection_string, async=True)
-        wait(connection)
- 
-        cursor = connection.cursor()
- 
-        try:
-            cursor.execute(query)
-            wait(connection)
- 
-            column_names = [col.name for col in cursor.description]
- 
-            rows = [dict(zip(column_names, row)) for row in cursor]
-            columns = [{'name': col.name,
-                        'friendly_name': column_friendly_name(col.name),
-                        'type': None} for col in cursor.description]
- 
-            data = {'columns': columns, 'rows': rows}
-            json_data = json.dumps(data, cls=JSONEncoder)
-            error = None
-            cursor.close()
-        except psycopg2.DatabaseError as e:
-            json_data = None
-            error = e.message
-        except KeyboardInterrupt:
-            connection.cancel()
-            error = "Query cancelled by user."
-            json_data = None
-        except Exception as e:
-            raise sys.exc_info()[1], None, sys.exc_info()[2]
-        finally:
-            connection.close()
- 
-        return json_data, error
- 
-    return query_runner
+    return runner

redash/data/query_runner_bigquery.py

@@ -0,0 +1,98 @@
+import httplib2
+import json
+import logging
+import sys
+
+try:
+    import apiclient.errors
+    from apiclient.discovery import build
+    from apiclient.errors import HttpError
+    from oauth2client.client import SignedJwtAssertionCredentials
+except ImportError:
+    print "Missing dependencies. Please install google-api-python-client and oauth2client."
+    print "You can use pip:   pip install google-api-python-client oauth2client"
+
+from redash.utils import JSONEncoder
+
+def bigquery(connection_string):
+    def load_key(filename):
+        f = file(filename, "rb")
+        try:
+            return f.read()
+        finally:
+            f.close()
+
+    def get_bigquery_service():
+        scope = [
+            "https://www.googleapis.com/auth/bigquery",
+        ]
+
+        credentials = SignedJwtAssertionCredentials(connection_string["serviceAccount"], load_key(connection_string["privateKey"]), scope=scope)
+        http = httplib2.Http()
+        http = credentials.authorize(http)
+
+        return build("bigquery", "v2", http=http)
+
+    def query_runner(query):
+        bigquery_service = get_bigquery_service()
+
+        jobs = bigquery_service.jobs()
+        job_data = {
+            "configuration": {
+                "query": {
+                    "query": query,
+                }
+            }
+        }
+
+        logging.debug("bigquery got query: %s", query)
+
+        project_id = connection_string["projectId"]
+
+        try:
+            insert_response = jobs.insert(projectId=project_id, body=job_data).execute()
+            current_row = 0
+            query_reply = jobs.getQueryResults(projectId=project_id, jobId=insert_response['jobReference']['jobId'], startIndex=current_row).execute()
+
+            rows = []
+            field_names = []
+            for f in query_reply["schema"]["fields"]:
+                field_names.append(f["name"])
+
+            while(("rows" in query_reply) and current_row < query_reply['totalRows']):
+                for row in query_reply["rows"]:
+                    row_data = {}
+                    column_index = 0
+                    for cell in row["f"]:
+                        row_data[field_names[column_index]] = cell["v"]
+                        column_index += 1
+
+                    rows.append(row_data)
+
+                current_row += len(query_reply['rows'])
+                query_reply = jobs.getQueryResults(projectId=project_id, jobId=query_reply['jobReference']['jobId'], startIndex=current_row).execute()
+
+            columns = [{'name': name,
+                        'friendly_name': name,
+                        'type': None} for name in field_names]
+
+            data = {
+                "columns" : columns,
+                "rows" : rows
+            }
+            error = None
+
+            json_data = json.dumps(data, cls=JSONEncoder)
+        except apiclient.errors.HttpError, e:
+            json_data = None
+            error = e.args[1]
+        except KeyboardInterrupt:
+            error = "Query cancelled by user."
+            json_data = None
+        except Exception as e:
+            raise sys.exc_info()[1], None, sys.exc_info()[2]
+
+        return json_data, error
+
+
+    return query_runner

redash/data/query_runner_mysql.py

@@ -28,18 +28,24 @@ def mysql(connection_string):
             
             data = cursor.fetchall()
             
-            num_fields = len(cursor.description)
-            column_names = [i[0] for i in cursor.description]
+            cursor_desc = cursor.description
+            if (cursor_desc != None):
+                num_fields = len(cursor_desc)
+                column_names = [i[0] for i in cursor.description]
             
-            rows = [dict(zip(column_names, row)) for row in data]
+                rows = [dict(zip(column_names, row)) for row in data]
 
-            columns = [{'name': col_name,
-                        'friendly_name': col_name,
-                        'type': None} for col_name in column_names]
+                columns = [{'name': col_name,
+                            'friendly_name': col_name,
+                            'type': None} for col_name in column_names]
             
-            data = {'columns': columns, 'rows': rows}
-            json_data = json.dumps(data, cls=JSONEncoder)
-            error = None
+                data = {'columns': columns, 'rows': rows}
+                json_data = json.dumps(data, cls=JSONEncoder)
+                error = None
+            else:
+                json_data = None
+                error = "No data was returned."
+                
             cursor.close()
         except MySQLdb.Error, e:
             json_data = None

redash/data/query_runner_pg.py

@@ -0,0 +1,68 @@
+"""
+QueryRunner is the function that the workers use, to execute queries. This is the PostgreSQL
+version, but easily we can write another to support additional databases (MySQL and others).
+
+Because the worker just pass the query, this can be used with any data store that has some sort of
+query language (for example: HiveQL).
+"""
+import json
+import sys
+import select
+
+import psycopg2
+
+from redash.utils import JSONEncoder
+
+
+def pg(connection_string):
+    def column_friendly_name(column_name):
+        return column_name
+
+    def wait(conn):
+        while 1:
+            state = conn.poll()
+            if state == psycopg2.extensions.POLL_OK:
+                break
+            elif state == psycopg2.extensions.POLL_WRITE:
+                select.select([], [conn.fileno()], [])
+            elif state == psycopg2.extensions.POLL_READ:
+                select.select([conn.fileno()], [], [])
+            else:
+                raise psycopg2.OperationalError("poll() returned %s" % state)
+
+    def query_runner(query):
+        connection = psycopg2.connect(connection_string, async=True)
+        wait(connection)
+
+        cursor = connection.cursor()
+
+        try:
+            cursor.execute(query)
+            wait(connection)
+
+            column_names = [col.name for col in cursor.description]
+
+            rows = [dict(zip(column_names, row)) for row in cursor]
+            columns = [{'name': col.name,
+                        'friendly_name': column_friendly_name(col.name),
+                        'type': None} for col in cursor.description]
+
+            data = {'columns': columns, 'rows': rows}
+            json_data = json.dumps(data, cls=JSONEncoder)
+            error = None
+            cursor.close()
+        except psycopg2.DatabaseError as e:
+            json_data = None
+            error = e.message
+        except KeyboardInterrupt:
+            connection.cancel()
+            error = "Query cancelled by user."
+            json_data = None
+        except Exception as e:
+            raise sys.exc_info()[1], None, sys.exc_info()[2]
+        finally:
+            connection.close()
+
+        return json_data, error
+
+    return query_runner

redash/data/query_runner_script.py

@@ -0,0 +1,48 @@
+import json
+import logging
+import sys
+import os
+import subprocess
+
+# We use subprocess.check_output because we are lazy.
+# If someone will really want to run this on Python < 2.7 they can easily update the code to run
+# Popen, check the retcodes and other things and read the standard output to a variable.
+if not "check_output" in subprocess.__dict__:
+    print "ERROR: This runner uses subprocess.check_output function which exists in Python 2.7"
+
+def script(connection_string):
+
+    def query_runner(query):
+        try:
+            json_data = None
+            error = None
+
+            # Poor man's protection against running scripts from output the scripts directory
+            if connection_string.find("../") > -1:
+                return None, "Scripts can only be run from the configured scripts directory"
+
+            query = query.strip()
+
+            script = os.path.join(connection_string, query)
+            if not os.path.exists(script):
+                return None, "Script '%s' not found in script directory" % query
+
+            output = subprocess.check_output(script, shell=False)
+            if output != None:
+                output = output.strip()
+                if output != "":
+                    return output, None
+
+            error = "Error reading output"
+        except subprocess.CalledProcessError as e:
+            return None, str(e)
+        except KeyboardInterrupt:
+            error = "Query cancelled by user."
+            json_data = None
+        except Exception as e:
+            raise sys.exc_info()[1], None, sys.exc_info()[2]
+
+        return json_data, error
+
+    query_runner.annotate_query = False
+    return query_runner

redash/data/query_runner_url.py

@@ -0,0 +1,45 @@
+import json
+import logging
+import sys
+import os
+import urllib2
+
+def url(connection_string):
+
+    def query_runner(query):
+        base_url = connection_string
+
+        try:
+            json_data = None
+            error = None
+
+            query = query.strip()
+
+            if base_url is not None and base_url != "":
+                if query.find("://") > -1:
+                    return None, "Accepting only relative URLs to '%s'" % base_url
+
+            if base_url is None:
+                base_url = ""
+
+            url = base_url + query
+
+            json_data = urllib2.urlopen(url).read().strip()
+
+            if not json_data:
+                error = "Error reading data from '%s'" % url
+
+            return json_data, error
+
+        except urllib2.URLError as e:
+            return None, str(e)
+        except KeyboardInterrupt:
+            error = "Query cancelled by user."
+            json_data = None
+        except Exception as e:
+            raise sys.exc_info()[1], None, sys.exc_info()[2]
+
+        return json_data, error
+
+    query_runner.annotate_query = False
+    return query_runner

redash/data/worker.py

@@ -11,10 +11,83 @@ import time
 import signal
 import setproctitle
 import redis
+from statsd import StatsClient
 from redash.utils import gen_query_hash
+from redash.data.query_runner import get_query_runner
+from redash import settings
 
 
-class Job(object):
+class RedisObject(object):
+    # The following should be overriden in the inheriting class:
+    fields = {}
+    conversions = {}
+    id_field = ''
+    name = ''
+
+    def __init__(self, redis_connection, **kwargs):
+        self.redis_connection = redis_connection
+        self.values = {}
+
+        if not self.fields:
+            raise ValueError("You must set the fields dictionary, before using RedisObject.")
+
+        if not self.name:
+            raise ValueError("You must set the name, before using RedisObject")
+
+        self.update(**kwargs)
+
+    def __getattr__(self, name):
+        if name in self.values:
+            return self.values[name]
+        else:
+            raise AttributeError
+
+    def update(self, **kwargs):
+        for field, default_value in self.fields.iteritems():
+            value = kwargs.get(field, self.values.get(field, default_value))
+            if callable(value):
+                value = value()
+
+            if value == 'None':
+                value = None
+
+            if field in self.conversions and value:
+                value = self.conversions[field](value)
+
+            self.values[field] = value
+
+    @classmethod
+    def _redis_key(cls, object_id):
+        return '{}:{}'.format(cls.name, object_id)
+
+    def save(self, pipe):
+        if not pipe:
+            pipe = self.redis_connection.pipeline()
+
+        pipe.sadd('{}_set'.format(self.name), self.id)
+        pipe.hmset(self._redis_key(self.id), self.values)
+        pipe.publish(self._redis_key(self.id), json.dumps(self.to_dict()))
+
+        pipe.execute()
+
+    @classmethod
+    def load(cls, redis_connection, object_id):
+        object_dict = redis_connection.hgetall(cls._redis_key(object_id))
+        obj = None
+        if object_dict:
+            obj = cls(redis_connection, **object_dict)
+
+        return obj
+
+
+def fix_unicode(string):
+    if isinstance(string, unicode):
+        return string
+
+    return string.decode('utf-8')
+
+
+class Job(RedisObject):
     HIGH_PRIORITY = 1
     LOW_PRIORITY = 2
 
@@ -23,37 +96,43 @@ class Job(object):
     DONE = 3
     FAILED = 4
 
-    def __init__(self, redis_connection, query, priority,
-                 job_id=None,
-                 wait_time=None, query_time=None,
-                 updated_at=None, status=None, error=None, query_result_id=None,
-                 process_id=0):
-        self.redis_connection = redis_connection
-        self.query = query
-        self.priority = priority
-        self.query_hash = gen_query_hash(self.query)
-        self.query_result_id = query_result_id
-        if process_id == 'None':
-            self.process_id = None
-        else:
-            self.process_id = int(process_id)
+    fields = {
+        'id': lambda: str(uuid.uuid1()),
+        'query': None,
+        'priority': None,
+        'query_hash': None,
+        'wait_time': 0,
+        'query_time': 0,
+        'error': None,
+        'updated_at': time.time,
+        'status': WAITING,
+        'process_id': None,
+        'query_result_id': None,
+        'data_source_id': None,
+        'data_source_name': None,
+        'data_source_type': None,
+        'data_source_options': None
+    }
 
-        if job_id is None:
-            self.id = str(uuid.uuid1())
-            self.new_job = True
-            self.wait_time = 0
-            self.query_time = 0
-            self.error = None
-            self.updated_at = time.time() # job_dict.get('updated_at', time.time())
-            self.status = self.WAITING # int(job_dict.get('status', self.WAITING))
-        else:
-            self.id = job_id
-            self.new_job = False
-            self.error = error
-            self.wait_time = wait_time
-            self.query_time = query_time
-            self.updated_at = updated_at
-            self.status = status
+    conversions = {
+        'query': fix_unicode,
+        'priority': int,
+        'updated_at': float,
+        'status': int,
+        'wait_time': float,
+        'query_time': float,
+        'process_id': int,
+        'query_result_id': int
+    }
+
+    name = 'job'
+
+    def __init__(self, redis_connection, query, priority, **kwargs):
+        kwargs['query'] = fix_unicode(query)
+        kwargs['priority'] = priority
+        kwargs['query_hash'] = gen_query_hash(kwargs['query'])
+        self.new_job = 'id' not in kwargs
+        super(Job, self).__init__(redis_connection, **kwargs)
 
     def to_dict(self):
         return {
@@ -66,13 +145,11 @@ class Job(object):
             'status': self.status,
             'error': self.error,
             'query_result_id': self.query_result_id,
-            'process_id': self.process_id
+            'process_id': self.process_id,
+            'data_source_name': self.data_source_name,
+            'data_source_type': self.data_source_type
         }
 
-    @staticmethod
-    def _redis_key(job_id):
-        return 'job:%s' % job_id
-
     def cancel(self):
         # TODO: Race condition:
         # it's possible that it will be picked up by worker while processing the cancel order
@@ -94,16 +171,14 @@ class Job(object):
         if self.is_finished():
             pipe.delete('query_hash_job:%s' % self.query_hash)
 
-        pipe.sadd('jobs_set', self.id)
-        pipe.hmset(self._redis_key(self.id), self.to_dict())
-        pipe.publish(self._redis_key(self.id), json.dumps(self.to_dict()))
-        pipe.execute()
+        super(Job, self).save(pipe)
 
     def processing(self, process_id):
-        self.status = self.PROCESSING
-        self.process_id = process_id
-        self.wait_time = time.time() - self.updated_at
-        self.updated_at = time.time()
+        self.update(status=self.PROCESSING,
+                    process_id=process_id,
+                    wait_time=time.time() - self.updated_at,
+                    updated_at=time.time())
+
         self.save()
 
     def is_finished(self):
@@ -111,50 +186,37 @@ class Job(object):
 
     def done(self, query_result_id, error):
         if error:
-            self.status = self.FAILED
+            new_status = self.FAILED
         else:
-            self.status = self.DONE
+            new_status = self.DONE
+
+        self.update(status=new_status,
+                    query_result_id=query_result_id,
+                    error=error,
+                    query_time=time.time() - self.updated_at,
+                    updated_at=time.time())
 
-        self.query_result_id = query_result_id
-        self.error = error
-        self.query_time = time.time() - self.updated_at
-        self.updated_at = time.time()
         self.save()
 
     def __str__(self):
         return "<Job:%s,priority:%d,status:%d>" % (self.id, self.priority, self.status)
 
-    @classmethod
-    def _load(cls, redis_connection, job_id):
-        return redis_connection.hgetall(cls._redis_key(job_id))
-
-    @classmethod
-    def load(cls, redis_connection, job_id):
-        job_dict = cls._load(redis_connection, job_id)
-        job = None
-        if job_dict:
-            job = Job(redis_connection, job_id=job_dict['id'], query=job_dict['query'].decode('utf-8'),
-                      priority=int(job_dict['priority']), updated_at=float(job_dict['updated_at']),
-                      status=int(job_dict['status']), wait_time=float(job_dict['wait_time']),
-                      query_time=float(job_dict['query_time']), error=job_dict['error'],
-                      query_result_id=job_dict['query_result_id'],
-                      process_id=job_dict['process_id'])
-
-        return job
-
 
 class Worker(threading.Thread):
-    def __init__(self, manager, redis_connection_params, query_runner, sleep_time=0.1):
+    def __init__(self, worker_id, manager, redis_connection_params, sleep_time=0.1):
         self.manager = manager
 
+        self.statsd_client = StatsClient(host=settings.STATSD_HOST, port=settings.STATSD_PORT,
+                                         prefix=settings.STATSD_PREFIX)
         self.redis_connection_params = {k: v for k, v in redis_connection_params.iteritems()
                                         if k in ('host', 'db', 'password', 'port')}
+
         self.continue_working = True
-        self.query_runner = query_runner
         self.sleep_time = sleep_time
         self.child_pid = None
-        self.worker_id = uuid.uuid1()
+        self.worker_id = worker_id
         self.status = {
+            'id': self.worker_id,
             'jobs_count': 0,
             'cancelled_jobs_count': 0,
             'done_jobs_count': 0,
@@ -234,14 +296,23 @@ class Worker(threading.Thread):
         start_time = time.time()
         self.set_title("running query %s" % job_id)
 
-        if getattr(self.query_runner, 'annotate_query', True):
+        logging.info("[%s][%s] Loading query runner (%s, %s)...", self.name, job.id,
+                     job.data_source_name, job.data_source_type)
+
+        query_runner = get_query_runner(job.data_source_type, job.data_source_options)
+
+        if getattr(query_runner, 'annotate_query', True):
             annotated_query = "/* Pid: %s, Job Id: %s, Query hash: %s, Priority: %s */ %s" % \
                               (pid, job.id, job.query_hash, job.priority, job.query)
         else:
             annotated_query = job.query
 
         # TODO: here's the part that needs to be forked, not all of the worker process...
-        data, error = self.query_runner(annotated_query)
+        with self.statsd_client.timer('worker_{}.query_runner.{}.{}.run_time'.format(self.worker_id,
+                                                                                     job.data_source_type,
+                                                                                     job.data_source_name)):
+            data, error = query_runner(annotated_query)
+
         run_time = time.time() - start_time
         logging.info("[%s][%s] query finished... data length=%s, error=%s",
                      self.name, job.id, data and len(data), error)
@@ -251,7 +322,8 @@ class Worker(threading.Thread):
         query_result_id = None
         if not error:
             self.set_title("storing results %s" % job_id)
-            query_result_id = self.manager.store_query_result(job.query, data, run_time,
+            query_result_id = self.manager.store_query_result(job.data_source_id,
+                                                              job.query, data, run_time,
                                                               datetime.datetime.utcnow())
 
         self.set_title("marking job as done %s" % job_id)

redash/import_export.py

@@ -0,0 +1,158 @@
+import contextlib
+import json
+from redash import models
+from flask.ext.script import Manager
+
+
+class Importer(object):
+    def __init__(self, object_mapping=None, data_source=None):
+        if object_mapping is None:
+            object_mapping = {}
+        self.object_mapping = object_mapping
+        self.data_source = data_source
+
+    def import_query_result(self, query_result):
+        query_result = self._get_or_create(models.QueryResult, query_result['id'],
+                                           data_source=self.data_source,
+                                           data=json.dumps(query_result['data']),
+                                           query_hash=query_result['query_hash'],
+                                           retrieved_at=query_result['retrieved_at'],
+                                           query=query_result['query'],
+                                           runtime=query_result['runtime'])
+
+        return query_result
+
+
+    def import_query(self, user, query):
+        query_result = self.import_query_result(query['latest_query_data'])
+
+        new_query = self._get_or_create(models.Query, query['id'], name=query['name'],
+                                        user=user,
+                                        ttl=-1,
+                                        query=query['query'],
+                                        query_hash=query['query_hash'],
+                                        description=query['description'],
+                                        latest_query_data=query_result,
+                                        data_source=self.data_source)
+
+        return new_query
+
+
+    def import_visualization(self, user, visualization):
+        query = self.import_query(user, visualization['query'])
+
+        new_visualization = self._get_or_create(models.Visualization, visualization['id'],
+                                                name=visualization['name'],
+                                                description=visualization['description'],
+                                                type=visualization['type'],
+                                                options=json.dumps(visualization['options']),
+                                                query=query)
+        return new_visualization
+
+    def import_widget(self, dashboard, widget):
+        visualization = self.import_visualization(dashboard.user, widget['visualization'])
+
+        new_widget = self._get_or_create(models.Widget, widget['id'],
+                                         dashboard=dashboard,
+                                         width=widget['width'],
+                                         options=json.dumps(widget['options']),
+                                         visualization=visualization)
+
+        return new_widget
+
+    def import_dashboard(self, user, dashboard):
+        """
+        Imports dashboard along with widgets, visualizations and queries from another re:dash.
+
+        user - the user to associate all objects with.
+        dashboard - dashboard to import (can be result of loading a json output).
+        """
+
+        new_dashboard = self._get_or_create(models.Dashboard, dashboard['id'],
+                                            name=dashboard['name'],
+                                            slug=dashboard['slug'],
+                                            layout='[]',
+                                            user=user)
+
+        layout = []
+
+        for widgets in dashboard['widgets']:
+            row = []
+            for widget in widgets:
+                widget_id = self.import_widget(new_dashboard, widget).id
+                row.append(widget_id)
+
+            layout.append(row)
+
+        new_dashboard.layout = json.dumps(layout)
+        new_dashboard.save()
+
+        return new_dashboard
+
+    def _get_or_create(self, object_type, external_id, **properties):
+        internal_id = self._get_mapping(object_type, external_id)
+        if internal_id:
+            update = object_type.update(**properties).where(object_type.id == internal_id)
+            update.execute()
+            obj = object_type.get_by_id(internal_id)
+        else:
+            obj = object_type.create(**properties)
+            self._update_mapping(object_type, external_id, obj.id)
+
+        return obj
+
+    def _get_mapping(self, object_type, external_id):
+        self.object_mapping.setdefault(object_type.__name__, {})
+        return self.object_mapping[object_type.__name__].get(str(external_id), None)
+
+    def _update_mapping(self, object_type, external_id, internal_id):
+        self.object_mapping.setdefault(object_type.__name__, {})
+        self.object_mapping[object_type.__name__][str(external_id)] = internal_id
+
+import_manager = Manager(help="import utilities")
+export_manager = Manager(help="export utilities")
+
+
+@contextlib.contextmanager
+def importer_with_mapping_file(mapping_filename):
+    with open(mapping_filename) as f:
+        mapping = json.loads(f.read())
+
+    importer = Importer(object_mapping=mapping, data_source=get_data_source())
+    yield importer
+
+    with open(mapping_filename, 'w') as f:
+        f.write(json.dumps(importer.object_mapping, indent=2))
+
+
+def get_data_source():
+    try:
+        data_source = models.DataSource.get(models.DataSource.name=="Import")
+    except models.DataSource.DoestNotExist:
+        data_source = models.DataSource.create(name="Import", type="import", options='{}')
+
+    return data_source
+
+@import_manager.command
+def query(mapping_filename, query_filename, user_id):
+    user = models.User.get_by_id(user_id)
+    with open(query_filename) as f:
+        query = json.loads(f.read())
+
+    with importer_with_mapping_file(mapping_filename) as importer:
+        imported_query = importer.import_query(user, query)
+
+        print "New query id: {}".format(imported_query.id)
+
+
+@import_manager.command
+def dashboard(mapping_filename, dashboard_filename, user_id):
+    user = models.User.get_by_id(user_id)
+    with open(dashboard_filename) as f:
+        dashboard = json.loads(f.read())
+
+    with importer_with_mapping_file(mapping_filename) as importer:
+        importer.import_dashboard(user, dashboard)
+
+
+

redash/models.py

@@ -3,9 +3,10 @@ import hashlib
 import time
 import datetime
 from flask.ext.peewee.utils import slugify
-from flask.ext.login import UserMixin
+from flask.ext.login import UserMixin, AnonymousUserMixin
 from passlib.apps import custom_app_context as pwd_context
 import peewee
+from playhouse.postgres_ext import ArrayField
 from redash import db, utils
 
 
@@ -15,12 +16,31 @@ class BaseModel(db.Model):
         return cls.get(cls.id == model_id)
 
 
+class AnonymousUser(AnonymousUserMixin):
+    @property
+    def permissions(self):
+        return []
+
+
+class ApiUser(UserMixin):
+    def __init__(self, api_key):
+        self.id = api_key
+
+    @property
+    def permissions(self):
+        return ['view_query']
+
+
 class User(BaseModel, UserMixin):
+    DEFAULT_PERMISSIONS = ['create_dashboard', 'create_query', 'edit_dashboard', 'edit_query',
+                           'view_query', 'view_source', 'execute_query']
+
     id = peewee.PrimaryKeyField()
     name = peewee.CharField(max_length=320)
     email = peewee.CharField(max_length=320, index=True, unique=True)
     password_hash = peewee.CharField(max_length=128, null=True)
     is_admin = peewee.BooleanField(default=False)
+    permissions = ArrayField(peewee.CharField, default=DEFAULT_PERMISSIONS)
 
     class Meta:
         db_table = 'users'
@@ -43,8 +63,52 @@ class User(BaseModel, UserMixin):
         return self.password_hash and pwd_context.verify(password, self.password_hash)
 
 
-class QueryResult(db.Model):
+class ActivityLog(BaseModel):
+    QUERY_EXECUTION = 1
+    
     id = peewee.PrimaryKeyField()
+    user = peewee.ForeignKeyField(User)
+    type = peewee.IntegerField()
+    activity = peewee.TextField()
+    created_at = peewee.DateTimeField(default=datetime.datetime.now)
+
+    class Meta:
+        db_table = 'activity_log'
+
+    def to_dict(self):
+        return {
+            'id': self.id,
+            'user': self.user.to_dict(),
+            'type': self.type,
+            'activity': self.activity,
+            'created_at': self.created_at
+        }
+
+    def __unicode__(self):
+        return unicode(self.id)
+
+
+class DataSource(BaseModel):
+    id = peewee.PrimaryKeyField()
+    name = peewee.CharField()
+    type = peewee.CharField()
+    options = peewee.TextField()
+    created_at = peewee.DateTimeField(default=datetime.datetime.now)
+
+    class Meta:
+        db_table = 'data_sources'
+
+    def to_dict(self):
+        return {
+            'id': self.id,
+            'name': self.name,
+            'type': self.type
+        }
+
+
+class QueryResult(BaseModel):
+    id = peewee.PrimaryKeyField()
+    data_source = peewee.ForeignKeyField(DataSource)
     query_hash = peewee.CharField(max_length=32, index=True)
     query = peewee.TextField()
     data = peewee.TextField()
@@ -60,16 +124,27 @@ class QueryResult(db.Model):
             'query_hash': self.query_hash,
             'query': self.query,
             'data': json.loads(self.data),
+            'data_source_id': self._data.get('data_source', None),
             'runtime': self.runtime,
             'retrieved_at': self.retrieved_at
         }
 
+    @classmethod
+    def get_latest(cls, data_source, query, ttl=0):
+        query_hash = utils.gen_query_hash(query)
+
+        query = cls.select().where(cls.query_hash == query_hash, cls.data_source == data_source,
+                                   peewee.SQL("retrieved_at + interval '%s second' >= now() at time zone 'utc'", ttl)).order_by(cls.retrieved_at.desc())
+
+        return query.first()
+
     def __unicode__(self):
         return u"%d | %s | %s" % (self.id, self.query_hash, self.retrieved_at)
 
 
 class Query(BaseModel):
     id = peewee.PrimaryKeyField()
+    data_source = peewee.ForeignKeyField(DataSource)
     latest_query_data = peewee.ForeignKeyField(QueryResult, null=True)
     name = peewee.CharField(max_length=255)
     description = peewee.CharField(max_length=4096, null=True)
@@ -101,6 +176,7 @@ class Query(BaseModel):
             'ttl': self.ttl,
             'api_key': self.api_key,
             'created_at': self.created_at,
+            'data_source_id': self._data.get('data_source', None)
         }
 
         if with_user:
@@ -246,7 +322,7 @@ class Visualization(BaseModel):
         return u"%s %s" % (self.id, self.type)
 
 
-class Widget(db.Model):
+class Widget(BaseModel):
     id = peewee.PrimaryKeyField()
     visualization = peewee.ForeignKeyField(Visualization, related_name='widgets')
 
@@ -274,7 +350,7 @@ class Widget(db.Model):
     def __unicode__(self):
         return u"%s" % self.id
 
-all_models = (User, QueryResult, Query, Dashboard, Visualization, Widget)
+all_models = (DataSource, User, QueryResult, Query, Dashboard, Visualization, Widget, ActivityLog)
 
 
 def create_db(create_tables, drop_tables):
@@ -286,7 +362,7 @@ def create_db(create_tables, drop_tables):
             db.database.execute_sql('DROP TABLE %s CASCADE' % model._meta.db_table)
             #model.drop_table()
 
-        if create_tables:
+        if create_tables and not model.table_exists():
             model.create_table()
 
     db.close_db(None)

redash/permissions.py

@@ -0,0 +1,27 @@
+import functools
+from flask.ext.login import current_user
+from flask.ext.restful import abort
+
+
+class require_permissions(object):
+    def __init__(self, permissions):
+        self.permissions = permissions
+
+    def __call__(self, fn):
+        @functools.wraps(fn)
+        def decorated(*args, **kwargs):
+            has_permissions = reduce(lambda a, b: a and b,
+                                      map(lambda permission: permission in current_user.permissions,
+                                          self.permissions),
+                                      True)
+
+            if has_permissions:
+                return fn(*args, **kwargs)
+            else:
+                abort(403)
+
+        return decorated
+
+
+def require_permission(permission):
+    return require_permissions((permission,))

redash/settings.py

@@ -40,12 +40,14 @@ def parse_boolean(str):
 
 REDIS_URL = os.environ.get('REDASH_REDIS_URL', "redis://localhost:6379")
 
-# "pg", "graphite" or "mysql"
+STATSD_HOST = os.environ.get('REDASH_STATSD_HOST', "127.0.0.1")
+STATSD_PORT = int(os.environ.get('REDASH_STATSD_PORT', "8125"))
+STATSD_PREFIX = os.environ.get('REDASH_STATSD_PREFIX', "redash")
+
+NAME = os.environ.get('REDASH_NAME', 're:dash')
+
+# The following is kept for backward compatability, and shouldn't be used any more.
 CONNECTION_ADAPTER = os.environ.get("REDASH_CONNECTION_ADAPTER", "pg")
-# Connection string for the database that is used to run queries against. Examples:
-# -- mysql:    CONNECTION_STRING = "Server=;User=;Pwd=;Database="
-# -- pg:       CONNECTION_STRING = "user= password= host= port=5439 dbname="
-# -- graphite: CONNECTION_STRING = {"url": "https://graphite.yourcompany.com", "auth": ["user", "password"], "verify": true}
 CONNECTION_STRING = os.environ.get("REDASH_CONNECTION_STRING", "user= password= host= port=5439 dbname=")
 
 # Connection settings for re:dash's own database (where we store the queries, results, etc)

requirements.txt

@@ -12,7 +12,7 @@ atfork==0.1.2
 blinker==1.3
 flask-peewee==0.6.5
 itsdangerous==0.23
-peewee==2.2.0
+peewee==2.2.2
 psycopg2==2.5.1
 python-dateutil==2.1
 pytz==2013.9
@@ -26,3 +26,4 @@ wsgiref==0.1.2
 wtf-peewee==0.2.2
 Flask-Script==0.6.6
 honcho==0.5.0
+statsd==2.1.2

tests/__init__.py

@@ -1,3 +1,4 @@
+import logging
 from unittest import TestCase
 from redash import settings, db, app
 import redash.models
@@ -11,6 +12,8 @@ settings.DATABASE_CONFIG = {
 app.config['DATABASE'] = settings.DATABASE_CONFIG
 db.load_database()
 
+logging.getLogger('peewee').setLevel(logging.INFO)
+
 for model in redash.models.all_models:
     model._meta.database = db.database
 

tests/factories.py

@@ -1,5 +1,6 @@
 import datetime
 import redash.models
+from redash.utils import gen_query_hash
 
 
 class ModelFactory(object):
@@ -43,6 +44,12 @@ user_factory = ModelFactory(redash.models.User,
                             is_admin=False)
 
 
+data_source_factory = ModelFactory(redash.models.DataSource,
+                                   name='Test',
+                                   type='pg',
+                                   options='')
+
+
 dashboard_factory = ModelFactory(redash.models.Dashboard,
                                  name='test', user=user_factory.create, layout='[]')
 
@@ -52,14 +59,16 @@ query_factory = ModelFactory(redash.models.Query,
                              description='',
                              query='SELECT 1',
                              ttl=-1,
-                             user=user_factory.create)
+                             user=user_factory.create,
+                             data_source=data_source_factory.create)
 
 query_result_factory = ModelFactory(redash.models.QueryResult,
                                     data='{"columns":{}, "rows":[]}',
                                     runtime=1,
-                                    retrieved_at=datetime.datetime.now(),
-                                    query=query_factory.create,
-                                    query_hash='')
+                                    retrieved_at=datetime.datetime.utcnow,
+                                    query="SELECT 1",
+                                    query_hash=gen_query_hash('SELECT 1'),
+                                    data_source=data_source_factory.create)
 
 visualization_factory = ModelFactory(redash.models.Visualization,
                                      type='CHART',
@@ -73,4 +82,4 @@ widget_factory = ModelFactory(redash.models.Widget,
                               width=1,
                               options='{}',
                               dashboard=dashboard_factory.create,
-                              visualization=visualization_factory.create)
+                              visualization=visualization_factory.create)

tests/test_controllers.py

@@ -7,7 +7,7 @@ from flask.ext.login import current_user
 from mock import patch
 from tests import BaseTestCase
 from tests.factories import dashboard_factory, widget_factory, visualization_factory, query_factory, \
-    query_result_factory, user_factory
+    query_result_factory, user_factory, data_source_factory
 from redash import app, models, settings
 from redash.utils import json_dumps
 from redash.authentication import sign
@@ -75,10 +75,22 @@ class IndexTest(BaseTestCase, AuthenticationTestMixin):
         super(IndexTest, self).setUp()
 
 
-class StatusTest(BaseTestCase, AuthenticationTestMixin):
-    def setUp(self):
-        self.paths = ['/status.json']
-        super(StatusTest, self).setUp()
+class StatusTest(BaseTestCase):
+    def test_returns_data_for_admin(self):
+        admin = user_factory.create(permissions=['admin'])
+        with app.test_client() as c, authenticated_user(c, user=admin):
+            rv = c.get('/status.json')
+            self.assertEqual(rv.status_code, 200)
+
+    def test_returns_403_for_non_admin(self):
+        with app.test_client() as c, authenticated_user(c):
+            rv = c.get('/status.json')
+            self.assertEqual(rv.status_code, 403)
+
+    def test_redirects_non_authenticated_user(self):
+        with app.test_client() as c:
+            rv = c.get('/status.json')
+            self.assertEqual(rv.status_code, 302)
 
 
 class DashboardAPITest(BaseTestCase, AuthenticationTestMixin):
@@ -199,10 +211,12 @@ class QueryAPITest(BaseTestCase, AuthenticationTestMixin):
 
     def test_create_query(self):
         user = user_factory.create()
+        data_source = data_source_factory.create()
         query_data = {
             'name': 'Testing',
             'query': 'SELECT 1',
-            'ttl': 3600
+            'ttl': 3600,
+            'data_source_id': data_source.id
         }
 
         with app.test_client() as c, authenticated_user(c, user=user):
@@ -292,12 +306,13 @@ class CsvQueryResultAPITest(BaseTestCase, AuthenticationTestMixin):
         super(CsvQueryResultAPITest, self).setUp()
         self.paths = []
         self.query_result = query_result_factory.create()
-        self.path = '/api/queries/{0}/results/{1}.csv'.format(self.query_result.query.id, self.query_result.id)
+        self.query = query_factory.create()
+        self.path = '/api/queries/{0}/results/{1}.csv'.format(self.query.id, self.query_result.id)
 
     # TODO: factor out the HMAC authentication tests
 
     def signature(self, expires):
-        return sign(self.query_result.query.api_key, self.path, expires)
+        return sign(self.query.api_key, self.path, expires)
 
     def test_redirect_when_unauthenticated(self):
         with app.test_client() as c:
@@ -306,34 +321,34 @@ class CsvQueryResultAPITest(BaseTestCase, AuthenticationTestMixin):
 
     def test_redirect_for_wrong_signature(self):
         with app.test_client() as c:
-            rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query_result.query.id, self.query_result.id), query_string={'signature': 'whatever', 'expires': 0})
+            rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query.id, self.query_result.id), query_string={'signature': 'whatever', 'expires': 0})
             self.assertEquals(rv.status_code, 302)
 
     def test_redirect_for_correct_signature_and_wrong_expires(self):
         with app.test_client() as c:
-            rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query_result.query.id, self.query_result.id), query_string={'signature': self.signature(0), 'expires': 0})
+            rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query.id, self.query_result.id), query_string={'signature': self.signature(0), 'expires': 0})
             self.assertEquals(rv.status_code, 302)
 
     def test_redirect_for_correct_signature_and_no_expires(self):
         with app.test_client() as c:
-            rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query_result.query.id, self.query_result.id), query_string={'signature': self.signature(time.time()+3600)})
+            rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query.id, self.query_result.id), query_string={'signature': self.signature(time.time()+3600)})
             self.assertEquals(rv.status_code, 302)
 
     def test_redirect_for_correct_signature_and_expires_too_long(self):
         with app.test_client() as c:
             expires = time.time()+(10*3600)
-            rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query_result.query.id, self.query_result.id), query_string={'signature': self.signature(expires), 'expires': expires})
+            rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query.id, self.query_result.id), query_string={'signature': self.signature(expires), 'expires': expires})
             self.assertEquals(rv.status_code, 302)
 
     def test_returns_200_for_correct_signature(self):
         with app.test_client() as c:
-            expires = time.time()+3600
-            rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query_result.query.id, self.query_result.id), query_string={'signature': self.signature(expires), 'expires': expires})
+            expires = time.time()+1800
+            rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query.id, self.query_result.id), query_string={'signature': self.signature(expires), 'expires': expires})
             self.assertEquals(rv.status_code, 200)
 
     def test_returns_200_for_authenticated_user(self):
         with app.test_client() as c, authenticated_user(c):
-            rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query_result.query.id, self.query_result.id))
+            rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query.id, self.query_result.id))
             self.assertEquals(rv.status_code, 200)
 
 

tests/test_import.py

@@ -0,0 +1,53 @@
+import json
+import os.path
+from tests import BaseTestCase
+from redash import models
+from redash import import_export
+from factories import user_factory, dashboard_factory, data_source_factory
+
+
+class ImportTest(BaseTestCase):
+    def setUp(self):
+        super(ImportTest, self).setUp()
+
+        with open(os.path.join(os.path.dirname(__file__), 'flights.json')) as f:
+            self.dashboard = json.loads(f.read())
+            self.user = user_factory.create()
+
+    def test_imports_dashboard_correctly(self):
+        importer = import_export.Importer(data_source=data_source_factory.create())
+        dashboard = importer.import_dashboard(self.user, self.dashboard)
+
+        self.assertIsNotNone(dashboard)
+        self.assertEqual(dashboard.name, self.dashboard['name'])
+        self.assertEqual(dashboard.slug, self.dashboard['slug'])
+        self.assertEqual(dashboard.user, self.user)
+
+        self.assertEqual(dashboard.widgets.count(),
+                         reduce(lambda s, row: s + len(row), self.dashboard['widgets'], 0))
+
+        self.assertEqual(models.Visualization.select().count(), dashboard.widgets.count())
+        self.assertEqual(models.Query.select().count(), dashboard.widgets.count()-1)
+        self.assertEqual(models.QueryResult.select().count(), dashboard.widgets.count()-1)
+
+    def test_imports_updates_existing_models(self):
+        importer = import_export.Importer(data_source=data_source_factory.create())
+        importer.import_dashboard(self.user, self.dashboard)
+
+        self.dashboard['name'] = 'Testing #2'
+        dashboard = importer.import_dashboard(self.user, self.dashboard)
+        self.assertEqual(dashboard.name, self.dashboard['name'])
+        self.assertEquals(models.Dashboard.select().count(), 1)
+
+    def test_using_existing_mapping(self):
+        dashboard = dashboard_factory.create()
+        mapping = {
+            'Dashboard': {
+                "1": dashboard.id
+            }
+        }
+
+        importer = import_export.Importer(object_mapping=mapping, data_source=data_source_factory.create())
+        imported_dashboard = importer.import_dashboard(self.user, self.dashboard)
+
+        self.assertEqual(imported_dashboard, dashboard)

tests/test_job.py

@@ -0,0 +1,100 @@
+# coding=utf-8
+
+import time
+from unittest import TestCase
+from mock import patch
+from redash.data.worker import Job
+from redash import redis_connection
+from redash.utils import gen_query_hash
+
+
+class TestJob(TestCase):
+    def setUp(self):
+        self.priority = 1
+        self.query = "SELECT 1"
+        self.query_hash = gen_query_hash(self.query)
+
+    def test_job_creation(self):
+        now = time.time()
+        with patch('time.time', return_value=now):
+            job = Job(redis_connection, query=self.query, priority=self.priority)
+        self.assertIsNotNone(job.id)
+        self.assertTrue(job.new_job)
+        self.assertEquals(0, job.wait_time)
+        self.assertEquals(0, job.query_time)
+        self.assertEquals(None, job.process_id)
+        self.assertEquals(Job.WAITING, job.status)
+        self.assertEquals(self.priority, job.priority)
+        self.assertEquals(self.query, job.query)
+        self.assertEquals(self.query_hash, job.query_hash)
+        self.assertIsNone(job.error)
+        self.assertIsNone(job.query_result_id)
+
+    def test_job_loading(self):
+        job = Job(redis_connection, query=self.query, priority=self.priority)
+        job.save()
+
+        loaded_job = Job.load(redis_connection, job.id)
+
+        self.assertFalse(loaded_job.new_job)
+
+        self.assertEquals(loaded_job.id, job.id)
+        self.assertEquals(loaded_job.wait_time, job.wait_time)
+        self.assertEquals(loaded_job.query_time, job.query_time)
+        self.assertEquals(loaded_job.process_id, job.process_id)
+        self.assertEquals(loaded_job.status, job.status)
+        self.assertEquals(loaded_job.priority, job.priority)
+        self.assertEquals(loaded_job.query_hash, job.query_hash)
+        self.assertEquals(loaded_job.query, job.query)
+        self.assertEquals(loaded_job.error, job.error)
+        self.assertEquals(loaded_job.query_result_id, job.query_result_id)
+
+    def test_update(self):
+        job = Job(redis_connection, query=self.query, priority=self.priority)
+
+        job.update(process_id=1)
+        self.assertEquals(1, job.process_id)
+        self.assertEquals(self.query, job.query)
+        self.assertEquals(self.priority, job.priority)
+
+    def test_processing(self):
+        job = Job(redis_connection, query=self.query, priority=self.priority)
+
+        updated_at = job.updated_at
+        now = time.time()+10
+        with patch('time.time', return_value=now):
+            job.processing(10)
+
+        job = Job.load(redis_connection, job.id)
+
+        self.assertEquals(10, job.process_id)
+        self.assertEquals(Job.PROCESSING, job.status)
+        self.assertEquals(now, job.updated_at)
+        self.assertEquals(now - updated_at, job.wait_time)
+
+    def test_done(self):
+        job = Job(redis_connection, query=self.query, priority=self.priority)
+
+        updated_at = job.updated_at
+        now = time.time()+10
+        with patch('time.time', return_value=now):
+            job.done(1, None)
+
+        job = Job.load(redis_connection, job.id)
+
+        self.assertEquals(Job.DONE, job.status)
+        self.assertEquals(1, job.query_result_id)
+        self.assertEquals(now, job.updated_at)
+        self.assertEquals(now - updated_at, job.query_time)
+        self.assertIsNone(job.error)
+
+    def test_unicode_serialization(self):
+        unicode_query = u"יוניקוד"
+        job = Job(redis_connection, query=unicode_query, priority=self.priority)
+
+        self.assertEquals(job.query, unicode_query)
+
+        job.save()
+        loaded_job = Job.load(redis_connection, job.id)
+        self.assertEquals(loaded_job.query, unicode_query)
+

tests/test_manager.py

@@ -0,0 +1,149 @@
+import datetime
+from mock import patch, call
+from tests import BaseTestCase
+from redash.data import worker
+from redash import data_manager, models
+from tests.factories import query_factory, query_result_factory, data_source_factory
+from redash.utils import gen_query_hash
+
+
+class TestManagerRefresh(BaseTestCase):
+    def test_enqueues_outdated_queries(self):
+        query = query_factory.create(ttl=60)
+        retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10)
+        query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query,
+                                                   query_hash=query.query_hash)
+        query.latest_query_data = query_result
+        query.save()
+
+        with patch('redash.data.Manager.add_job') as add_job_mock:
+            data_manager.refresh_queries()
+            add_job_mock.assert_called_with(query.query, worker.Job.LOW_PRIORITY, query.data_source)
+
+    def test_skips_fresh_queries(self):
+        query = query_factory.create(ttl=1200)
+        retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10)
+        query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query,
+                                                   query_hash=query.query_hash)
+
+        with patch('redash.data.Manager.add_job') as add_job_mock:
+            data_manager.refresh_queries()
+            self.assertFalse(add_job_mock.called)
+
+    def test_skips_queries_with_no_ttl(self):
+        query = query_factory.create(ttl=-1)
+        retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10)
+        query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query,
+                                                   query_hash=query.query_hash)
+
+        with patch('redash.data.Manager.add_job') as add_job_mock:
+            data_manager.refresh_queries()
+            self.assertFalse(add_job_mock.called)
+
+    def test_enqueues_query_only_once(self):
+        query = query_factory.create(ttl=60)
+        query2 = query_factory.create(ttl=60, query=query.query, query_hash=query.query_hash,
+                                      data_source=query.data_source)
+        retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10)
+        query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query,
+                                                   query_hash=query.query_hash)
+        query.latest_query_data = query_result
+        query2.latest_query_data = query_result
+        query.save()
+        query2.save()
+
+        with patch('redash.data.Manager.add_job') as add_job_mock:
+            data_manager.refresh_queries()
+            add_job_mock.assert_called_once_with(query.query, worker.Job.LOW_PRIORITY, query.data_source)
+
+    def test_enqueues_query_with_correct_data_source(self):
+        query = query_factory.create(ttl=60)
+        query2 = query_factory.create(ttl=60, query=query.query, query_hash=query.query_hash)
+        retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10)
+        query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query,
+                                                   query_hash=query.query_hash)
+        query.latest_query_data = query_result
+        query2.latest_query_data = query_result
+        query.save()
+        query2.save()
+
+        with patch('redash.data.Manager.add_job') as add_job_mock:
+            data_manager.refresh_queries()
+            add_job_mock.assert_has_calls([call(query2.query, worker.Job.LOW_PRIORITY, query2.data_source), call(query.query, worker.Job.LOW_PRIORITY, query.data_source)], any_order=True)
+            self.assertEquals(2, add_job_mock.call_count)
+
+    def test_enqueues_only_for_relevant_data_source(self):
+        query = query_factory.create(ttl=60)
+        query2 = query_factory.create(ttl=3600, query=query.query, query_hash=query.query_hash)
+        retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10)
+        query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query,
+                                                   query_hash=query.query_hash)
+        query.latest_query_data = query_result
+        query2.latest_query_data = query_result
+        query.save()
+        query2.save()
+
+        with patch('redash.data.Manager.add_job') as add_job_mock:
+            data_manager.refresh_queries()
+            add_job_mock.assert_called_once_with(query.query, worker.Job.LOW_PRIORITY, query.data_source)
+
+
+class TestManagerStoreResults(BaseTestCase):
+    def setUp(self):
+        super(TestManagerStoreResults, self).setUp()
+        self.data_source = data_source_factory.create()
+        self.query = "SELECT 1"
+        self.query_hash = gen_query_hash(self.query)
+        self.runtime = 123
+        self.utcnow = datetime.datetime.utcnow()
+        self.data = "data"
+
+    def test_stores_the_result(self):
+        query_result_id = data_manager.store_query_result(self.data_source.id, self.query,
+                                                          self.data, self.runtime, self.utcnow)
+
+        query_result = models.QueryResult.get_by_id(query_result_id)
+
+        self.assertEqual(query_result.data, self.data)
+        self.assertEqual(query_result.runtime, self.runtime)
+        self.assertEqual(query_result.retrieved_at, self.utcnow)
+        self.assertEqual(query_result.query, self.query)
+        self.assertEqual(query_result.query_hash, self.query_hash)
+        self.assertEqual(query_result.data_source, self.data_source)
+
+    def test_updates_existing_queries(self):
+        query1 = query_factory.create(query=self.query, data_source=self.data_source)
+        query2 = query_factory.create(query=self.query, data_source=self.data_source)
+        query3 = query_factory.create(query=self.query, data_source=self.data_source)
+
+        query_result_id = data_manager.store_query_result(self.data_source.id, self.query,
+                                                          self.data, self.runtime, self.utcnow)
+
+        self.assertEqual(models.Query.get_by_id(query1.id)._data['latest_query_data'], query_result_id)
+        self.assertEqual(models.Query.get_by_id(query2.id)._data['latest_query_data'], query_result_id)
+        self.assertEqual(models.Query.get_by_id(query3.id)._data['latest_query_data'], query_result_id)
+
+    def test_doesnt_update_queries_with_different_hash(self):
+        query1 = query_factory.create(query=self.query, data_source=self.data_source)
+        query2 = query_factory.create(query=self.query, data_source=self.data_source)
+        query3 = query_factory.create(query=self.query + "123", data_source=self.data_source)
+
+        query_result_id = data_manager.store_query_result(self.data_source.id, self.query,
+                                                          self.data, self.runtime, self.utcnow)
+
+        self.assertEqual(models.Query.get_by_id(query1.id)._data['latest_query_data'], query_result_id)
+        self.assertEqual(models.Query.get_by_id(query2.id)._data['latest_query_data'], query_result_id)
+        self.assertNotEqual(models.Query.get_by_id(query3.id)._data['latest_query_data'], query_result_id)
+
+
+    def test_doesnt_update_queries_with_different_data_source(self):
+        query1 = query_factory.create(query=self.query, data_source=self.data_source)
+        query2 = query_factory.create(query=self.query, data_source=self.data_source)
+        query3 = query_factory.create(query=self.query, data_source=data_source_factory.create())
+
+        query_result_id = data_manager.store_query_result(self.data_source.id, self.query,
+                                                          self.data, self.runtime, self.utcnow)
+
+        self.assertEqual(models.Query.get_by_id(query1.id)._data['latest_query_data'], query_result_id)
+        self.assertEqual(models.Query.get_by_id(query2.id)._data['latest_query_data'], query_result_id)
+        self.assertNotEqual(models.Query.get_by_id(query3.id)._data['latest_query_data'], query_result_id)

tests/test_models.py

@@ -1,6 +1,7 @@
+import datetime
 from tests import BaseTestCase
 from redash import models
-from factories import dashboard_factory, query_factory
+from factories import dashboard_factory, query_factory, data_source_factory, query_result_factory
 
 
 class DashboardTest(BaseTestCase):
@@ -25,4 +26,58 @@ class QueryTest(BaseTestCase):
 
         q = models.Query.get_by_id(q.id)
 
-        self.assertNotEquals(old_hash, q.query_hash)
+        self.assertNotEquals(old_hash, q.query_hash)
+
+
+class QueryResultTest(BaseTestCase):
+    def setUp(self):
+        super(QueryResultTest, self).setUp()
+
+    def test_get_latest_returns_none_if_not_found(self):
+        ds = data_source_factory.create()
+        found_query_result = models.QueryResult.get_latest(ds, "SELECT 1", 60)
+        self.assertIsNone(found_query_result)
+
+    def test_get_latest_returns_when_found(self):
+        qr = query_result_factory.create()
+        found_query_result = models.QueryResult.get_latest(qr.data_source, qr.query, 60)
+
+        self.assertEqual(qr, found_query_result)
+
+    def test_get_latest_works_with_data_source_id(self):
+        qr = query_result_factory.create()
+        found_query_result = models.QueryResult.get_latest(qr.data_source.id, qr.query, 60)
+
+        self.assertEqual(qr, found_query_result)
+
+    def test_get_latest_doesnt_return_query_from_different_data_source(self):
+        qr = query_result_factory.create()
+        data_source = data_source_factory.create()
+        found_query_result = models.QueryResult.get_latest(data_source, qr.query, 60)
+
+        self.assertIsNone(found_query_result)
+
+    def test_get_latest_doesnt_return_if_ttl_expired(self):
+        yesterday = datetime.datetime.now() - datetime.timedelta(days=1)
+        qr = query_result_factory.create(retrieved_at=yesterday)
+
+        found_query_result = models.QueryResult.get_latest(qr.data_source, qr.query, ttl=60)
+
+        self.assertIsNone(found_query_result)
+
+    def test_get_latest_returns_if_ttl_not_expired(self):
+        yesterday = datetime.datetime.now() - datetime.timedelta(seconds=30)
+        qr = query_result_factory.create(retrieved_at=yesterday)
+
+        found_query_result = models.QueryResult.get_latest(qr.data_source, qr.query, ttl=120)
+
+        self.assertEqual(found_query_result, qr)
+
+    def test_get_latest_returns_the_most_recent_result(self):
+        yesterday = datetime.datetime.now() - datetime.timedelta(seconds=30)
+        old_qr = query_result_factory.create(retrieved_at=yesterday)
+        qr = query_result_factory.create()
+
+        found_query_result = models.QueryResult.get_latest(qr.data_source, qr.query, 60)
+
+        self.assertEqual(found_query_result.id, qr.id)