Fix repo name

Feature: schema browser and simple autocomplete

Makefile

@@ -1,6 +1,7 @@
 NAME=redash
 VERSION=`python ./manage.py version`
 FULL_VERSION=$(VERSION)+b$(CIRCLE_BUILD_NUM)
+BASE_VERSION=$(shell python ./manage.py version | cut -d + -f 1)
 # VERSION gets evaluated every time it's referenced, therefore we need to use VERSION here instead of FULL_VERSION.
 FILENAME=$(CIRCLE_ARTIFACTS)/$(NAME).$(VERSION).tar.gz
 
@@ -15,8 +16,8 @@ pack:
 	tar -zcv -f $(FILENAME) --exclude=".git*" --exclude="*.pyc" --exclude="*.pyo" --exclude="venv" --exclude="rd_ui/node_modules" --exclude="rd_ui/dist/bower_components" --exclude="rd_ui/app" *
 
 upload:
-	python bin/upload_version.py $(VERSION) $(FILENAME)
+	python bin/release_manager.py $(CIRCLE_SHA1) $(BASE_VERSION) $(FILENAME)
 
 test:
 	nosetests --with-coverage --cover-package=redash tests/*.py
-	cd rd_ui && grunt test
+	#cd rd_ui && grunt test

bin/latest_release.py

@@ -1,30 +0,0 @@
-#!/usr/bin/env python
-import sys
-import requests
-
-if __name__ == '__main__':
-    response = requests.get('https://api.github.com/repos/EverythingMe/redash/releases')
-
-    if response.status_code != 200:
-        exit("Failed getting releases (status code: %s)." % response.status_code)
-
-    sorted_releases = sorted(response.json(), key=lambda release: release['id'], reverse=True)
-
-    latest_release = sorted_releases[0]
-    asset_url = latest_release['assets'][0]['url']
-    filename = latest_release['assets'][0]['name']
-
-    wget_command = 'wget --header="Accept: application/octet-stream" %s -O %s' % (asset_url, filename)
-
-    if '--url-only' in sys.argv:
-        print asset_url
-    elif '--wget' in sys.argv:
-        print wget_command
-    else:
-        print "Latest release: %s" % latest_release['tag_name']
-        print latest_release['body']
-
-        print "\nTarball URL: %s" % asset_url
-        print 'wget: %s' % (wget_command)
-
-

bin/release_manager.py

@@ -0,0 +1,130 @@
+import os
+import sys
+import json
+import re
+import subprocess
+import requests
+
+github_token = os.environ['GITHUB_TOKEN']
+auth = (github_token, 'x-oauth-basic')
+repo = 'EverythingMe/redash'
+
+def _github_request(method, path, params=None, headers={}):
+    if not path.startswith('https://api.github.com'):
+        url = "https://api.github.com/{}".format(path)
+    else:
+        url = path
+
+    if params is not None:
+        params = json.dumps(params)
+
+    response = requests.request(method, url, data=params, auth=auth)
+    return response
+
+def exception_from_error(message, response):
+    return Exception("({}) {}: {}".format(response.status_code, message, response.json().get('message', '?')))
+
+def rc_tag_name(version):
+    return "v{}-rc".format(version)
+
+def get_rc_release(version):
+    tag = rc_tag_name(version)
+    response = _github_request('get', 'repos/{}/releases/tags/{}'.format(repo, tag))
+
+    if response.status_code == 404:
+        return None
+    elif response.status_code == 200:
+        return response.json()
+
+    raise exception_from_error("Unknown error while looking RC release: ", response)
+
+def create_release(version, commit_sha):
+    tag = rc_tag_name(version)
+
+    params = {
+        'tag_name': tag,
+        'name': "{} - RC".format(version),
+        'target_commitish': commit_sha,
+        'prerelease': True
+    }
+
+    response = _github_request('post', 'repos/{}/releases'.format(repo), params)
+
+    if response.status_code != 201:
+        raise exception_from_error("Failed creating new release", response)
+
+    return response.json()
+
+def upload_asset(release, filepath):
+    upload_url = release['upload_url'].replace('{?name}', '')
+    filename = filepath.split('/')[-1]
+
+    with open(filepath) as file_content:
+        headers = {'Content-Type': 'application/gzip'}
+        response = requests.post(upload_url, file_content, params={'name': filename}, headers=headers, auth=auth, verify=False)
+
+    if response.status_code != 201: # not 200/201/...
+        raise exception_from_error('Failed uploading asset', response)
+
+    return response
+
+def remove_previous_builds(release):
+    for asset in release['assets']:
+        response = _github_request('delete', asset['url'])
+        if response.status_code != 204:
+            raise exception_from_error("Failed deleting asset", response)
+
+def get_changelog(commit_sha):
+    latest_release = _github_request('get', 'repos/{}/releases/latest'.format(repo))
+    if latest_release.status_code != 200:
+        raise exception_from_error('Failed getting latest release', latest_release)
+
+    latest_release = latest_release.json()
+    previous_sha = latest_release['target_commitish']
+
+    args = ['git', '--no-pager', 'log', '--merges', '--grep', 'Merge pull request', '--pretty=format:"%h|%s|%b|%p"', '{}...{}'.format(previous_sha, commit_sha)]
+    log = subprocess.check_output(args)
+    changes = ["Changes since {}:".format(latest_release['name'])]
+
+    for line in log.split('\n'):
+        try:
+            sha, subject, body, parents = line[1:-1].split('|')
+        except ValueError:
+            continue
+
+        try:
+            pull_request = re.match("Merge pull request #(\d+)", subject).groups()[0]
+            pull_request = " #{}".format(pull_request)
+        except Exception, ex:
+            pull_request = ""
+
+        author = subprocess.check_output(['git', 'log', '-1', '--pretty=format:"%an"', parents.split(' ')[-1]])[1:-1]
+
+        changes.append("{}{}: {} ({})".format(sha, pull_request, body.strip(), author))
+
+    return "\n".join(changes)
+
+def update_release(version, build_filepath, commit_sha):
+    try:
+        release = get_rc_release(version) or create_release(version, commit_sha)
+        print "Using release id: {}".format(release['id'])
+
+        remove_previous_builds(release)
+        response = upload_asset(release, build_filepath)
+
+        changelog = get_changelog(commit_sha)
+
+        response = _github_request('patch', release['url'], {'body': changelog})
+        print response.status_code
+        print response.text
+
+    except Exception, ex:
+        print ex
+
+if __name__ == '__main__':
+    commit_sha = sys.argv[1]
+    version = sys.argv[2]
+    filepath = sys.argv[3]
+
+    # TODO: make sure running from git directory & remote = repo
+    update_release(version, filepath, commit_sha)

bin/upload_version.py

@@ -1,46 +0,0 @@
-#!python
-import os
-import sys
-import json
-import requests
-import subprocess
-
-
-def capture_output(command):
-    proc = subprocess.Popen(command, stdout=subprocess.PIPE)
-    return proc.stdout.read()
-
-
-if __name__ == '__main__':
-    version = sys.argv[1]
-    filepath = sys.argv[2]
-    filename = filepath.split('/')[-1]
-    github_token = os.environ['GITHUB_TOKEN']
-    auth = (github_token, 'x-oauth-basic')
-    commit_sha = os.environ['CIRCLE_SHA1']
-
-    commit_body = capture_output(["git", "log", "--format=%b", "-n", "1", commit_sha])
-    file_md5_checksum = capture_output(["md5sum", filepath]).split()[0]
-    file_sha256_checksum = capture_output(["sha256sum", filepath]).split()[0]
-    version_body = "%s\n\nMD5: %s\nSHA256: %s" % (commit_body, file_md5_checksum, file_sha256_checksum)
-
-    params = json.dumps({
-        'tag_name': 'v{0}'.format(version),
-        'name': 're:dash v{0}'.format(version),
-        'body': version_body,
-        'target_commitish': commit_sha,
-        'prerelease': True
-    })
-
-    response = requests.post('https://api.github.com/repos/everythingme/redash/releases',
-                             data=params,
-                             auth=auth)
-
-    upload_url = response.json()['upload_url']
-    upload_url = upload_url.replace('{?name}', '')
-
-    with open(filepath) as file_content:
-        headers = {'Content-Type': 'application/gzip'}
-        response = requests.post(upload_url, file_content, params={'name': filename}, auth=auth,
-                                 headers=headers, verify=False)
-

circle.yml

@@ -19,6 +19,10 @@ test:
   post:
     - make pack
 deployment:
+  test:
+    branch: chore/release_process
+    commands:
+      - make upload
   github:
     branch: master
     commands:
@@ -26,3 +30,7 @@ deployment:
 notify:
   webhooks:
     - url: https://webhooks.gitter.im/e/895d09c3165a0913ac2f
+general:
+  branches:
+    ignore:
+      - gh-pages

migrations/0004_allow_null_in_event_user.py

@@ -0,0 +1,12 @@
+from playhouse.migrate import PostgresqlMigrator, migrate
+
+from redash.models import db
+
+if __name__ == '__main__':
+    db.connect_db()
+    migrator = PostgresqlMigrator(db.database)
+
+    with db.database.transaction():
+        migrate(
+            migrator.drop_not_null('events', 'user_id')
+        )

migrations/0005_add_updated_at.py

@@ -0,0 +1,26 @@
+from playhouse.migrate import PostgresqlMigrator, migrate
+
+from redash.models import db
+from redash import models
+
+if __name__ == '__main__':
+    db.connect_db()
+    migrator = PostgresqlMigrator(db.database)
+
+    with db.database.transaction():
+        migrate(
+            migrator.add_column('queries', 'updated_at', models.Query.updated_at),
+            migrator.add_column('dashboards', 'updated_at', models.Dashboard.updated_at),
+            migrator.add_column('widgets', 'updated_at', models.Widget.updated_at),
+            migrator.add_column('users', 'created_at', models.User.created_at),
+            migrator.add_column('users', 'updated_at', models.User.updated_at),
+            migrator.add_column('visualizations', 'created_at', models.Visualization.created_at),
+            migrator.add_column('visualizations', 'updated_at', models.Visualization.updated_at)
+        )
+
+        db.database.execute_sql("UPDATE queries SET updated_at = created_at;")
+        db.database.execute_sql("UPDATE dashboards SET updated_at = created_at;")
+        db.database.execute_sql("UPDATE widgets SET updated_at = created_at;")
+
+    db.close_db(None)
+

migrations/0006_queries_last_edit_by.py

@@ -0,0 +1,19 @@
+from playhouse.migrate import PostgresqlMigrator, migrate
+
+from redash.models import db
+from redash import models
+
+if __name__ == '__main__':
+    db.connect_db()
+    migrator = PostgresqlMigrator(db.database)
+
+    with db.database.transaction():
+        migrate(
+            migrator.add_column('queries', 'last_modified_by_id', models.Query.last_modified_by)
+        )
+
+        db.database.execute_sql("UPDATE queries SET last_modified_by_id = user_id;")
+
+    db.close_db(None)
+
+

migrations/0007_add_schedule_to_queries.py

@@ -0,0 +1,23 @@
+from playhouse.migrate import PostgresqlMigrator, migrate
+
+from redash.models import db
+from redash import models
+
+if __name__ == '__main__':
+    db.connect_db()
+    migrator = PostgresqlMigrator(db.database)
+
+    with db.database.transaction():
+        migrate(
+            migrator.add_column('queries', 'schedule', models.Query.schedule),
+        )
+
+        db.database.execute_sql("UPDATE queries SET schedule = ttl WHERE ttl > 0;")
+
+        migrate(
+            migrator.drop_column('queries', 'ttl')
+        )
+
+    db.close_db(None)
+
+

rd_ui/app/index.html

@@ -18,6 +18,7 @@
     <link rel="stylesheet" href="/bower_components/angular-ui-select/dist/select.css">
     <link rel="stylesheet" href="/bower_components/pace/themes/pace-theme-minimal.css">
     <link rel="stylesheet" href="/bower_components/font-awesome/css/font-awesome.css">
+    <link rel="stylesheet" href="/bower_components/codemirror/addon/hint/show-hint.css">
     <link rel="stylesheet" href="/styles/redash.css">
     <!-- endbuild -->
 </head>
@@ -39,9 +40,9 @@
     <div class="collapse navbar-collapse navbar-ex1-collapse">
         <ul class="nav navbar-nav">
             <li class="active" ng-show="pageTitle"><a class="page-title" ng-bind="pageTitle"></a></li>
-            <li class="dropdown" ng-show="groupedDashboards.length > 0 || otherDashboards.length > 0 || currentUser.hasPermission('create_dashboard')">
-                <a href="#" class="dropdown-toggle" data-toggle="dropdown"><span class="glyphicon glyphicon-th-large"></span> <b class="caret"></b></a>
-                <ul class="dropdown-menu">
+            <li class="dropdown" ng-show="groupedDashboards.length > 0 || otherDashboards.length > 0 || currentUser.hasPermission('create_dashboard')" dropdown>
+                <a href="#" class="dropdown-toggle" dropdown-toggle><span class="glyphicon glyphicon-th-large"></span> <b class="caret"></b></a>
+                <ul class="dropdown-menu" dropdown-menu>
                     <span ng-repeat="(name, group) in groupedDashboards">
                         <li class="dropdown-submenu">
                           <a href="#" ng-bind="name"></a>
@@ -59,9 +60,9 @@
                     <li><a data-toggle="modal" href="#new_dashboard_dialog" ng-show="currentUser.hasPermission('create_dashboard')">New Dashboard</a></li>
                 </ul>
             </li>
-            <li class="dropdown" ng-show="currentUser.hasPermission('view_query')">
-                <a href="#" class="dropdown-toggle" data-toggle="dropdown">Queries <b class="caret"></b></a>
-                <ul class="dropdown-menu">
+            <li class="dropdown" ng-show="currentUser.hasPermission('view_query')" dropdown>
+                <a href="#" class="dropdown-toggle" dropdown-toggle>Queries <b class="caret"></b></a>
+                <ul class="dropdown-menu" dropdown-menu>
                     <li ng-show="currentUser.hasPermission('create_query')"><a href="/queries/new">New Query</a></li>
                     <li><a href="/queries">Queries</a></li>
                 </ul>
@@ -105,9 +106,10 @@
 <script src="/bower_components/codemirror/lib/codemirror.js"></script>
 <script src="/bower_components/codemirror/addon/edit/matchbrackets.js"></script>
 <script src="/bower_components/codemirror/addon/edit/closebrackets.js"></script>
+<script src="/bower_components/codemirror/addon/hint/show-hint.js"></script>
+<script src="/bower_components/codemirror/addon/hint/anyword-hint.js"></script>
 <script src="/bower_components/codemirror/mode/sql/sql.js"></script>
 <script src="/bower_components/codemirror/mode/javascript/javascript.js"></script>
-<script src="/bower_components/angular-ui-codemirror/ui-codemirror.js"></script>
 <script src="/bower_components/highcharts/highcharts.js"></script>
 <script src="/bower_components/highcharts/modules/exporting.js"></script>
 <script src="/bower_components/gridster/dist/jquery.gridster.js"></script>
@@ -123,7 +125,7 @@
 <script src="/bower_components/marked/lib/marked.js"></script>
 <script src="/scripts/ng_highchart.js"></script>
 <script src="/scripts/ng_smart_table.js"></script>
-<script src="/scripts/ui-bootstrap-tpls-0.5.0.min.js"></script>
+<script src="/bower_components/angular-ui-bootstrap-bower/ui-bootstrap-tpls.js"></script>
 <script src="/bower_components/bucky/bucky.js"></script>
 <script src="/bower_components/pace/pace.js"></script>
 <script src="/bower_components/mustache/mustache.js"></script>

rd_ui/app/scripts/app.js

@@ -6,7 +6,6 @@ angular.module('redash', [
     'redash.services',
     'redash.renderers',
     'redash.visualization',
-    'ui.codemirror',
     'highchart',
     'ui.select2',
     'angular-growl',

rd_ui/app/scripts/controllers/admin_controllers.js

@@ -16,16 +16,9 @@
       $timeout(refresh, 59 * 1000);
     };
 
-    $scope.flowerUrl = featureFlags.flowerUrl;
-
     refresh();
   }
 
-  var AdminWorkersCtrl = function ($scope, $sce) {
-    $scope.flowerUrl = $sce.trustAsResourceUrl(featureFlags.flowerUrl);
-  };
-
   angular.module('redash.admin_controllers', [])
          .controller('AdminStatusCtrl', ['$scope', 'Events', '$http', '$timeout', AdminStatusCtrl])
-         .controller('AdminWorkersCtrl', ['$scope', '$sce', AdminWorkersCtrl])
 })();

rd_ui/app/scripts/controllers/controllers.js

@@ -1,4 +1,11 @@
 (function () {
+  var dateFormatter = function (value) {
+    if (!value) {
+      return "-";
+    }
+    return value.toDate().toLocaleString();
+  };
+
   var QuerySearchCtrl = function($scope, $location, $filter, Events, Query) {
     $scope.$parent.pageTitle = "Queries Search";
 
@@ -8,11 +15,6 @@
       maxSize: 8,
     };
 
-    var dateFormatter = function (value) {
-      if (!value) return "-";
-      return value.format("DD/MM/YY HH:mm");
-    }
-
     $scope.gridColumns = [
       {
         "label": "Name",
@@ -21,7 +23,7 @@
       },
       {
         'label': 'Created By',
-        'map': 'user.name'
+        'map': 'user_name'
       },
       {
         'label': 'Created At',
@@ -30,9 +32,9 @@
       },
       {
         'label': 'Update Schedule',
-        'map': 'ttl',
+        'map': 'schedule',
         'formatFunction': function (value) {
-          return $filter('refreshRateHumanize')(value);
+          return $filter('scheduleHumanize')(value);
         }
       }
     ];
@@ -43,6 +45,7 @@
     Query.search({q: $scope.term }, function(results) {
       $scope.queries = _.map(results, function(query) {
         query.created_at = moment(query.created_at);
+        query.user_name = query.user.name;
         return query;
       });
     });
@@ -70,11 +73,6 @@
     $scope.allQueries = [];
     $scope.queries = [];
 
-    var dateFormatter = function (value) {
-      if (!value) return "-";
-      return value.format("DD/MM/YY HH:mm");
-    }
-
     var filterQueries = function () {
       $scope.queries = _.filter($scope.allQueries, function (query) {
         if (!$scope.selectedTab) {
@@ -95,6 +93,7 @@
       $scope.allQueries = _.map(queries, function (query) {
         query.created_at = moment(query.created_at);
         query.retrieved_at = moment(query.retrieved_at);
+        query.user_name = query.user.name;
         return query;
       });
 
@@ -109,7 +108,7 @@
       },
       {
         'label': 'Created By',
-        'map': 'user.name'
+        'map': 'user_name'
       },
       {
         'label': 'Created At',
@@ -130,9 +129,9 @@
       },
       {
         'label': 'Update Schedule',
-        'map': 'ttl',
+        'map': 'schedule',
         'formatFunction': function (value) {
-          return $filter('refreshRateHumanize')(value);
+          return $filter('scheduleHumanize')(value);
         }
       }
     ]

rd_ui/app/scripts/controllers/dashboard.js

@@ -100,9 +100,13 @@
       Events.record(currentUser, "autorefresh", "dashboard", dashboard.id, {'enable': $scope.refreshEnabled});
 
       if ($scope.refreshEnabled) {
-        var refreshRate = _.min(_.flatten($scope.dashboard.widgets), function(widget) {
-          return widget.visualization.query.ttl;
-        }).visualization.query.ttl;
+        var refreshRate = _.min(_.map(_.flatten($scope.dashboard.widgets), function(widget) {
+          var schedule = widget.visualization.query.schedule;
+          if (schedule === null || schedule.match(/\d\d:\d\d/) !== null) {
+            return 60;
+          }
+          return widget.visualization.query.schedule;
+        }));
 
         $scope.refreshRate = _.max([120, refreshRate * 2]) * 1000;
 
@@ -138,7 +142,6 @@
       var parameters = Query.collectParamsFromQueryString($location, $scope.query);
       var maxAge = $location.search()['maxAge'];
       $scope.queryResult = $scope.query.getQueryResult(maxAge, parameters);
-      $scope.nextUpdateTime = moment(new Date(($scope.query.updated_at + $scope.query.ttl + $scope.query.runtime + 300) * 1000)).fromNow();
 
       $scope.type = 'visualization';
     } else {

rd_ui/app/scripts/controllers/query_source.js

@@ -17,7 +17,7 @@
         saveQuery = $scope.saveQuery;
 
     $scope.sourceMode = true;
-    $scope.canEdit = currentUser.canEdit($scope.query);
+    $scope.canEdit = true;
     $scope.isDirty = false;
 
     $scope.newVisualization = undefined;
@@ -68,7 +68,7 @@
     $scope.duplicateQuery = function() {
       Events.record(currentUser, 'fork', 'query', $scope.query.id);
       $scope.query.id = null;
-      $scope.query.ttl = -1;
+      $scope.query.schedule = null;
 
       $scope.saveQuery({
         successMessage: 'Query forked',

rd_ui/app/scripts/controllers/query_view.js

@@ -1,27 +1,54 @@
 (function() {
   'use strict';
 
-  function QueryViewCtrl($scope, Events, $route, $location, notifications, growl, Query, DataSource) {
+  function QueryViewCtrl($scope, Events, $route, $location, notifications, growl, $modal, Query, DataSource) {
     var DEFAULT_TAB = 'table';
 
-    var getQueryResult = function(ttl) {
+    var getQueryResult = function(maxAge) {
       // Collect params, and getQueryResult with params; getQueryResult merges it into the query
       var parameters = Query.collectParamsFromQueryString($location, $scope.query);
-      if (ttl == undefined) {
-        ttl = $location.search()['maxAge'];
+      if (maxAge == undefined) {
+        maxAge = $location.search()['maxAge'];
       }
-      $scope.queryResult = $scope.query.getQueryResult(ttl, parameters);
+
+      if (maxAge == undefined) {
+        maxAge = -1;
+      }
+
+      $scope.queryResult = $scope.query.getQueryResult(maxAge, parameters);
     }
 
     $scope.query = $route.current.locals.query;
+
+    var updateSchema = function() {
+      $scope.hasSchema = false;
+      $scope.editorSize = "col-md-12";
+      var dataSourceId = $scope.query.data_source_id || $scope.dataSources[0].id;
+      DataSource.getSchema({id: dataSourceId}, function(data) {
+        if (data && data.length > 0) {
+          $scope.schema = data;
+          _.each(data, function(table) {
+            table.collapsed = true;
+          });
+
+          $scope.editorSize = "col-md-9";
+          $scope.hasSchema = true;
+        } else {
+          $scope.hasSchema = false;
+          $scope.editorSize = "col-md-12";
+        }
+      });
+    }
+
     Events.record(currentUser, 'view', 'query', $scope.query.id);
     getQueryResult();
     $scope.queryExecuting = false;
 
-    $scope.isQueryOwner = currentUser.id === $scope.query.user.id;
+    $scope.isQueryOwner = (currentUser.id === $scope.query.user.id) || currentUser.hasPermission('admin');
     $scope.canViewSource = currentUser.hasPermission('view_source');
 
     $scope.dataSources = DataSource.get(function(dataSources) {
+      updateSchema();
       $scope.query.data_source_id = $scope.query.data_source_id || dataSources[0].id;
     });
 
@@ -98,7 +125,7 @@
       
       return Query.delete({id: data.id}, function() {
         $scope.query.is_archived = true;
-        $scope.query.ttl = -1;
+        $scope.query.schedule = null;
         growl.addSuccessMessage(options.successMessage);
           // This feels dirty.
           $('#archive-confirmation-modal').modal('hide');
@@ -121,6 +148,7 @@
         });
       }
 
+      updateSchema();
       $scope.executeQuery();
     };
 
@@ -168,6 +196,28 @@
       }
     });
 
+    $scope.openScheduleForm = function() {
+      if (!$scope.isQueryOwner) {
+        return;
+      };
+
+      $modal.open({
+        templateUrl: '/views/schedule_form.html',
+        size: 'sm',
+        scope: $scope,
+        controller: ['$scope', '$modalInstance', function($scope, $modalInstance) {
+          $scope.close = function() {
+            $modalInstance.close();
+          }
+          if ($scope.query.hasDailySchedule()) {
+            $scope.refreshType = 'daily';
+          } else {
+            $scope.refreshType = 'periodic';
+          }
+        }]
+      });
+    };
+
     $scope.$watch(function() {
       return $location.hash()
     }, function(hash) {
@@ -180,5 +230,5 @@
 
   angular.module('redash.controllers')
     .controller('QueryViewCtrl',
-      ['$scope', 'Events', '$route', '$location', 'notifications', 'growl', 'Query', 'DataSource', QueryViewCtrl]);
+      ['$scope', 'Events', '$route', '$location', 'notifications', 'growl', '$modal', 'Query', 'DataSource', QueryViewCtrl]);
 })();

rd_ui/app/scripts/directives/query_directives.js

@@ -29,7 +29,7 @@
       restrict: 'E',
       template: '<span ng-show="query.id && canViewSource">\
                     <a ng-show="!sourceMode"\
-                      ng-href="{{query.id}}/source#{{selectedTab}}">Show Source\
+                      ng-href="/queries/{{query.id}}/source#{{selectedTab}}">Show Source\
                     </a>\
                     <a ng-show="sourceMode"\
                       ng-href="/queries/{{query.id}}#{{selectedTab}}">Hide Source\
@@ -63,26 +63,80 @@
       restrict: 'E',
       scope: {
         'query': '=',
-        'lock': '='
+        'lock': '=',
+        'schema': '='
       },
-      template: '<textarea\
-                  ui-codemirror="editorOptions"\
-                  ng-model="query.query">',
-      link: function($scope) {
-        $scope.editorOptions = {
+      template: '<textarea></textarea>',
+      link: {
+        pre: function ($scope, element) {
+          var textarea = element.children()[0];
+          var editorOptions = {
             mode: 'text/x-sql',
             lineWrapping: true,
             lineNumbers: true,
             readOnly: false,
             matchBrackets: true,
-            autoCloseBrackets: true
-        };
+            autoCloseBrackets: true,
+            extraKeys: {"Ctrl-Space": "autocomplete"}
+          };
 
-        $scope.$watch('lock', function(locked) {
-          $scope.editorOptions.readOnly = locked ? 'nocursor' : false;
-        });
+          var additionalHints = [];
+
+          CodeMirror.commands.autocomplete = function(cm) {
+            var hinter  = function(editor, options) {
+              var hints = CodeMirror.hint.anyword(editor, options);
+              var cur = editor.getCursor(), token = editor.getTokenAt(cur).string;
+
+              hints.list = _.union(hints.list, _.filter(additionalHints, function (h) {
+                return h.search(token) === 0;
+              }));
+
+              return hints;
+            };
+
+//            CodeMirror.showHint(cm, CodeMirror.hint.anyword);
+            CodeMirror.showHint(cm, hinter);
+          };
+
+          var codemirror = CodeMirror.fromTextArea(textarea, editorOptions);
+
+          codemirror.on('change', function(instance) {
+            var newValue = instance.getValue();
+
+            if (newValue !== $scope.query.query) {
+              $scope.$evalAsync(function() {
+                $scope.query.query = newValue;
+              });
+            }
+          });
+
+          $scope.$watch('query.query', function () {
+            if ($scope.query.query !== codemirror.getValue()) {
+              codemirror.setValue($scope.query.query);
+            }
+          });
+
+          $scope.$watch('schema', function (schema) {
+            if (schema) {
+              var keywords = [];
+              _.each(schema, function (table) {
+                keywords.push(table.name);
+                _.each(table.columns, function (c) {
+                  keywords.push(c);
+                });
+              });
+
+              additionalHints = _.unique(keywords);
+            }
+          });
+
+          $scope.$watch('lock', function (locked) {
+            var readOnly = locked ? 'nocursor' : false;
+            codemirror.setOption('readOnly', readOnly);
+          });
+        }
       }
-    }
+    };
   }
 
   function queryFormatter($http) {
@@ -111,42 +165,91 @@
     }
   }
 
+  function queryTimePicker() {
+    return {
+      restrict: 'E',
+      template: '<select ng-disabled="refreshType != \'daily\'" ng-model="hour" ng-change="updateSchedule()" ng-options="c as c for c in hourOptions"></select> :\
+                 <select ng-disabled="refreshType != \'daily\'" ng-model="minute" ng-change="updateSchedule()" ng-options="c as c for c in minuteOptions"></select>',
+      link: function($scope) {
+        var padWithZeros = function(size, v) {
+          v = String(v);
+          if (v.length < size) {
+            v = "0" + v;
+          }
+          return v;
+        };
+
+        $scope.hourOptions = _.map(_.range(0, 24), _.partial(padWithZeros, 2));
+        $scope.minuteOptions = _.map(_.range(0, 60, 5), _.partial(padWithZeros, 2));
+
+        if ($scope.query.hasDailySchedule()) {
+          var parts = $scope.query.scheduleInLocalTime().split(':');
+          $scope.minute = parts[1];
+          $scope.hour = parts[0];
+        } else {
+          $scope.minute = "15";
+          $scope.hour = "00";
+        }
+
+        $scope.updateSchedule = function() {
+          var newSchedule = moment().hour($scope.hour).minute($scope.minute).utc().format('HH:mm');
+          if (newSchedule != $scope.query.schedule) {
+            $scope.query.schedule = newSchedule;
+            $scope.saveQuery();
+          }
+        };
+
+        $scope.$watch('refreshType', function() {
+          if ($scope.refreshType == 'daily') {
+            $scope.updateSchedule();
+          }
+        });
+      }
+    }
+  }
+
   function queryRefreshSelect() {
     return {
       restrict: 'E',
       template: '<select\
-                  ng-disabled="!isQueryOwner"\
-                  ng-model="query.ttl"\
+                  ng-disabled="refreshType != \'periodic\'"\
+                  ng-model="query.schedule"\
                   ng-change="saveQuery()"\
                   ng-options="c.value as c.name for c in refreshOptions">\
+                  <option value="">No Refresh</option>\
                   </select>',
       link: function($scope) {
         $scope.refreshOptions = [
             {
-                value: -1,
-                name: 'No Refresh'
-            },
-            {
-                value: 60,
+                value: "60",
                 name: 'Every minute'
-            },
+            }
         ]
 
         _.each(_.range(1, 13), function (i) {
             $scope.refreshOptions.push({
-                value: i * 3600,
+                value: String(i * 3600),
                 name: 'Every ' + i + 'h'
             });
         })
 
         $scope.refreshOptions.push({
-            value: 24 * 3600,
+            value: String(24 * 3600),
             name: 'Every 24h'
         });
         $scope.refreshOptions.push({
-            value: 7 * 24 * 3600,
+            value: String(7 * 24 * 3600),
             name: 'Once a week'
         });
+
+        $scope.$watch('refreshType', function() {
+          if ($scope.refreshType == 'periodic') {
+            if ($scope.query.hasDailySchedule()) {
+              $scope.query.schedule = null;
+              $scope.saveQuery();
+            }
+          }
+        });
       }
 
     }
@@ -158,5 +261,6 @@
   .directive('queryResultLink', queryResultCSVLink)
   .directive('queryEditor', queryEditor)
   .directive('queryRefreshSelect', queryRefreshSelect)
+  .directive('queryTimePicker', queryTimePicker)
   .directive('queryFormatter', ['$http', queryFormatter]);
 })();

rd_ui/app/scripts/filters.js

@@ -24,13 +24,17 @@ angular.module('redash.filters', []).
     return durationHumanize;
   })
 
-  .filter('refreshRateHumanize', function () {
-    return function (ttl) {
-      if (ttl == -1) {
+  .filter('scheduleHumanize', function() {
+    return function (schedule) {
+      if (schedule === null) {
         return "Never";
-      } else {
-        return "Every " + durationHumanize(ttl);
+      } else if (schedule.match(/\d\d:\d\d/) !== null) {
+        var parts = schedule.split(':');
+        var localTime = moment.utc().hour(parts[0]).minute(parts[1]).local().format('HH:mm');
+        return "Every day at " + localTime;
       }
+
+      return "Every " + durationHumanize(parseInt(schedule));
     }
   })
 

rd_ui/app/scripts/ng_highchart.js

@@ -50,7 +50,7 @@
         ;
 
         if (moment.isMoment(this.x)) {
-          var s = '<b>' + moment(this.x).format("DD/MM/YY HH:mm") + '</b>',
+          var s = '<b>' + this.x.toDate().toLocaleString() + '</b>',
               pointsCount = this.points.length;
 
           $.each(this.points, function (i, point) {

rd_ui/app/scripts/ng_smart_table.js

@@ -91,7 +91,7 @@
 
                     //insert columns from column config
                     //TODO add a way to clean all columns
-                    scope.$watch('columnCollection', function (oldValue, newValue) {
+                    scope.$watchCollection('columnCollection', function (oldValue, newValue) {
                         if (scope.columnCollection) {
                             scope.columns.length = 0;
                             for (var i = 0, l = scope.columnCollection.length; i < l; i++) {
@@ -205,11 +205,10 @@
                         column = scope.column,
                         row = scope.dataRow,
                         format = filter('format'),
-                        getter = parse(column.map),
                         childScope;
 
                     //can be useful for child directives
-                    scope.formatedValue = format(getter(row), column.formatFunction, column.formatParameter);
+                    scope.formatedValue = format(row[column.map], column.formatFunction, column.formatParameter);
 
                     function defaultContent() {
                         //clear content
@@ -267,12 +266,11 @@
                 replace: true,
                 link: function (scope, element, attrs, ctrl) {
                     var form = angular.element(element.children()[1]),
-                        input = angular.element(form.children()[0]),
-                        getter = parse(scope.column.map);
+                        input = angular.element(form.children()[0]);
 
                     //init values
                     scope.isEditMode = false;
-                    scope.value = getter(scope.row);
+                    scope.value = scope.row[scope.column.map];
 
 
                     scope.submit = function () {
@@ -285,7 +283,7 @@
                     };
 
                     scope.toggleEditMode = function () {
-                        scope.value = getter(scope.row);
+                        scope.value = scope.row[scope.column.map];
                         scope.isEditMode = scope.isEditMode !== true;
                     };
 
@@ -595,13 +593,11 @@
              */
             this.updateDataRow = function (dataRow, propertyName, newValue) {
                 var index = scope.displayedCollection.indexOf(dataRow),
-                    getter = parse(propertyName),
-                    setter = getter.assign,
                     oldValue;
                 if (index !== -1) {
-                    oldValue = getter(scope.displayedCollection[index]);
+                    oldValue = scope.displayedCollection[index][propertyName];
                     if (oldValue !== newValue) {
-                        setter(scope.displayedCollection[index], newValue);
+                        scope.displayedCollection[index][propertyName] = newValue;
                         scope.$emit('updateDataRow', {item: scope.displayedCollection[index]});
                     }
                 }

rd_ui/app/scripts/services/resources.js

@@ -243,26 +243,9 @@
       return parts[0];
     };
 
-    var charConversionMap = {
-      '__pct': /%/g,
-      '_': / /g,
-      '__qm': /\?/g,
-      '__brkt': /[\(\)\[\]]/g,
-      '__dash': /-/g,
-      '__amp': /&/g,
-      '__sl': /\//g,
-      '__fsl': /\\/g,
-    };
-
     QueryResult.prototype.getColumnCleanName = function (column) {
       var name = this.getColumnNameWithoutType(column);
 
-      if (name != '') {
-        _.each(charConversionMap, function(regex, replacement) {
-          name = name.replace(regex, replacement);
-        });
-      }
-
       return name;
     }
 
@@ -325,7 +308,7 @@
       this.filters = filters;
     }
 
-    var refreshStatus = function (queryResult, query, ttl) {
+    var refreshStatus = function (queryResult, query) {
       Job.get({'id': queryResult.job.id}, function (response) {
         queryResult.update(response);
 
@@ -335,7 +318,7 @@
           });
         } else if (queryResult.getStatus() != "failed") {
           $timeout(function () {
-            refreshStatus(queryResult, query, ttl);
+            refreshStatus(queryResult, query);
           }, 3000);
         }
       })
@@ -355,14 +338,14 @@
       return this.deferred.promise;
     }
 
-    QueryResult.get = function (data_source_id, query, ttl) {
+    QueryResult.get = function (data_source_id, query, maxAge) {
       var queryResult = new QueryResult();
 
-      QueryResultResource.post({'data_source_id': data_source_id, 'query': query, 'ttl': ttl}, function (response) {
+      QueryResultResource.post({'data_source_id': data_source_id, 'query': query, 'max_age': maxAge}, function (response) {
         queryResult.update(response);
 
         if ('job' in response) {
-          refreshStatus(queryResult, query, ttl);
+          refreshStatus(queryResult, query);
         }
       });
 
@@ -390,7 +373,7 @@
       return new Query({
         query: "",
         name: "New Query",
-        ttl: -1,
+        schedule: null,
         user: currentUser
       });
     };
@@ -414,10 +397,19 @@
       return '/queries/' + this.id + '/source';
     };
 
-    Query.prototype.getQueryResult = function (ttl, parameters) {
-      if (ttl == undefined) {
-        ttl = this.ttl;
-      }
+    Query.prototype.hasDailySchedule = function() {
+      return (this.schedule && this.schedule.match(/\d\d:\d\d/) !== null);
+    }
+
+    Query.prototype.scheduleInLocalTime = function() {
+      var parts = this.schedule.split(':');
+      return moment.utc().hour(parts[0]).minute(parts[1]).local().format('HH:mm');
+    }
+
+    Query.prototype.getQueryResult = function (maxAge, parameters) {
+//      if (ttl == undefined) {
+//        ttl = this.ttl;
+//      }
 
       var queryText = this.query;
 
@@ -443,16 +435,16 @@
         this.latest_query_data_id = null;
       }
 
-      if (this.latest_query_data && ttl != 0) {
+      if (this.latest_query_data && maxAge != 0) {
         if (!this.queryResult) {
           this.queryResult = new QueryResult({'query_result': this.latest_query_data});
         }
-      } else if (this.latest_query_data_id && ttl != 0) {
+      } else if (this.latest_query_data_id && maxAge != 0) {
         if (!this.queryResult) {
           this.queryResult = QueryResult.getById(this.latest_query_data_id);
         }
       } else if (this.data_source_id) {
-        this.queryResult = QueryResult.get(this.data_source_id, queryText, ttl);
+        this.queryResult = QueryResult.get(this.data_source_id, queryText, maxAge);
       }
 
       return this.queryResult;
@@ -488,7 +480,12 @@
 
 
   var DataSource = function ($resource) {
-    var DataSourceResource = $resource('/api/data_sources/:id', {id: '@id'}, {'get': {'method': 'GET', 'cache': true, 'isArray': true}});
+    var actions = {
+      'get': {'method': 'GET', 'cache': true, 'isArray': true},
+      'getSchema': {'method': 'GET', 'cache': true, 'isArray': true, 'url': '/api/data_sources/:id/schema'}
+    };
+
+    var DataSourceResource = $resource('/api/data_sources/:id', {id: '@id'}, actions);
 
     return DataSourceResource;
   }

rd_ui/app/scripts/visualizations/table.js

@@ -78,15 +78,15 @@
                 };
               } else if (columnType === 'date') {
                 columnDefinition.formatFunction = function (value) {
-                  if (value) {
-                    return value.format("DD/MM/YY");
+                  if (value && moment.isMoment(value)) {
+                    return value.toDate().toLocaleDateString();
                   }
                   return value;
                 };
               } else if (columnType === 'datetime') {
                 columnDefinition.formatFunction = function (value) {
-                  if (value) {
-                    return value.format("DD/MM/YY HH:mm");
+                  if (value && moment.isMoment(value)) {
+                    return value.toDate().toLocaleString();
                   }
                   return value;
                 };

rd_ui/app/styles/redash.css

@@ -156,7 +156,7 @@ li.widget:hover {
 /* CodeMirror */
 .CodeMirror {
     border: 1px solid #eee;
-    height: auto;
+    /*height: auto;*/
     min-height: 300px;
     margin-bottom: 10px;
 }
@@ -308,6 +308,18 @@ counter-renderer counter-name {
     height: 100%;
 }
 
+.schema-browser {
+    height: 300px;
+    overflow: scroll;
+}
+
+div.table-name {
+    overflow: scroll;
+    text-overflow: ellipsis;
+    white-space: nowrap;
+    cursor: pointer;
+}
+
 /*
 bootstrap's hidden-xs class adds display:block when not hidden
 use this class when you need to keep the original display value

rd_ui/app/views/admin_status.html

@@ -24,10 +24,6 @@
                     <span class="badge">{{manager.outdated_queries_count}}</span>
                     Outdated Queries Count
                 </li>
-
-                <li class="list-group-item" ng-if="flowerUrl">
-                    <a href="/admin/workers">Workers' Status</a>
-                </li>
             </ul>
             <ul class="list-group col-lg-4">
                 <li class="list-group-item active">Queues</li>

rd_ui/app/views/admin_workers.html

@@ -1,3 +0,0 @@
-<div class="container-fluid iframe-container">
-    <iframe src="{{flowerUrl}}" style="width:100%; height:100%; background-color:transparent;"></iframe>
-</div>

rd_ui/app/views/dashboard.html

@@ -37,7 +37,7 @@
 
                 <div class="panel-footer">
                     <span class="label label-default"
-                          tooltip="next update {{nextUpdateTime}} (query runtime: {{queryResult.getRuntime() | durationHumanize}})"
+                          tooltip="(query runtime: {{queryResult.getRuntime() | durationHumanize}})"
                           tooltip-placement="bottom">Updated: <span am-time-ago="queryResult.getUpdatedAt()"></span></span>
 
                     <span class="pull-right">

rd_ui/app/views/query.html

@@ -59,9 +59,9 @@
 
     <hr>
 
-    <div class="row">
-        <div class="col-lg-12">
-            <div ng-show="sourceMode">
+    <div class="row" ng-if="sourceMode">
+        <div ng-class="editorSize">
+            <div>
                 <p>
                     <button type="button" class="btn btn-primary btn-xs" ng-disabled="queryExecuting" ng-click="executeQuery()">
                         <span class="glyphicon glyphicon-play"></span> Execute
@@ -77,21 +77,43 @@
                         </button>
                     </span>
                 </p>
-            </div>
 
-            <!-- code editor -->
-            <div ng-show="sourceMode">
                 <p>
-                    <query-editor query="query" lock="queryFormatting"></query-editor>
+                    <query-editor query="query" schema="schema" lock="queryFormatting"></query-editor>
                 </p>
-                <hr>
+            </div>
+        </div>
+        <div class="col-md-3" ng-show="hasSchema">
+            <div>
+                <input type="text" placeholder="Search schema..." class="form-control" ng-model="schemaFilter">
+            </div>
+            <div class="schema-browser">
+                <div ng-repeat="table in schema | filter:schemaFilter">
+                    <div class="table-name" ng-click="table.collapsed = !table.collapsed">
+                        <i class="fa fa-table"></i> <strong><span title="{{table.name}}">{{table.name}}</span></strong>
+                    </div>
+                    <div collapse="table.collapsed">
+                        <div ng-repeat="column in table.columns | filter:schemaFilter" style="padding-left:16px;">{{column}}</div>
+                    </div>
+                </div>
             </div>
         </div>
 
-    </div>
 
+    </div>
+    <hr ng-if="sourceMode">
     <div class="row">
         <div class="col-lg-3 rd-hidden-xs">
+            <p>
+                <span class="glyphicon glyphicon-user"></span>
+                <span class="text-muted">Created By </span>
+                <strong>{{query.user.name}}</strong>
+            </p>
+            <p ng-if="query.last_modified_by && query.user.id != query.last_modified_by.id">
+                <span class="glyphicon glyphicon-user"></span>
+                <span class="text-muted">Last Modified By </span>
+                <strong>{{query.last_modified_by.name}}</strong>
+            </p>
             <p>
                 <span class="glyphicon glyphicon-time"></span>
                 <span class="text-muted">Last update </span>
@@ -99,12 +121,6 @@
                     <rd-time-ago value="queryResult.query_result.retrieved_at"></rd-time-ago>
                 </strong>
             </p>
-            <p>
-                <span class="glyphicon glyphicon-user"></span>
-                <span class="text-muted">Created By </span>
-                <strong ng-hide="isQueryOwner">{{query.user.name}}</strong>
-                <strong ng-show="isQueryOwner">You</strong>
-            </p>
             <p>
                 <span class="glyphicon glyphicon-play"></span>
                 <span class="text-muted">Runtime </span>
@@ -117,8 +133,8 @@
             </p>
             <p>
                 <span class="glyphicon glyphicon-refresh"></span>
-                <span class="text-muted">Refresh Interval</span>
-                <query-refresh-select></query-refresh-select>
+                <span class="text-muted">Refresh Schedule</span>
+                <a href="" ng-click="openScheduleForm()">{{query.schedule | scheduleHumanize}}</a>
             </p>
 
             <p>
@@ -186,7 +202,7 @@
                             <rd-tab tab-id="{{vis.id}}" name="{{vis.name}}" ng-if="vis.type!='TABLE'" ng-repeat="vis in query.visualizations">
                                 <span class="remove" ng-click="deleteVisualization($event, vis)" ng-show="canEdit"> &times;</span>
                             </rd-tab>
-                            <rd-tab tab-id="add" name="&plus; New" removeable="true" ng-show="canEdit"></rd-tab>
+                            <rd-tab tab-id="add" name="&plus; New Visualization" removeable="true" ng-show="canEdit"></rd-tab>
                             <li ng-if="!sourceMode" class="rd-tab-btn"><button class="btn btn-sm btn-default" ng-click="executeQuery()" ng-disabled="queryExecuting" title="Refresh Dataset"><span class="glyphicon glyphicon-refresh"></span></button></li>
                         </ul>
                     </div>

rd_ui/app/views/schedule_form.html

@@ -0,0 +1,18 @@
+<div class="modal-header">
+    <button type="button" class="close" aria-label="Close" ng-click="close()"><span aria-hidden="true">&times;</span></button>
+    <h4 class="modal-title">Refresh Schedule</h4>
+</div>
+<div class="modal-body">
+    <div class="radio">
+        <label>
+            <input type="radio" value="periodic" ng-model="refreshType">
+            <query-refresh-select ng-disabled="refreshType != 'periodic'"></query-refresh-select>
+        </label>
+    </div>
+    <div class="radio">
+        <label>
+            <input type="radio" value="daily" ng-model="refreshType">
+            <query-time-picker ng-disabled="refreshType != 'daily'"></query-time-picker>
+        </label>
+    </div>
+</div>

rd_ui/bower.json

@@ -12,8 +12,7 @@
     "es5-shim": "2.0.8",
     "angular-moment": "0.2.0",
     "moment": "2.1.0",
-    "angular-ui-bootstrap": "0.5.0",
-    "angular-ui-codemirror": "0.0.5",
+    "codemirror": "4.8.0",
     "highcharts": "3.0.10",
     "underscore": "1.5.1",
     "pivottable": "~1.1.1",
@@ -29,7 +28,8 @@
     "angular-ui-select": "0.8.2",
     "font-awesome": "~4.2.0",
     "mustache": "~1.0.0",
-    "canvg": "gabelerner/canvg"
+    "canvg": "gabelerner/canvg",
+    "angular-ui-bootstrap-bower": "~0.12.1"
   },
   "devDependencies": {
     "angular-mocks": "1.2.18",

redash/cache.py

@@ -1,6 +1,3 @@
-from flask import make_response
-from functools import update_wrapper
-
 ONE_YEAR = 60 * 60 * 24 * 365.25
 
 headers = {

redash/controllers.py

@@ -7,9 +7,8 @@ but this is only due to configuration issues and temporary.
 import csv
 import hashlib
 import json
-import numbers
 import cStringIO
-import datetime
+import time
 import logging
 
 from flask import render_template, send_from_directory, make_response, request, jsonify, redirect, \
@@ -53,8 +52,7 @@ def index(**kwargs):
     }
 
     features = {
-        'clientSideMetrics': settings.CLIENT_SIDE_METRICS,
-        'flowerUrl': settings.CELERY_FLOWER_URL
+        'clientSideMetrics': settings.CLIENT_SIDE_METRICS
     }
 
     return render_template("index.html", user=json.dumps(user), name=settings.NAME,
@@ -110,7 +108,7 @@ def status_api():
 
     manager_status = redis_connection.hgetall('redash:status')
     status['manager'] = manager_status
-    status['manager']['outdated_queries_count'] = models.Query.outdated_queries().count()
+    status['manager']['outdated_queries_count'] = len(models.Query.outdated_queries())
 
     queues = {}
     for ds in models.DataSource.select():
@@ -137,6 +135,24 @@ def format_sql_query():
     return sqlparse.format(query, reindent=True, keyword_case='upper')
 
 
+@app.route('/queries/new', methods=['POST'])
+@auth.required
+def create_query_route():
+    query = request.form.get('query', None)
+    data_source_id = request.form.get('data_source_id', None)
+
+    if query is None or data_source_id is None:
+        abort(400)
+
+    query = models.Query.create(name="New Query",
+                                query=query,
+                                data_source=data_source_id,
+                                user=current_user._get_current_object(),
+                                schedule=None)
+
+    return redirect('/queries/{}'.format(query.id), 303)
+
+
 class BaseResource(Resource):
     decorators = [auth.required]
 
@@ -203,10 +219,18 @@ class DataSourceListAPI(BaseResource):
 
         return datasource.to_dict()
 
-
 api.add_resource(DataSourceListAPI, '/api/data_sources', endpoint='data_sources')
 
 
+class DataSourceSchemaAPI(BaseResource):
+    def get(self, data_source_id):
+        data_source = models.DataSource.get_by_id(data_source_id)
+        schema = data_source.get_schema()
+
+        return schema
+
+api.add_resource(DataSourceSchemaAPI, '/api/data_sources/<data_source_id>/schema')
+
 class DashboardRecentAPI(BaseResource):
     def get(self):
         return [d.to_dict() for d in models.Dashboard.recent(current_user.id).limit(20)]
@@ -320,7 +344,7 @@ class QueryListAPI(BaseResource):
     @require_permission('create_query')
     def post(self):
         query_def = request.get_json(force=True)
-        for field in ['id', 'created_at', 'api_key', 'visualizations', 'latest_query_data']:
+        for field in ['id', 'created_at', 'api_key', 'visualizations', 'latest_query_data', 'last_modified_by']:
             query_def.pop(field, None)
 
         query_def['user'] = self.current_user
@@ -328,8 +352,6 @@ class QueryListAPI(BaseResource):
         query = models.Query(**query_def)
         query.save()
 
-        query.create_default_visualizations()
-
         return query.to_dict()
 
     @require_permission('view_query')
@@ -343,7 +365,7 @@ class QueryAPI(BaseResource):
         query = models.Query.get_by_id(query_id)
         
         query_def = request.get_json(force=True)
-        for field in ['id', 'created_at', 'api_key', 'visualizations', 'latest_query_data', 'user']:
+        for field in ['id', 'created_at', 'api_key', 'visualizations', 'latest_query_data', 'user', 'last_modified_by']:
             query_def.pop(field, None)
 
         if 'latest_query_data_id' in query_def:
@@ -352,6 +374,9 @@ class QueryAPI(BaseResource):
         if 'data_source_id' in query_def:
             query_def['data_source'] = query_def.pop('data_source_id')
 
+        query_def['last_modified_by'] = self.current_user
+
+        # TODO: use #save() with #dirty_fields.
         models.Query.update_instance(query_id, **query_def)
 
         query = models.Query.get_by_id(query_id)
@@ -451,10 +476,12 @@ class QueryResultListAPI(BaseResource):
             activity=params['query']
         ).save()
 
-        if params['ttl'] == 0:
+        max_age = int(params['max_age'])
+
+        if max_age == 0:
             query_result = None
         else:
-            query_result = models.QueryResult.get_latest(params['data_source_id'], params['query'], int(params['ttl']))
+            query_result = models.QueryResult.get_latest(params['data_source_id'], params['query'], max_age)
 
         if query_result:
             return {'query_result': query_result.to_dict()}
@@ -491,6 +518,24 @@ class QueryResultAPI(BaseResource):
             query_result = models.QueryResult.get_by_id(query_result_id)
 
         if query_result:
+            if isinstance(self.current_user, models.ApiUser):
+                event = {
+                    'user_id': None,
+                    'action': 'api_get',
+                    'timestamp': int(time.time()),
+                    'api_key': self.current_user.id,
+                    'file_type': filetype
+                }
+
+                if query_id:
+                    event['object_type'] = 'query'
+                    event['object_id'] = query_id
+                else:
+                    event['object_type'] = 'query_result'
+                    event['object_id'] = query_result_id
+
+                record_event.delay(event)
+
             if filetype == 'json':
                 data = json.dumps({'query_result': query_result.to_dict()}, cls=utils.JSONEncoder)
                 return make_response(data, 200, cache_headers)

redash/import_export.py

@@ -28,7 +28,7 @@ class Importer(object):
     def import_query(self, user, query):
         new_query = self._get_or_create(models.Query, query['id'], name=query['name'],
                                         user=user,
-                                        ttl=-1,
+                                        schedule=None,
                                         query=query['query'],
                                         query_hash=query['query_hash'],
                                         description=query['description'],

redash/models.py

@@ -11,8 +11,10 @@ import peewee
 from passlib.apps import custom_app_context as pwd_context
 from playhouse.postgres_ext import ArrayField, DateTimeTZField, PostgresqlExtDatabase
 from flask.ext.login import UserMixin, AnonymousUserMixin
+import psycopg2
 
-from redash import utils, settings
+from redash import utils, settings, redis_connection
+from redash.query_runner import get_query_runner
 
 
 class Database(object):
@@ -60,6 +62,30 @@ class BaseModel(peewee.Model):
     def get_by_id(cls, model_id):
         return cls.get(cls.id == model_id)
 
+    def pre_save(self, created):
+        pass
+
+    def post_save(self, created):
+        # Handler for post_save operations. Overriding if needed.
+        pass
+
+    def save(self, *args, **kwargs):
+        pk_value = self._get_pk_value()
+        created = kwargs.get('force_insert', False) or not bool(pk_value)
+        self.pre_save(created)
+        super(BaseModel, self).save(*args, **kwargs)
+        self.post_save(created)
+
+
+class ModelTimestampsMixin(BaseModel):
+    updated_at = DateTimeTZField(default=datetime.datetime.now)
+    created_at = DateTimeTZField(default=datetime.datetime.now)
+
+    def pre_save(self, created):
+        super(ModelTimestampsMixin, self).pre_save(created)
+
+        self.updated_at = datetime.datetime.now()
+
 
 class PermissionsCheckMixin(object):
     def has_permission(self, permission):
@@ -118,7 +144,7 @@ class Group(BaseModel):
         return unicode(self.id)
 
 
-class User(BaseModel, UserMixin, PermissionsCheckMixin):
+class User(ModelTimestampsMixin, BaseModel, UserMixin, PermissionsCheckMixin):
     DEFAULT_GROUPS = ['default']
 
     id = peewee.PrimaryKeyField()
@@ -134,7 +160,9 @@ class User(BaseModel, UserMixin, PermissionsCheckMixin):
         return {
             'id': self.id,
             'name': self.name,
-            'email': self.email
+            'email': self.email,
+            'updated_at': self.updated_at,
+            'created_at': self.created_at
         }
 
     def __init__(self, *args, **kwargs):
@@ -214,6 +242,23 @@ class DataSource(BaseModel):
             'type': self.type
         }
 
+    def get_schema(self, refresh=False):
+        key = "data_source:schema:{}".format(self.id)
+
+        cache = None
+        if not refresh:
+            cache = redis_connection.get(key)
+
+        if cache is None:
+            query_runner = get_query_runner(self.type, self.options)
+            schema = sorted(query_runner.get_schema(), key=lambda t: t['name'])
+
+            redis_connection.set(key, json.dumps(schema))
+        else:
+            schema = json.loads(cache)
+
+        return schema
+
     @classmethod
     def all(cls):
         return cls.select().order_by(cls.id.asc())
@@ -252,16 +297,16 @@ class QueryResult(BaseModel):
         return unused_results
 
     @classmethod
-    def get_latest(cls, data_source, query, ttl=0):
+    def get_latest(cls, data_source, query, max_age=0):
         query_hash = utils.gen_query_hash(query)
 
-        if ttl == -1:
+        if max_age == -1:
             query = cls.select().where(cls.query_hash == query_hash,
                                        cls.data_source == data_source).order_by(cls.retrieved_at.desc())
         else:
             query = cls.select().where(cls.query_hash == query_hash, cls.data_source == data_source,
                                        peewee.SQL("retrieved_at + interval '%s second' >= now() at time zone 'utc'",
-                                                  ttl)).order_by(cls.retrieved_at.desc())
+                                                  max_age)).order_by(cls.retrieved_at.desc())
 
         return query.first()
 
@@ -288,7 +333,28 @@ class QueryResult(BaseModel):
         return u"%d | %s | %s" % (self.id, self.query_hash, self.retrieved_at)
 
 
-class Query(BaseModel):
+def should_schedule_next(previous_iteration, now, schedule):
+    if schedule.isdigit():
+        ttl = int(schedule)
+        next_iteration = previous_iteration + datetime.timedelta(seconds=ttl)
+    else:
+        hour, minute = schedule.split(':')
+        hour, minute = int(hour), int(minute)
+
+        # The following logic is needed for cases like the following:
+        # - The query scheduled to run at 23:59.
+        # - The scheduler wakes up at 00:01.
+        # - Using naive implementation of comparing timestamps, it will skip the execution.
+        normalized_previous_iteration = previous_iteration.replace(hour=hour, minute=minute)
+        if normalized_previous_iteration > previous_iteration:
+            previous_iteration = normalized_previous_iteration - datetime.timedelta(days=1)
+
+        next_iteration = (previous_iteration + datetime.timedelta(days=1)).replace(hour=hour, minute=minute)
+
+    return now > next_iteration
+
+
+class Query(ModelTimestampsMixin, BaseModel):
     id = peewee.PrimaryKeyField()
     data_source = peewee.ForeignKeyField(DataSource)
     latest_query_data = peewee.ForeignKeyField(QueryResult, null=True)
@@ -297,21 +363,15 @@ class Query(BaseModel):
     query = peewee.TextField()
     query_hash = peewee.CharField(max_length=32)
     api_key = peewee.CharField(max_length=40)
-    ttl = peewee.IntegerField()
     user_email = peewee.CharField(max_length=360, null=True)
     user = peewee.ForeignKeyField(User)
+    last_modified_by = peewee.ForeignKeyField(User, null=True, related_name="modified_queries")
     is_archived = peewee.BooleanField(default=False, index=True)
-    created_at = DateTimeTZField(default=datetime.datetime.now)
+    schedule = peewee.CharField(max_length=10, null=True)
 
     class Meta:
         db_table = 'queries'
 
-    def create_default_visualizations(self):
-        table_visualization = Visualization(query=self, name="Table",
-                                            description='',
-                                            type="TABLE", options="{}")
-        table_visualization.save()
-
     def to_dict(self, with_stats=False, with_visualizations=False, with_user=True):
         d = {
             'id': self.id,
@@ -320,15 +380,17 @@ class Query(BaseModel):
             'description': self.description,
             'query': self.query,
             'query_hash': self.query_hash,
-            'ttl': self.ttl,
+            'schedule': self.schedule,
             'api_key': self.api_key,
             'is_archived': self.is_archived,
+            'updated_at': self.updated_at,
             'created_at': self.created_at,
             'data_source_id': self._data.get('data_source', None)
         }
 
         if with_user:
             d['user'] = self.user.to_dict()
+            d['last_modified_by'] = self.last_modified_by.to_dict()
         else:
             d['user_id'] = self._data['user']
 
@@ -344,7 +406,7 @@ class Query(BaseModel):
 
     def archive(self):
         self.is_archived = True
-        self.ttl = -1
+        self.schedule = None
 
         for vis in self.visualizations:
             for w in vis.widgets:
@@ -365,21 +427,19 @@ class Query(BaseModel):
 
     @classmethod
     def outdated_queries(cls):
-        # TODO: this will only find scheduled queries that were executed before. I think this is
-        # a reasonable assumption, but worth revisiting.
-        outdated_queries_ids = cls.select(
-            peewee.Func('first_value', cls.id).over(partition_by=[cls.query_hash, cls.data_source])) \
-            .join(QueryResult) \
-            .where(cls.ttl > 0,
-                   cls.is_archived==False,
-                   (QueryResult.retrieved_at +
-                    (cls.ttl * peewee.SQL("interval '1 second'"))) <
-                   peewee.SQL("(now() at time zone 'utc')"))
+        queries = cls.select(cls, QueryResult.retrieved_at, DataSource)\
+            .join(QueryResult)\
+            .switch(Query).join(DataSource)\
+            .where(cls.schedule != None)
 
-        queries = cls.select(cls, DataSource).join(DataSource) \
-            .where(cls.id << outdated_queries_ids)
+        now = datetime.datetime.utcnow().replace(tzinfo=psycopg2.tz.FixedOffsetTimezone(offset=0, name=None))
+        outdated_queries = {}
+        for query in queries:
+            if should_schedule_next(query.latest_query_data.retrieved_at, now, query.schedule):
+                key = "{}:{}".format(query.query_hash, query.data_source.id)
+                outdated_queries[key] = query
 
-        return queries
+        return outdated_queries.values()
 
     @classmethod
     def search(cls, term):
@@ -396,6 +456,7 @@ class Query(BaseModel):
 
     @classmethod
     def recent(cls, user_id):
+        # TODO: instead of t2 here, we should define table_alias for Query table
         return cls.select().where(Event.created_at > peewee.SQL("current_date - 7")).\
             join(Event, on=(Query.id == peewee.SQL("t2.object_id::integer"))).\
             where(Event.action << ('edit', 'execute', 'edit_name', 'edit_description', 'view_source')).\
@@ -414,10 +475,23 @@ class Query(BaseModel):
         update = cls.update(**kwargs).where(cls.id == query_id)
         return update.execute()
 
-    def save(self, *args, **kwargs):
+    def pre_save(self, created):
+        super(Query, self).pre_save(created)
         self.query_hash = utils.gen_query_hash(self.query)
         self._set_api_key()
-        super(Query, self).save(*args, **kwargs)
+
+        if self.last_modified_by is None:
+            self.last_modified_by = self.user
+
+    def post_save(self, created):
+        if created:
+            self._create_default_visualizations()
+
+    def _create_default_visualizations(self):
+        table_visualization = Visualization(query=self, name="Table",
+                                            description='',
+                                            type="TABLE", options="{}")
+        table_visualization.save()
 
     def _set_api_key(self):
         if not self.api_key:
@@ -436,7 +510,7 @@ class Query(BaseModel):
         return unicode(self.id)
 
 
-class Dashboard(BaseModel):
+class Dashboard(ModelTimestampsMixin, BaseModel):
     id = peewee.PrimaryKeyField()
     slug = peewee.CharField(max_length=140, index=True)
     name = peewee.CharField(max_length=100)
@@ -445,7 +519,6 @@ class Dashboard(BaseModel):
     layout = peewee.TextField()
     dashboard_filters_enabled = peewee.BooleanField(default=False)
     is_archived = peewee.BooleanField(default=False, index=True)
-    created_at = DateTimeTZField(default=datetime.datetime.now)
 
     class Meta:
         db_table = 'dashboards'
@@ -487,7 +560,9 @@ class Dashboard(BaseModel):
             'user_id': self._data['user'],
             'layout': layout,
             'dashboard_filters_enabled': self.dashboard_filters_enabled,
-            'widgets': widgets_layout
+            'widgets': widgets_layout,
+            'updated_at': self.updated_at,
+            'created_at': self.created_at
         }
 
     @classmethod
@@ -520,7 +595,7 @@ class Dashboard(BaseModel):
         return u"%s=%s" % (self.id, self.name)
 
 
-class Visualization(BaseModel):
+class Visualization(ModelTimestampsMixin, BaseModel):
     id = peewee.PrimaryKeyField()
     type = peewee.CharField(max_length=100)
     query = peewee.ForeignKeyField(Query, related_name='visualizations')
@@ -538,6 +613,8 @@ class Visualization(BaseModel):
             'name': self.name,
             'description': self.description,
             'options': json.loads(self.options),
+            'updated_at': self.updated_at,
+            'created_at': self.created_at
         }
 
         if with_query:
@@ -549,14 +626,13 @@ class Visualization(BaseModel):
         return u"%s %s" % (self.id, self.type)
 
 
-class Widget(BaseModel):
+class Widget(ModelTimestampsMixin, BaseModel):
     id = peewee.PrimaryKeyField()
     visualization = peewee.ForeignKeyField(Visualization, related_name='widgets', null=True)
     text = peewee.TextField(null=True)
     width = peewee.IntegerField()
     options = peewee.TextField()
     dashboard = peewee.ForeignKeyField(Dashboard, related_name='widgets', index=True)
-    created_at = DateTimeTZField(default=datetime.datetime.now)
 
     # unused; kept for backward compatability:
     type = peewee.CharField(max_length=100, null=True)
@@ -571,7 +647,9 @@ class Widget(BaseModel):
             'width': self.width,
             'options': json.loads(self.options),
             'dashboard_id': self._data['dashboard'],
-            'text': self.text
+            'text': self.text,
+            'updated_at': self.updated_at,
+            'created_at': self.created_at
         }
 
         if self.visualization and self.visualization.id:
@@ -592,7 +670,7 @@ class Widget(BaseModel):
 
 
 class Event(BaseModel):
-    user = peewee.ForeignKeyField(User, related_name="events")
+    user = peewee.ForeignKeyField(User, related_name="events", null=True)
     action = peewee.CharField()
     object_type = peewee.CharField()
     object_id = peewee.CharField(null=True)

redash/query_runner/__init__.py

@@ -57,6 +57,9 @@ class BaseQueryRunner(object):
     def run_query(self, query):
         raise NotImplementedError()
 
+    def get_schema(self):
+        return []
+
     @classmethod
     def to_dict(cls):
         return {

redash/query_runner/mysql.py

@@ -44,6 +44,41 @@ class Mysql(BaseQueryRunner):
     def __init__(self, configuration_json):
         super(Mysql, self).__init__(configuration_json)
 
+    def get_schema(self):
+        query = """
+        SELECT col.table_schema,
+               col.table_name,
+               col.column_name
+        FROM `information_schema`.`columns` col
+        INNER JOIN
+          (SELECT table_schema,
+                  TABLE_NAME
+           FROM information_schema.tables
+           WHERE table_type <> 'SYSTEM VIEW' AND table_schema NOT IN ('performance_schema', 'mysql')) tables ON tables.table_schema = col.table_schema
+        AND tables.TABLE_NAME = col.TABLE_NAME;
+        """
+
+        results, error = self.run_query(query)
+
+        if error is not None:
+            raise Exception("Failed getting schema.")
+
+        results = json.loads(results)
+
+        schema = {}
+        for row in results['rows']:
+            if row['table_schema'] != self.configuration['db']:
+                table_name = '{}.{}'.format(row['table_schema'], row['table_name'])
+            else:
+                table_name = row['table_name']
+
+            if table_name not in schema:
+                schema[table_name] = {'name': table_name, 'columns': []}
+
+            schema[table_name]['columns'].append(row['column_name'])
+
+        return schema.values()
+
     def run_query(self, query):
         import MySQLdb
 

redash/query_runner/pg.py

@@ -83,6 +83,34 @@ class PostgreSQL(BaseQueryRunner):
 
         self.connection_string = " ".join(values)
 
+    def get_schema(self):
+        query = """
+        SELECT table_schema, table_name, column_name
+        FROM information_schema.columns
+        WHERE table_schema NOT IN ('pg_catalog', 'information_schema');
+        """
+
+        results, error = self.run_query(query)
+
+        if error is not None:
+           raise Exception("Failed getting schema.")
+
+        results = json.loads(results)
+
+        schema = {}
+        for row in results['rows']:
+            if row['table_schema'] != 'public':
+                table_name = '{}.{}'.format(row['table_schema'], row['table_name'])
+            else:
+                table_name = row['table_name']
+
+            if table_name not in schema:
+                schema[table_name] = {'name': table_name, 'columns': []}
+
+            schema[table_name]['columns'].append(row['column_name'])
+
+        return schema.values()
+
     def run_query(self, query):
         connection = psycopg2.connect(self.connection_string, async=True)
         _wait(connection)

redash/settings.py

@@ -44,23 +44,19 @@ STATSD_HOST = os.environ.get('REDASH_STATSD_HOST', "127.0.0.1")
 STATSD_PORT = int(os.environ.get('REDASH_STATSD_PORT', "8125"))
 STATSD_PREFIX = os.environ.get('REDASH_STATSD_PREFIX', "redash")
 
-# The following is kept for backward compatability, and shouldn't be used any more.
-CONNECTION_ADAPTER = os.environ.get("REDASH_CONNECTION_ADAPTER", "pg")
-CONNECTION_STRING = os.environ.get("REDASH_CONNECTION_STRING", "user= password= host= port=5439 dbname=")
-
 # Connection settings for re:dash's own database (where we store the queries, results, etc)
 DATABASE_CONFIG = parse_db_url(os.environ.get("REDASH_DATABASE_URL", "postgresql://postgres"))
 
 # Celery related settings
 CELERY_BROKER = os.environ.get("REDASH_CELERY_BROKER", REDIS_URL)
 CELERY_BACKEND = os.environ.get("REDASH_CELERY_BACKEND", REDIS_URL)
-CELERY_FLOWER_URL = os.environ.get("REDASH_CELERY_FLOWER_URL", "/flower")
 
 # The following enables periodic job (every 5 minutes) of removing unused query results. Behind this "feature flag" until
 # proved to be "safe".
 QUERY_RESULTS_CLEANUP_ENABLED = parse_boolean(os.environ.get("REDASH_QUERY_RESULTS_CLEANUP_ENABLED", "false"))
 
 AUTH_TYPE = os.environ.get("REDASH_AUTH_TYPE", "hmac")
+PASSWORD_LOGIN_ENABLED = parse_boolean(os.environ.get("REDASH_PASSWORD_LOGIN_ENABLED", "true"))
 
 # Google Apps domain to allow access from; any user with email in this Google Apps will be allowed
 # access
@@ -70,17 +66,15 @@ GOOGLE_CLIENT_ID = os.environ.get("REDASH_GOOGLE_CLIENT_ID", "")
 GOOGLE_CLIENT_SECRET = os.environ.get("REDASH_GOOGLE_CLIENT_SECRET", "")
 GOOGLE_OAUTH_ENABLED = GOOGLE_CLIENT_ID and GOOGLE_CLIENT_SECRET
 
-PASSWORD_LOGIN_ENABLED = parse_boolean(os.environ.get("REDASH_PASSWORD_LOGIN_ENABLED", "true"))
 STATIC_ASSETS_PATH = fix_assets_path(os.environ.get("REDASH_STATIC_ASSETS_PATH", "../rd_ui/app/"))
-WORKERS_COUNT = int(os.environ.get("REDASH_WORKERS_COUNT", "2"))
-JOB_EXPIRY_TIME = int(os.environ.get("REDASH_JOB_EXPIRY_TIME", 3600*6))
+JOB_EXPIRY_TIME = int(os.environ.get("REDASH_JOB_EXPIRY_TIME", 3600 * 6))
 COOKIE_SECRET = os.environ.get("REDASH_COOKIE_SECRET", "c292a0a3aa32397cdb050e233733900f")
 LOG_LEVEL = os.environ.get("REDASH_LOG_LEVEL", "INFO")
 CLIENT_SIDE_METRICS = parse_boolean(os.environ.get("REDASH_CLIENT_SIDE_METRICS", "false"))
 ANALYTICS = os.environ.get("REDASH_ANALYTICS", "")
 
 # Query Runners
-QUERY_RUNNERS = [
+QUERY_RUNNERS = array_from_string(os.environ.get("REDASH_ENABLED_QUERY_RUNNERS", ",".join([
     'redash.query_runner.big_query',
     'redash.query_runner.graphite',
     'redash.query_runner.mongodb',
@@ -88,7 +82,7 @@ QUERY_RUNNERS = [
     'redash.query_runner.pg',
     'redash.query_runner.script',
     'redash.query_runner.url',
-]
+])))
 
 # Features:
 FEATURE_TABLES_PERMISSIONS = parse_boolean(os.environ.get("REDASH_FEATURE_TABLES_PERMISSIONS", "false"))

redash/tasks.py

@@ -218,6 +218,17 @@ def cleanup_query_results():
     logger.info("Deleted %d unused query results out of total of %d." % (deleted_count, total_unused_query_results))
 
 
+@celery.task(base=BaseTask)
+def refresh_schemas():
+    """
+    Refershs the datasources schema.
+    """
+
+    for ds in models.DataSource.all():
+        logger.info("Refreshing schema for: {}".format(ds.name))
+        ds.get_schema(refresh=True)
+
+
 @celery.task(bind=True, base=BaseTask, track_started=True)
 def execute_query(self, query, data_source_id):
     # TODO: maybe this should be a class?

redash/worker.py

@@ -15,6 +15,10 @@ celery_schedule = {
     'cleanup_tasks': {
         'task': 'redash.tasks.cleanup_tasks',
         'schedule': timedelta(minutes=5)
+    },
+    'refresh_schemas': {
+        'task': 'redash.tasks.refresh_schemas',
+        'schedule': timedelta(minutes=30)
     }
 }
 

setup/bootstrap.sh

@@ -98,7 +98,9 @@ if [ ! -f "/opt/redash/.env" ]; then
 fi
 
 # Install latest version
-REDASH_VERSION=${REDASH_VERSION-0.4.0.b589}
+# REDASH_VERSION=${REDASH_VERSION-0.4.0.b589}
+# modified by @fedex1 3/15/2015 seems to be the latest version at this point in time. 
+REDASH_VERSION=${REDASH_VERSION-0.6.0.b722}
 LATEST_URL="https://github.com/EverythingMe/redash/releases/download/v${REDASH_VERSION/.b/%2Bb}/redash.$REDASH_VERSION.tar.gz"
 VERSION_DIR="/opt/redash/redash.$REDASH_VERSION"
 REDASH_TARBALL=/tmp/redash.tar.gz

tests/__init__.py

@@ -1,12 +1,17 @@
+import os
+os.environ['REDASH_REDIS_URL'] = "redis://localhost:6379/5"
+
 import logging
 from unittest import TestCase
+import datetime
 from redash import settings
+
 settings.DATABASE_CONFIG = {
     'name': 'circle_test',
     'threadlocals': True
 }
 
-from redash import models
+from redash import models, redis_connection
 
 logging.getLogger('peewee').setLevel(logging.INFO)
 
@@ -18,4 +23,19 @@ class BaseTestCase(TestCase):
 
     def tearDown(self):
         models.db.close_db(None)
-        models.create_db(False, True)
+        models.create_db(False, True)
+        redis_connection.flushdb()
+
+    def assertResponseEqual(self, expected, actual):
+        for k, v in expected.iteritems():
+            if isinstance(v, datetime.datetime) or isinstance(actual[k], datetime.datetime):
+                continue
+
+            if isinstance(v, list):
+                continue
+
+            if isinstance(v, dict):
+                self.assertResponseEqual(v, actual[k])
+                continue
+
+            self.assertEqual(v, actual[k], "{} not equal (expected: {}, actual: {}).".format(k, v, actual[k]))

tests/factories.py

@@ -47,7 +47,7 @@ user_factory = ModelFactory(redash.models.User,
 data_source_factory = ModelFactory(redash.models.DataSource,
                                    name='Test',
                                    type='pg',
-                                   options='')
+                                   options='{"dbname": "test"}')
 
 
 dashboard_factory = ModelFactory(redash.models.Dashboard,
@@ -58,9 +58,9 @@ query_factory = ModelFactory(redash.models.Query,
                              name='New Query',
                              description='',
                              query='SELECT 1',
-                             ttl=-1,
                              user=user_factory.create,
                              is_archived=False,
+                             schedule=None,
                              data_source=data_source_factory.create)
 
 query_result_factory = ModelFactory(redash.models.QueryResult,
@@ -83,4 +83,4 @@ widget_factory = ModelFactory(redash.models.Widget,
                               width=1,
                               options='{}',
                               dashboard=dashboard_factory.create,
-                              visualization=visualization_factory.create)
+                              visualization=visualization_factory.create)

tests/test_controllers.py

@@ -1,6 +1,7 @@
 from contextlib import contextmanager
 import json
 import time
+import datetime
 from unittest import TestCase
 from flask import url_for
 from flask.ext.login import current_user
@@ -104,7 +105,11 @@ class DashboardAPITest(BaseTestCase, AuthenticationTestMixin):
         with app.test_client() as c, authenticated_user(c):
             rv = c.get('/api/dashboards/{0}'.format(d1.slug))
             self.assertEquals(rv.status_code, 200)
-            self.assertDictEqual(json.loads(rv.data), d1.to_dict(with_widgets=True))
+
+            expected = d1.to_dict(with_widgets=True)
+            actual = json.loads(rv.data)
+
+            self.assertResponseEqual(expected, actual)
 
     def test_get_non_existint_dashbaord(self):
         with app.test_client() as c, authenticated_user(c):
@@ -222,10 +227,13 @@ class QueryAPITest(BaseTestCase, AuthenticationTestMixin):
     def test_update_query(self):
         query = query_factory.create()
 
-        with app.test_client() as c, authenticated_user(c):
+        other_user = user_factory.create()
+
+        with app.test_client() as c, authenticated_user(c, user=other_user):
             rv = json_request(c.post, '/api/queries/{0}'.format(query.id), data={'name': 'Testing'})
             self.assertEqual(rv.status_code, 200)
-            self.assertEquals(rv.json['name'], 'Testing')
+            self.assertEqual(rv.json['name'], 'Testing')
+            self.assertEqual(rv.json['last_modified_by']['id'], other_user.id)
 
     def test_create_query(self):
         user = user_factory.create()
@@ -233,7 +241,7 @@ class QueryAPITest(BaseTestCase, AuthenticationTestMixin):
         query_data = {
             'name': 'Testing',
             'query': 'SELECT 1',
-            'ttl': 3600,
+            'schedule': "3600",
             'data_source_id': data_source.id
         }
 
@@ -256,9 +264,7 @@ class QueryAPITest(BaseTestCase, AuthenticationTestMixin):
             rv = json_request(c.get, '/api/queries/{0}'.format(query.id))
 
             self.assertEquals(rv.status_code, 200)
-            d = query.to_dict(with_visualizations=True)
-            d.pop('created_at')
-            self.assertDictContainsSubset(d, rv.json)
+            self.assertResponseEqual(rv.json, query.to_dict(with_visualizations=True))
 
     def test_get_all_queries(self):
         queries = [query_factory.create() for _ in range(10)]
@@ -294,7 +300,8 @@ class VisualizationAPITest(BaseTestCase):
             rv = json_request(c.delete, '/api/visualizations/{0}'.format(visualization.id))
 
             self.assertEquals(rv.status_code, 200)
-            self.assertEquals(models.Visualization.select().count(), 0)
+            # =1 because each query has a default table visualization.
+            self.assertEquals(models.Visualization.select().count(), 1)
 
     def test_update_visualization(self):
         visualization = visualization_factory.create()

tests/test_import.py

@@ -26,8 +26,10 @@ class ImportTest(BaseTestCase):
         self.assertEqual(dashboard.widgets.count(),
                          reduce(lambda s, row: s + len(row), self.dashboard['widgets'], 0))
 
-        self.assertEqual(models.Visualization.select().count(), dashboard.widgets.count()-1)
-        self.assertEqual(models.Query.select().count(), dashboard.widgets.count()-2)
+        queries_count = models.Query.select().count()
+
+        self.assertEqual(models.Visualization.select().count(), dashboard.widgets.count()+queries_count-1)
+        self.assertEqual(queries_count, dashboard.widgets.count()-2)
 
     def test_imports_updates_existing_models(self):
         importer = import_export.Importer(data_source=data_source_factory.create())

tests/test_models.py

@@ -1,10 +1,13 @@
 #encoding: utf8
 import datetime
 import json
+from unittest import TestCase
+import mock
 from tests import BaseTestCase
 from redash import models
 from factories import dashboard_factory, query_factory, data_source_factory, query_result_factory, user_factory, widget_factory
 from redash.utils import gen_query_hash
+from redash import query_runner
 
 
 class DashboardTest(BaseTestCase):
@@ -65,20 +68,102 @@ class QueryTest(BaseTestCase):
         self.assertNotIn(q1, queries)
         self.assertNotIn(q2, queries)
 
+    def test_save_creates_default_visualization(self):
+        q = query_factory.create()
+        self.assertEquals(q.visualizations.count(), 1)
+
+    def test_save_updates_updated_at_field(self):
+        # This should be a test of ModelTimestampsMixin, but it's easier to test in context of existing model... :-\
+        one_day_ago = datetime.datetime.today() - datetime.timedelta(days=1)
+        q = query_factory.create(created_at=one_day_ago, updated_at=one_day_ago)
+
+        q.save()
+
+        self.assertNotEqual(q.updated_at, one_day_ago)
+
+
+class ShouldScheduleNextTest(TestCase):
+    def test_interval_schedule_that_needs_reschedule(self):
+        now = datetime.datetime.now()
+        two_hours_ago = now - datetime.timedelta(hours=2)
+        self.assertTrue(models.should_schedule_next(two_hours_ago, now, "3600"))
+
+    def test_interval_schedule_that_doesnt_need_reschedule(self):
+        now = datetime.datetime.now()
+        half_an_hour_ago = now - datetime.timedelta(minutes=30)
+        self.assertFalse(models.should_schedule_next(half_an_hour_ago, now, "3600"))
+
+    def test_exact_time_that_needs_reschedule(self):
+        now = datetime.datetime.now()
+        yesterday = now - datetime.timedelta(days=1)
+        schedule = "{:02d}:00".format(now.hour - 3)
+        self.assertTrue(models.should_schedule_next(yesterday, now, schedule))
+
+    def test_exact_time_that_doesnt_need_reschedule(self):
+        now = datetime.datetime.now()
+        yesterday = (now - datetime.timedelta(days=1)).replace(hour=now.hour+3, minute=now.minute+1)
+        schedule = "{:02d}:00".format(now.hour + 3)
+        self.assertFalse(models.should_schedule_next(yesterday, now, schedule))
+
+    def test_exact_time_with_day_change(self):
+        now = datetime.datetime.now().replace(hour=0, minute=1)
+        previous = (now - datetime.timedelta(days=2)).replace(hour=23, minute=59)
+        schedule = "23:59".format(now.hour + 3)
+        self.assertTrue(models.should_schedule_next(previous, now, schedule))
+
+
+class QueryOutdatedQueriesTest(BaseTestCase):
+    # TODO: this test can be refactored to use mock version of should_schedule_next to simplify it.
+    def test_outdated_queries_skips_unscheduled_queries(self):
+        query = query_factory.create(schedule=None)
+        queries = models.Query.outdated_queries()
+
+        self.assertNotIn(query, queries)
+
+    def test_outdated_queries_works_with_ttl_based_schedule(self):
+        two_hours_ago = datetime.datetime.now() - datetime.timedelta(hours=2)
+        query = query_factory.create(schedule="3600")
+        query_result = query_result_factory.create(query=query, retrieved_at=two_hours_ago)
+        query.latest_query_data = query_result
+        query.save()
+
+        queries = models.Query.outdated_queries()
+        self.assertIn(query, queries)
+
+    def test_skips_fresh_queries(self):
+        half_an_hour_ago = datetime.datetime.now() - datetime.timedelta(minutes=30)
+        query = query_factory.create(schedule="3600")
+        query_result = query_result_factory.create(query=query, retrieved_at=half_an_hour_ago)
+        query.latest_query_data = query_result
+        query.save()
+
+        queries = models.Query.outdated_queries()
+        self.assertNotIn(query, queries)
+
+    def test_outdated_queries_works_with_specific_time_schedule(self):
+        half_an_hour_ago = datetime.datetime.utcnow() - datetime.timedelta(minutes=30)
+        query = query_factory.create(schedule=half_an_hour_ago.strftime('%H:%M'))
+        query_result = query_result_factory.create(query=query, retrieved_at=half_an_hour_ago-datetime.timedelta(days=1))
+        query.latest_query_data = query_result
+        query.save()
+
+        queries = models.Query.outdated_queries()
+        self.assertIn(query, queries)
+
 
 class QueryArchiveTest(BaseTestCase):
     def setUp(self):
         super(QueryArchiveTest, self).setUp()
 
     def test_archive_query_sets_flag(self):
-        query = query_factory.create(ttl=1)
+        query = query_factory.create()
         query.archive()
         query = models.Query.get_by_id(query.id)
 
         self.assertEquals(query.is_archived, True)
 
     def test_archived_query_doesnt_return_in_all(self):
-        query = query_factory.create(ttl=1)
+        query = query_factory.create(schedule="1")
         yesterday = datetime.datetime.now() - datetime.timedelta(days=1)
         query_result = models.QueryResult.store_result(query.data_source.id, query.query_hash, query.query, "1",
                                                        123, yesterday)
@@ -103,15 +188,53 @@ class QueryArchiveTest(BaseTestCase):
         self.assertRaises(models.Widget.DoesNotExist, models.Widget.get_by_id, widget.id)
 
     def test_removes_scheduling(self):
-        query = query_factory.create(ttl=1)
+        query = query_factory.create(schedule="1")
 
         query.archive()
 
         query = models.Query.get_by_id(query.id)
 
-        self.assertEqual(-1, query.ttl)
+        self.assertEqual(None, query.schedule)
 
 
+class DataSourceTest(BaseTestCase):
+    def test_get_schema(self):
+        return_value = [{'name': 'table', 'columns': []}]
+
+        with mock.patch('redash.query_runner.pg.PostgreSQL.get_schema') as patched_get_schema:
+            patched_get_schema.return_value = return_value
+
+            ds = data_source_factory.create()
+            schema = ds.get_schema()
+
+            self.assertEqual(return_value, schema)
+
+    def test_get_schema_uses_cache(self):
+        return_value = [{'name': 'table', 'columns': []}]
+        with mock.patch('redash.query_runner.pg.PostgreSQL.get_schema') as patched_get_schema:
+            patched_get_schema.return_value = return_value
+
+            ds = data_source_factory.create()
+            ds.get_schema()
+            schema = ds.get_schema()
+
+            self.assertEqual(return_value, schema)
+            self.assertEqual(patched_get_schema.call_count, 1)
+
+    def test_get_schema_skips_cache_with_refresh_true(self):
+        return_value = [{'name': 'table', 'columns': []}]
+        with mock.patch('redash.query_runner.pg.PostgreSQL.get_schema') as patched_get_schema:
+            patched_get_schema.return_value = return_value
+
+            ds = data_source_factory.create()
+            ds.get_schema()
+            new_return_value = [{'name': 'new_table', 'columns': []}]
+            patched_get_schema.return_value = new_return_value
+            schema = ds.get_schema(refresh=True)
+
+            self.assertEqual(new_return_value, schema)
+            self.assertEqual(patched_get_schema.call_count, 2)
+
 
 class QueryResultTest(BaseTestCase):
     def setUp(self):
@@ -145,7 +268,7 @@ class QueryResultTest(BaseTestCase):
         yesterday = datetime.datetime.now() - datetime.timedelta(days=1)
         qr = query_result_factory.create(retrieved_at=yesterday)
 
-        found_query_result = models.QueryResult.get_latest(qr.data_source, qr.query, ttl=60)
+        found_query_result = models.QueryResult.get_latest(qr.data_source, qr.query, max_age=60)
 
         self.assertIsNone(found_query_result)
 
@@ -153,7 +276,7 @@ class QueryResultTest(BaseTestCase):
         yesterday = datetime.datetime.now() - datetime.timedelta(seconds=30)
         qr = query_result_factory.create(retrieved_at=yesterday)
 
-        found_query_result = models.QueryResult.get_latest(qr.data_source, qr.query, ttl=120)
+        found_query_result = models.QueryResult.get_latest(qr.data_source, qr.query, max_age=120)
 
         self.assertEqual(found_query_result, qr)
 

tests/test_refresh_queries.py

@@ -10,7 +10,7 @@ from redash.tasks import refresh_queries
 # 2. test for the refresh_query task
 class TestRefreshQueries(BaseTestCase):
     def test_enqueues_outdated_queries(self):
-        query = query_factory.create(ttl=60)
+        query = query_factory.create(schedule="60")
         retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10)
         query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query,
                                                    query_hash=query.query_hash)
@@ -22,7 +22,7 @@ class TestRefreshQueries(BaseTestCase):
             add_job_mock.assert_called_with(query.query, query.data_source, scheduled=True)
 
     def test_skips_fresh_queries(self):
-        query = query_factory.create(ttl=1200)
+        query = query_factory.create(schedule="1200")
         retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10)
         query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query,
                                                    query_hash=query.query_hash)
@@ -32,7 +32,7 @@ class TestRefreshQueries(BaseTestCase):
             self.assertFalse(add_job_mock.called)
 
     def test_skips_queries_with_no_ttl(self):
-        query = query_factory.create(ttl=-1)
+        query = query_factory.create(schedule=None)
         retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10)
         query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query,
                                                    query_hash=query.query_hash)
@@ -42,8 +42,8 @@ class TestRefreshQueries(BaseTestCase):
             self.assertFalse(add_job_mock.called)
 
     def test_enqueues_query_only_once(self):
-        query = query_factory.create(ttl=60)
-        query2 = query_factory.create(ttl=60, query=query.query, query_hash=query.query_hash,
+        query = query_factory.create(schedule="60")
+        query2 = query_factory.create(schedule="60", query=query.query, query_hash=query.query_hash,
                                       data_source=query.data_source)
         retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10)
         query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query,
@@ -58,8 +58,8 @@ class TestRefreshQueries(BaseTestCase):
             add_job_mock.assert_called_once_with(query.query, query.data_source, scheduled=True)
 
     def test_enqueues_query_with_correct_data_source(self):
-        query = query_factory.create(ttl=60)
-        query2 = query_factory.create(ttl=60, query=query.query, query_hash=query.query_hash)
+        query = query_factory.create(schedule="60")
+        query2 = query_factory.create(schedule="60", query=query.query, query_hash=query.query_hash)
         retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10)
         query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query,
                                                    query_hash=query.query_hash)
@@ -74,9 +74,10 @@ class TestRefreshQueries(BaseTestCase):
             self.assertEquals(2, add_job_mock.call_count)
 
     def test_enqueues_only_for_relevant_data_source(self):
-        query = query_factory.create(ttl=60)
-        query2 = query_factory.create(ttl=3600, query=query.query, query_hash=query.query_hash)
-        retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10)
+        query = query_factory.create(schedule="60")
+        query2 = query_factory.create(schedule="3600", query=query.query, query_hash=query.query_hash)
+        import psycopg2
+        retrieved_at = datetime.datetime.utcnow().replace(tzinfo=psycopg2.tz.FixedOffsetTimezone(offset=0, name=None)) - datetime.timedelta(minutes=10)
         query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query,
                                                    query_hash=query.query_hash)
         query.latest_query_data = query_result