Merge pull request #239 from EverythingMe/feature_outdated_queries_monitor

Model and import script for events

.gitignore

@@ -5,6 +5,9 @@
 rd_ui/dist
 .DS_Store
 celerybeat-schedule
+.#*
+\#*#
+*~
 
 # Vagrant related
 .vagrant
@@ -12,4 +15,5 @@ Berksfile.lock
 redash/dump.rdb
 .env
 .ruby-version
-venv
+venv
+

Procfile.heroku

@@ -1,2 +1,2 @@
 web: ./manage.py runserver -p $PORT --host 0.0.0.0 -d -r
-worker: ./manage.py runworkers
+worker: ./bin/run celery worker --app=redash.worker --beat -Qqueries,celery,scheduled_queries

manage.py

@@ -2,6 +2,7 @@
 """
 CLI to manage redash.
 """
+import datetime
 from flask.ext.script import Manager, prompt_pass
 
 from redash import settings, models, __version__
@@ -39,6 +40,36 @@ def check_settings():
         if not callable(item) and not name.startswith("__") and not isinstance(item, ModuleType):
             print "{} = {}".format(name, item)
 
+@manager.command
+def import_events(events_file):
+    import json
+
+    count = 0
+    with open(events_file) as f:
+        for line in f:
+            try:
+                event = json.loads(line)
+
+                user = event.pop('user_id')
+                action = event.pop('action')
+                object_type = event.pop('object_type')
+                object_id = event.pop('object_id')
+                created_at = datetime.datetime.utcfromtimestamp(event.pop('timestamp'))
+                additional_properties = json.dumps(event)
+
+                models.Event.create(user=user, action=action, object_type=object_type, object_id=object_id,
+                                    additional_properties=additional_properties, created_at=created_at)
+
+                count += 1
+
+            except Exception as ex:
+                print "Failed importing line:"
+                print line
+                print ex.message
+
+    print "Importe %d rows" % count
+
+
 @database_manager.command
 def create_tables():
     """Creates the database tables."""
@@ -60,7 +91,7 @@ def drop_tables():
 @users_manager.option('--admin', dest='is_admin', action="store_true", default=False, help="set user as admin")
 @users_manager.option('--google', dest='google_auth', action="store_true", default=False, help="user uses Google Auth to login")
 @users_manager.option('--password', dest='password', default=None, help="Password for users who don't use Google Auth (leave blank for prompt).")
-@users_manager.option('--groups', dest='groups', default=models.Group.DEFAULT_PERMISSIONS, help="Comma seperated list of groups (leave blank for default).")
+@users_manager.option('--groups', dest='groups', default=models.User.DEFAULT_GROUPS, help="Comma seperated list of groups (leave blank for default).")
 def create(email, name, groups, is_admin=False, google_auth=False, password=None):
     print "Creating user (%s, %s)..." % (email, name)
     print "Admin: %r" % is_admin

migrations/create_events.py

@@ -0,0 +1,12 @@
+from redash.models import db
+from redash import models
+
+
+if __name__ == '__main__':
+    db.connect_db()
+
+    if not models.Event.table_exists():
+        print "Creating events table..."
+        models.Event.create_table()
+
+    db.close_db(None)

rd_ui/app/scripts/controllers/dashboard.js

@@ -1,14 +1,14 @@
 (function() {
   var DashboardCtrl = function($scope, Events, Widget, $routeParams, $http, $timeout, Dashboard) {
-    Events.record(currentUser, "view", "dashboard", dashboard.id);
-
     $scope.refreshEnabled = false;
     $scope.refreshRate = 60;
 
     var loadDashboard = _.throttle(function() {
       $scope.dashboard = Dashboard.get({ slug: $routeParams.dashboardSlug }, function (dashboard) {
-      $scope.$parent.pageTitle = dashboard.name;
-      var filters = {};
+        Events.record(currentUser, "view", "dashboard", dashboard.id);
+
+        $scope.$parent.pageTitle = dashboard.name;
+        var filters = {};
 
         $scope.dashboard.widgets = _.map($scope.dashboard.widgets, function (row) {
           return _.map(row, function (widget) {

rd_ui/app/scripts/controllers/query_view.js

@@ -24,7 +24,7 @@
       if (data) {
         data.id = $scope.query.id;
       } else {
-        data = $scope.query;
+        data = _.clone($scope.query);
       }
 
       options = _.extend({}, {
@@ -32,8 +32,8 @@
         errorMessage: 'Query could not be saved'
       }, options);
 
-      delete $scope.query.latest_query_data;
-      delete $scope.query.queryResult;
+      delete data.latest_query_data;
+      delete data.queryResult;
 
       return Query.save(data, function() {
         growl.addSuccessMessage(options.successMessage);

rd_ui/app/scripts/ng_highchart.js

@@ -13,11 +13,23 @@
     xAxis: {
       type: 'datetime'
     },
-    yAxis: {
-      title: {
-        text: null
+    yAxis: [
+      {
+        title: {
+          text: null
+        },
+        // showEmpty: true // by default
+      },
+      {
+        title: {
+          text: null
+        },
+        opposite: true,
+        showEmpty: false
       }
-    },
+    ],
+
+
     tooltip: {
       valueDecimals: 2,
       formatter: function () {

rd_ui/app/scripts/services/resources.js

@@ -10,13 +10,28 @@
         this.filters = undefined;
         this.filterFreeze = undefined;
 
+        var columnTypes = {};
+
         _.each(this.query_result.data.rows, function (row) {
           _.each(row, function (v, k) {
-            if (_.isString(v) && v.match(/^\d{4}-\d{2}-\d{2}/)) {
+            if (angular.isNumber(v)) {
+              columnTypes[k] = 'float';
+            } else if (_.isString(v) && v.match(/^\d{4}-\d{2}-\d{2}T/)) {
               row[k] = moment(v);
+              columnTypes[k] = 'datetime';
+            } else if (_.isString(v) && v.match(/^\d{4}-\d{2}-\d{2}/)) {
+              row[k] = moment(v);
+              columnTypes[k] = 'date';
             }
-          });
+          }, this);
+        }, this);
+
+        _.each(this.query_result.data.columns, function(column) {
+          if (columnTypes[column.name]) {
+            column.type = columnTypes[column.name];
+          }
         });
+
       } else if (this.job.status == 3) {
         this.status = "processing";
       } else {
@@ -133,7 +148,7 @@
       return this.filteredData;
     }
 
-    QueryResult.prototype.getChartData = function () {
+    QueryResult.prototype.getChartData = function (mapping) {
       var series = {};
 
       _.each(this.getData(), function (row) {
@@ -143,8 +158,15 @@
         var yValues = {};
 
         _.each(row, function (value, definition) {
-          var type = definition.split("::")[1];
           var name = definition.split("::")[0];
+          var type = definition.split("::")[1];
+          if (mapping) {
+            type = mapping[definition];
+          }
+
+          if (type == 'unused') {
+            return;
+          }
 
           if (type == 'x') {
             xValue = value;
@@ -374,9 +396,9 @@
         }
         queryResult = this.queryResult;
       } else if (this.latest_query_data_id && ttl != 0) {
-        queryResult = QueryResult.getById(this.latest_query_data_id);
+        this.queryResult = queryResult = QueryResult.getById(this.latest_query_data_id);
       } else if (this.data_source_id) {
-        queryResult = QueryResult.get(this.data_source_id, this.query, ttl);
+        this.queryResult = queryResult = QueryResult.get(this.data_source_id, this.query, ttl);
       }
 
       return queryResult;

rd_ui/app/scripts/visualizations/base.js

@@ -111,26 +111,23 @@
         scope.editRawOptions = currentUser.hasPermission('edit_raw_chart');
         scope.visTypes = Visualization.visualizationTypes;
 
-        scope.newVisualization = function (q) {
+        scope.newVisualization = function () {
           return {
-            'query_id': q.id,
             'type': Visualization.defaultVisualization.type,
             'name': Visualization.defaultVisualization.name,
-            'description': q.description || '',
+            'description': '',
             'options': Visualization.defaultVisualization.defaultOptions
           };
         }
 
         if (!scope.visualization) {
-          // create new visualization
-          // wait for query to load to populate with defaults
-          var unwatch = scope.$watch('query', function (q) {
-            if (q && q.id) {
+          var unwatch = scope.$watch('query.id', function (queryId) {
+            if (queryId) {
               unwatch();
 
-              scope.visualization = scope.newVisualization(q);
+              scope.visualization = scope.newVisualization();
             }
-          }, true);
+          });
         }
 
         scope.$watch('visualization.type', function (type, oldType) {
@@ -148,6 +145,8 @@
             Events.record(currentUser, "create", "visualization", null, {'type': scope.visualization.type});
           }
 
+          scope.visualization.query_id = scope.query.id;
+
           Visualization.save(scope.visualization, function success(result) {
             growl.addSuccessMessage("Visualization saved");
 

rd_ui/app/scripts/visualizations/chart.js

@@ -6,7 +6,7 @@
     var editTemplate = '<chart-editor></chart-editor>';
     var defaultOptions = {
       'series': {
-        'type': 'column',
+//        'type': 'column',
         'stacking': null
       }
     };
@@ -33,24 +33,45 @@
         $scope.chartSeries = [];
         $scope.chartOptions = {};
 
+        var reloadData = _.throttle(function(data) {
+          if (!data || ($scope.queryResult && $scope.queryResult.getData()) == null) {
+            $scope.chartSeries.splice(0, $scope.chartSeries.length);
+          } else {
+            $scope.chartSeries.splice(0, $scope.chartSeries.length);
+
+            _.each($scope.queryResult.getChartData($scope.options.columnMapping), function (s) {
+              var additional = {'stacking': 'normal'};
+              if ($scope.options.seriesOptions && $scope.options.seriesOptions[s.name]) {
+                additional = $scope.options.seriesOptions[s.name];
+                if (!additional.name || additional.name == "") {
+                  additional.name = s.name;
+                }
+              }
+              $scope.chartSeries.push(_.extend(s, additional));
+            });
+          }
+        }, 500);
+
         $scope.$watch('options', function (chartOptions) {
           if (chartOptions) {
             $scope.chartOptions = chartOptions;
           }
         });
-        $scope.$watch('queryResult && queryResult.getData()', function (data) {
-          if (!data || $scope.queryResult.getData() == null) {
-            $scope.chartSeries.splice(0, $scope.chartSeries.length);
-          } else {
-            $scope.chartSeries.splice(0, $scope.chartSeries.length);
 
-            _.each($scope.queryResult.getChartData(), function (s) {
-              $scope.chartSeries.push(_.extend(s, {'stacking': 'normal'}));
-            });
-          }
+        $scope.$watch('options.seriesOptions', function () {
+          reloadData(true);
+        }, true);
+
+
+        $scope.$watchCollection('options.columnMapping', function (chartOptions) {
+          reloadData(true);
+        });
+
+        $scope.$watch('queryResult && queryResult.getData()', function (data) {
+          reloadData(data);
         });
       }]
-    }
+    };
   });
 
   chartVisualization.directive('chartEditor', function () {
@@ -81,10 +102,26 @@
         scope.xAxisType = "datetime";
         scope.stacking = "none";
 
-        var chartOptionsUnwatch = null;
 
-        scope.$watch('visualization', function (visualization) {
-          if (visualization && visualization.type == 'CHART') {
+        scope.columnTypes = {
+          "X": "x",
+//          "X (Date time)": "x",
+//          "X (Linear)": "x-linear",
+//          "X (Category)": "x-category",
+          "Y": "y",
+          "Series": "series",
+          "Unused": "unused"
+        };
+
+        scope.series = [];
+
+        scope.columnTypeSelection = {};
+
+        var chartOptionsUnwatch = null,
+            columnsWatch = null;
+
+        scope.$watch('visualization.type', function (visualizationType) {
+          if (visualizationType == 'CHART') {
             if (scope.visualization.options.series.stacking === null) {
               scope.stacking = "none";
             } else if (scope.visualization.options.series.stacking === undefined) {
@@ -93,6 +130,65 @@
               scope.stacking = scope.visualization.options.series.stacking;
             }
 
+            columnsWatch = scope.$watch('queryResult.getId()', function(id) {
+              if (!id) {
+                return;
+              }
+
+              scope.columns = scope.queryResult.getColumns();
+
+              if (scope.visualization.options.columnMapping == undefined) {
+                scope.visualization.options.columnMapping = {};
+              }
+
+              scope.columnTypeSelection = scope.visualization.options.columnMapping;
+
+              _.each(scope.columns, function(column) {
+                var definition = column.name.split("::"),
+                    definedColumns = _.keys(scope.visualization.options.columnMapping);
+
+                if (_.indexOf(definedColumns, column.name) != -1) {
+                  // Skip already defined columns.
+                  return;
+                };
+
+                if (definition.length == 1) {
+                  scope.columnTypeSelection[column.name] = scope.visualization.options.columnMapping[column.name] = 'unused';
+                } else if (definition == 'multi-filter') {
+                  scope.columnTypeSelection[column.name] = scope.visualization.options.columnMapping[column.name] = 'series';
+                } else if (_.indexOf(_.values(scope.columnTypes), definition[1]) != -1) {
+                  scope.columnTypeSelection[column.name] = scope.visualization.options.columnMapping[column.name] = definition[1];
+                } else {
+                  scope.columnTypeSelection[column.name] = scope.visualization.options.columnMapping[column.name] = 'unused';
+                }
+              });
+            });
+
+            scope.$watchCollection('columnTypeSelection', function(selections) {
+              _.each(scope.columnTypeSelection, function(type, name) {
+                scope.visualization.options.columnMapping[name] = type;
+              });
+            });
+
+            scope.$watchCollection('visualization.options.columnMapping', function (chartOptions) {
+              scope.series = _.map(scope.query.getQueryResult().getChartData(scope.visualization.options.columnMapping), function (s) { return s.name; });
+
+              // TODO: remove uneeded ones?
+              if (scope.visualization.options.seriesOptions == undefined) {
+                scope.visualization.options.seriesOptions = {};
+              };
+
+              _.each(scope.series, function(s, i) {
+                if (scope.visualization.options.seriesOptions[s] == undefined) {
+                  scope.visualization.options.seriesOptions[s] = {'type': 'column', 'yAxis': 0};
+                }
+                scope.visualization.options.seriesOptions[s].zIndex = i;
+
+              });
+              scope.zIndexes = _.range(scope.series.length);
+              scope.yAxes = [[0, 'left'], [1, 'right']];
+            });
+
             chartOptionsUnwatch = scope.$watch("stacking", function (stacking) {
               if (stacking == "none") {
                 scope.visualization.options.series.stacking = null;
@@ -113,6 +209,11 @@
               chartOptionsUnwatch = null;
             }
 
+            if (columnsWatch) {
+              columnWatch();
+              columnWatch = null;
+            }
+
             if (xAxisUnwatch) {
               xAxisUnwatch();
               xAxisUnwatch = null;

rd_ui/app/scripts/visualizations/table.js

@@ -63,29 +63,19 @@
 
               var columnType = columns[i].type;
 
-              if (!columnType) {
-                var rawData = $scope.queryResult.getRawData();
-
-                if (rawData.length > 0) {
-                  var exampleData = rawData[0][col];
-                  if (angular.isNumber(exampleData)) {
-                    columnType = 'float';
-                  } else if (moment.isMoment(exampleData)) {
-                    if (exampleData._i.match(/^\d{4}-\d{2}-\d{2}T/)) {
-                      columnType = 'datetime';
-                    } else {
-                      columnType = 'date';
-                    }
-                  }
-                }
-              }
-
               if (columnType === 'integer') {
                 columnDefinition.formatFunction = 'number';
                 columnDefinition.formatParameter = 0;
               } else if (columnType === 'float') {
                 columnDefinition.formatFunction = 'number';
                 columnDefinition.formatParameter = 2;
+              } else if (columnType === 'boolean') {
+                columnDefinition.formatFunction = function (value) {
+                  if (value !== undefined) {
+                    return "" + value;
+                  }
+                  return value;
+                };
               } else if (columnType === 'date') {
                 columnDefinition.formatFunction = function (value) {
                   if (value) {

rd_ui/app/views/admin_status.html

@@ -20,31 +20,21 @@
                     <span class="badge" am-time-ago="manager.started_at*1000.0"></span>
                     Started
                 </li>
+                <li class="list-group-item">
+                    <span class="badge">{{manager.outdated_queries_count}}</span>
+                    Outdated Queries Count
+                </li>
 
                 <li class="list-group-item" ng-if="flowerUrl">
                     <a href="/admin/workers">Workers' Status</a>
                 </li>
             </ul>
             <ul class="list-group col-lg-4">
-                <div ng-repeat="worker in workers">
-                    <li class="list-group-item active">Worker {{$index+1}}</li>
-                    <li class="list-group-item">
-                    <span class="badge" am-time-ago="worker.updated_at*1000.0"></span>
-                    Updated
+                <li class="list-group-item active">Queues</li>
+                <li class="list-group-item" ng-repeat="(name, value) in manager.queues">
+                    <span class="badge">{{value.size}}</span>
+                    {{name}} ({{value.data_sources}})
                 </li>
-                <li class="list-group-item">
-                    <span class="badge" am-time-ago="worker.started_at*1000.0"></span>
-                    Started
-                </li>
-                <li class="list-group-item">
-                    <span class="badge">{{worker.jobs_count}}</span>
-                    Jobs Received
-                </li>
-                <li class="list-group-item">
-                    <span class="badge">{{worker.done_jobs_count}}</span>
-                    Jobs Done
-                </li>
-                </div>
             </ul>
         </div>
         <div class="panel-footer">Next refresh: <span am-time-ago="refresh_time"></span></div>

rd_ui/app/views/query.html

@@ -172,7 +172,7 @@
 
                         <div ng-show="selectedTab == 'add'">
                             <visualization-renderer visualization="newVisualization" query-result="queryResult"></visualization-renderer>
-                            <edit-visulatization-form visualization="newVisualization" query="query" ng-show="canEdit" open-editor="true" on-new-success="setVisualizationTab"></edit-visulatization-form>
+                            <edit-visulatization-form visualization="newVisualization" query="query" query-result="queryResult" ng-show="canEdit" open-editor="true" on-new-success="setVisualizationTab"></edit-visulatization-form>
                         </div>
                     </div>
                 </div>

rd_ui/app/views/visualizations/chart_editor.html

@@ -1,14 +1,91 @@
-<div>
-  <div class="form-group">
-      <label class="control-label">Chart Type</label>
-      <select required ng-model="visualization.options.series.type" ng-options="value as key for (key, value) in seriesTypes" class="form-control"></select>
-  </div>
+<div class="form-horizontal">
+    <div class="panel panel-default">
+        <div class="panel-body">
+            <div class="form-group">
+                <label class="control-label col-sm-2">Stacking</label>
 
-  <div class="form-group">
-      <label class="control-label">Stacking</label>
-      <select required ng-model="stacking" ng-options="value as key for (key, value) in stackingOptions" class="form-control"></select>
+                <div class="col-sm-10">
+                    <select required ng-model="stacking"
+                            ng-options="value as key for (key, value) in stackingOptions"
+                            class="form-control"></select>
+                </div>
+            </div>
+            <div class="form-group">
+                <label class="control-label col-sm-2">X Axis Type</label>
 
-      <label class="control-label">X Axis Type</label>
-      <select required ng-model="xAxisType" ng-options="value as key for (key, value) in xAxisOptions" class="form-control"></select>
-  </div>
+                <div class="col-sm-10">
+                    <select required ng-model="xAxisType" ng-options="value as key for (key, value) in xAxisOptions"
+                            class="form-control"></select>
+                </div>
+            </div>
+        </div>
+    </div>
+
+    <div class="row">
+        <div class="col-lg-6">
+            <div class="list-group">
+                <div class="list-group-item active">
+                    Columns Mapping
+                </div>
+                <div class="list-group-item">
+                    <div class="form-group" ng-repeat="column in columns">
+                        <label class="control-label col-sm-4">{{column.name}}</label>
+
+                        <div class="col-sm-8">
+                            <select ng-options="value as key for (key, value) in columnTypes" class="form-control"
+                                    ng-model="columnTypeSelection[column.name]"></select>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+
+        <div class="col-lg-6" ng-if="series.length > 0">
+            <div class="list-group" ng-repeat="seriesName in series">
+                <div class="list-group-item active">
+                    {{seriesName}}
+                </div>
+                <div class="list-group-item">
+                    <div class="form-group">
+                        <label class="control-label col-sm-3">Type</label>
+
+                        <div class="col-sm-9">
+                            <select required ng-model="visualization.options.seriesOptions[seriesName].type"
+                                    ng-options="value as key for (key, value) in seriesTypes"
+                                    class="form-control"></select>
+                        </div>
+                    </div>
+                    <div class="form-group">
+                        <label class="control-label col-sm-3">zIndex</label>
+
+                        <div class="col-sm-9">
+                            <select required ng-model="visualization.options.seriesOptions[seriesName].zIndex"
+                                    ng-options="o as o for o in zIndexes"
+                                    class="form-control"></select>
+                        </div>
+                    </div>
+                    <div class="form-group">
+                        <label class="control-label col-sm-3">y Axis</label>
+
+                        <div class="col-sm-9">
+                            <select required ng-model="visualization.options.seriesOptions[seriesName].yAxis"
+                                    ng-options="o[0] as o[1] for o in yAxes"
+                                    class="form-control"></select>
+                        </div>
+                    </div>
+                    <div class="form-group">
+                        <label class="control-label col-sm-3">Name</label>
+
+                        <div class="col-sm-9">
+                            <input name="seriesName" type="text" class="form-control"
+                                   ng-model="visualization.options.seriesOptions[seriesName].name"
+                                   placeholder="{{seriesName}}">
+                        </div>
+                    </div>
+                </div>
+            </div>
+
+
+        </div>
+    </div>
 </div>

redash/authentication.py

@@ -76,7 +76,7 @@ def create_and_login_user(app, user):
             user_object.save()
     except models.User.DoesNotExist:
         logger.debug("Creating user object (%r)", user.name)
-        user_object = models.User.create(name=user.name, email=user.email, groups = ['default'])
+        user_object = models.User.create(name=user.name, email=user.email, groups = models.User.DEFAULT_GROUPS)
 
     login_user(user_object, remember=True)
 

redash/controllers.py

@@ -110,7 +110,21 @@ def status_api():
 
     manager_status = redis_connection.hgetall('redash:status')
     status['manager'] = manager_status
-    status['manager']['queue_size'] = 'Unknown'#redis_connection.zcard('jobs')
+    status['manager']['queue_size'] = redis_connection.llen('queries') + redis_connection.llen('scheduled_queries')
+    status['manager']['outdated_queries_count'] = models.Query.outdated_queries().count()
+
+    queues = {}
+    for ds in models.DataSource.select():
+        for queue in (ds.queue_name, ds.scheduled_queue_name):
+            queues.setdefault(queue, set())
+            queues[queue].add(ds.name)
+
+    status['manager']['queues'] = {}
+    for queue, sources in queues.iteritems():
+        status['manager']['queues'][queue] = {
+            'data_sources': ', '.join(sources),
+            'size': redis_connection.llen(queue)
+        }
 
     return jsonify(status)
 
@@ -338,6 +352,7 @@ class VisualizationAPI(BaseResource):
         if 'options' in kwargs:
             kwargs['options'] = json.dumps(kwargs['options'])
         kwargs.pop('id', None)
+        kwargs.pop('query_id', None)
 
         update = models.Visualization.update(**kwargs).where(models.Visualization.id == visualization_id)
         update.execute()

redash/data/query_runner_bigquery.py

@@ -1,3 +1,4 @@
+import datetime
 import httplib2
 import json
 import logging
@@ -15,6 +16,38 @@ except ImportError:
 
 from redash.utils import JSONEncoder
 
+types_map = {
+    'INTEGER': 'integer',
+    'FLOAT': 'float',
+    'BOOLEAN': 'boolean',
+    'STRING': 'string',
+    'TIMESTAMP': 'datetime',
+}
+
+def transform_row(row, fields):
+    column_index = 0
+    row_data = {}
+
+    for cell in row["f"]:
+        field = fields[column_index]
+        cell_value = cell['v']
+
+        if cell_value is None:
+            pass
+        # Otherwise just cast the value
+        elif field['type'] == 'INTEGER':
+            cell_value = int(cell_value)
+        elif field['type'] == 'FLOAT':
+            cell_value = float(cell_value)
+        elif field['type'] == 'BOOLEAN':
+            cell_value = cell_value.lower() == "true"
+        elif field['type'] == 'TIMESTAMP':
+            cell_value = datetime.datetime.fromtimestamp(float(cell_value))
+
+        row_data[field["name"]] = cell_value
+        column_index += 1
+
+    return row_data
 
 def bigquery(connection_string):
     def load_key(filename):
@@ -67,28 +100,21 @@ def bigquery(connection_string):
             query_reply = get_query_results(jobs, project_id=project_id,
                                             job_id=insert_response['jobReference']['jobId'], start_index=current_row)
 
+            logging.debug("bigquery replied: %s", query_reply)
+
             rows = []
-            field_names = []
-            for f in query_reply["schema"]["fields"]:
-                field_names.append(f["name"])
 
             while ("rows" in query_reply) and current_row < query_reply['totalRows']:
                 for row in query_reply["rows"]:
-                    row_data = {}
-                    column_index = 0
-                    for cell in row["f"]:
-                        row_data[field_names[column_index]] = cell["v"]
-                        column_index += 1
-
-                    rows.append(row_data)
+                    rows.append(transform_row(row, query_reply["schema"]["fields"]))
 
                 current_row += len(query_reply['rows'])
                 query_reply = jobs.getQueryResults(projectId=project_id, jobId=query_reply['jobReference']['jobId'],
                                                    startIndex=current_row).execute()
 
-            columns = [{'name': name,
-                        'friendly_name': name,
-                        'type': None} for name in field_names]
+            columns = [{'name': f["name"],
+                        'friendly_name': f["name"],
+                        'type': types_map.get(f['type'], "string")} for f in query_reply["schema"]["fields"]]
 
             data = {
                 "columns": columns,

redash/models.py

@@ -102,11 +102,13 @@ class Group(BaseModel):
 
 
 class User(BaseModel, UserMixin):
+    DEFAULT_GROUPS = ['default']
+
     id = peewee.PrimaryKeyField()
     name = peewee.CharField(max_length=320)
     email = peewee.CharField(max_length=320, index=True, unique=True)
     password_hash = peewee.CharField(max_length=128, null=True)
-    groups = ArrayField(peewee.CharField, default=['default'])
+    groups = ArrayField(peewee.CharField, default=DEFAULT_GROUPS)
 
     class Meta:
         db_table = 'users'
@@ -337,7 +339,7 @@ class Query(BaseModel):
                    peewee.SQL("(now() at time zone 'utc')"))
 
         queries = cls.select(cls, DataSource).join(DataSource) \
-            .where(cls.id << outdated_queries_ids )
+            .where(cls.id << outdated_queries_ids)
 
         return queries
 
@@ -500,7 +502,24 @@ class Widget(BaseModel):
     def __unicode__(self):
         return u"%s" % self.id
 
-all_models = (DataSource, User, QueryResult, Query, Dashboard, Visualization, Widget, ActivityLog, Group)
+
+class Event(BaseModel):
+    # user, action, object_type, object_id, additional_properties
+    user = peewee.ForeignKeyField(User, related_name="events")
+    action = peewee.CharField()
+    object_type = peewee.CharField()
+    object_id = peewee.IntegerField()
+    additional_properties = peewee.TextField(null=True)
+    created_at = peewee.DateTimeField(default=datetime.datetime.now)
+
+    class Meta:
+        db_table = 'events'
+
+    def __unicode__(self):
+        return u"%s,%s,%s,%s" % (self._data['user'], self.action, self.object_type, self.object_id)
+
+
+all_models = (DataSource, User, QueryResult, Query, Dashboard, Visualization, Widget, ActivityLog, Group, Event)
 
 
 def init_db():

redash/tasks.py

@@ -120,6 +120,7 @@ class QueryTask(object):
     def _job_lock_id(query_hash, data_source_id):
         return "query_hash_job:%s:%s" % (data_source_id, query_hash)
 
+
 @celery.task(base=BaseTask)
 def refresh_queries():
     # self.status['last_refresh_at'] = time.time()
@@ -149,6 +150,7 @@ def refresh_queries():
 
     statsd_client.gauge('manager.seconds_since_refresh', now - float(status.get('last_refresh_at', now)))
 
+
 @celery.task(bind=True, base=BaseTask, track_started=True)
 def execute_query(self, query, data_source_id):
     # TODO: maybe this should be a class?