Merge pull request #5 from hailocab/permission_system

Permission system

bin/run

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+source .env
+"$@"

bin/test_multithreading.py

@@ -0,0 +1,63 @@
+"""
+Script to test concurrency (multithreading/multiprocess) issues with the workers. Use with caution.
+"""
+import json
+import atfork
+atfork.monkeypatch_os_fork_functions()
+import atfork.stdlib_fixer
+atfork.stdlib_fixer.fix_logging_module()
+
+import time
+from redash.data import worker
+from redash import models, data_manager, redis_connection
+
+if __name__ == '__main__':
+    models.create_db(True, False)
+
+    print "Creating data source..."
+    data_source = models.DataSource.create(name="Concurrency", type="pg", options="dbname=postgres")
+
+    print "Clear jobs/hashes:"
+    redis_connection.delete("jobs")
+    query_hashes = redis_connection.keys("query_hash_*")
+    if query_hashes:
+        redis_connection.delete(*query_hashes)
+
+    starting_query_results_count = models.QueryResult.select().count()
+    jobs_count = 5000
+    workers_count = 10
+
+    print "Creating jobs..."
+    for i in xrange(jobs_count):
+        query = "SELECT {}".format(i)
+        print "Inserting: {}".format(query)
+        data_manager.add_job(query=query, priority=worker.Job.LOW_PRIORITY,
+                             data_source=data_source)
+
+    print "Starting workers..."
+    workers = data_manager.start_workers(workers_count)
+
+    print "Waiting for jobs to be done..."
+    keep_waiting = True
+    while keep_waiting:
+        results_count = models.QueryResult.select().count() - starting_query_results_count
+        print "QueryResults: {}".format(results_count)
+        time.sleep(5)
+        if results_count == jobs_count:
+            print "Yay done..."
+            keep_waiting = False
+
+    data_manager.stop_workers()
+
+    qr_count = 0
+    for qr in models.QueryResult.select():
+        number = int(qr.query.split()[1])
+        data_number = json.loads(qr.data)['rows'][0].values()[0]
+
+        if number != data_number:
+            print "Oops? {} != {} ({})".format(number, data_number, qr.id)
+        qr_count += 1
+
+    print "Verified {} query results.".format(qr_count)
+
+    print "Done."

manage.py

@@ -33,7 +33,7 @@ def runworkers():
 
         logging.info("Cleaning old workers: %s", old_workers)
 
-        data_manager.start_workers(settings.WORKERS_COUNT, settings.CONNECTION_ADAPTER, settings.CONNECTION_STRING)
+        data_manager.start_workers(settings.WORKERS_COUNT)
         logging.info("Workers started.")
 
         while True:
@@ -52,6 +52,15 @@ def runworkers():
 def make_shell_context():
     return dict(app=app, db=db, models=models)
 
+@manager.command
+def check_settings():
+    from types import ModuleType
+
+    for name in dir(settings):
+        item = getattr(settings, name)
+        if not callable(item) and not name.startswith("__") and not isinstance(item, ModuleType):
+            print "{} = {}".format(name, item)
+
 @database_manager.command
 def create_tables():
     """Creates the database tables."""
@@ -71,18 +80,23 @@ def drop_tables():
 @users_manager.option('name', help="User's full name")
 @users_manager.option('--admin', dest='is_admin', action="store_true", default=False, help="set user as admin")
 @users_manager.option('--google', dest='google_auth', action="store_true", default=False, help="user uses Google Auth to login")
-def create(email, name, is_admin=False, google_auth=False):
+@users_manager.option('--password', dest='password', default=None, help="Password for users who don't use Google Auth (leave blank for prompt).")
+@users_manager.option('--groups', dest='groups', default=['default'], help="Comma seperated list of groups (leave blank for default).")
+def create(email, name, groups, is_admin=False, google_auth=False, password=None):
     print "Creating user (%s, %s)..." % (email, name)
     print "Admin: %r" % is_admin
     print "Login with Google Auth: %r\n" % google_auth
+    if isinstance(groups, basestring) and len(groups) > 0:
+        groups = groups.split(',')
+    else:
+        groups = ['default']
 
-    permissions = models.User.DEFAULT_PERMISSIONS
     if is_admin:
-        permissions += ['admin']
+        groups += ['admin']
 
-    user = models.User(email=email, name=name, permissions=permissions)
+    user = models.User(email=email, name=name, groups=groups)
     if not google_auth:
-        password = prompt_pass("Password")
+        password = password or prompt_pass("Password")
         user.hash_password(password)
 
     try:
@@ -101,8 +115,4 @@ manager.add_command("users", users_manager)
 manager.add_command("import", import_manager)
 
 if __name__ == '__main__':
-    channel = logging.StreamHandler()
-    logging.getLogger().addHandler(channel)
-    logging.getLogger().setLevel(settings.LOG_LEVEL)
-
     manager.run()

migrations/add_view_query_permission.py

@@ -0,0 +1,13 @@
+import peewee
+from redash import db
+from redash import models
+
+
+if __name__ == '__main__':
+    db.connect_db()
+
+    previous_default_permissions = models.User.DEFAULT_PERMISSIONS[:]
+    previous_default_permissions.remove('view_query')
+    models.User.update(permissions=peewee.fn.array_append(models.User.permissions, 'view_query')).where(peewee.SQL("'view_source' = any(permissions)")).execute()
+
+    db.close_db(None)

migrations/create_data_sources.py

@@ -0,0 +1,48 @@
+import logging
+import peewee
+from playhouse.migrate import Migrator
+from redash import db
+from redash import models
+from redash import settings
+
+if __name__ == '__main__':
+    db.connect_db()
+
+    if not models.DataSource.table_exists():
+        print "Creating data_sources table..."
+        models.DataSource.create_table()
+
+        default_data_source = models.DataSource.create(name="Default",
+                                                       type=settings.CONNECTION_ADAPTER,
+                                                       options=settings.CONNECTION_STRING)
+    else:
+        default_data_source = models.DataSource.select().first()
+
+    migrator = Migrator(db.database)
+    models.Query.data_source.null = True
+    models.QueryResult.data_source.null = True
+    try:
+        with db.database.transaction():
+            migrator.add_column(models.Query, models.Query.data_source, "data_source_id")
+    except peewee.ProgrammingError:
+        print "Failed to create data_source_id column -- assuming it already exists"
+
+    try:
+        with db.database.transaction():
+            migrator.add_column(models.QueryResult, models.QueryResult.data_source, "data_source_id")
+    except peewee.ProgrammingError:
+        print "Failed to create data_source_id column -- assuming it already exists"
+
+    print "Updating data source to existing one..."
+    models.Query.update(data_source=default_data_source.id).execute()
+    models.QueryResult.update(data_source=default_data_source.id).execute()
+
+    with db.database.transaction():
+        print "Setting data source to non nullable..."
+        migrator.set_nullable(models.Query, models.Query.data_source, False)
+
+    with db.database.transaction():
+        print "Setting data source to non nullable..."
+        migrator.set_nullable(models.QueryResult, models.QueryResult.data_source, False)
+
+    db.close_db(None)

migrations/permissions_migration.py

@@ -0,0 +1,24 @@
+from playhouse.migrate import Migrator
+from redash import db
+from redash import models
+
+
+if __name__ == '__main__':
+    db.connect_db()
+    migrator = Migrator(db.database)
+    
+    if not models.Group.table_exists():
+        print "Creating groups table..."
+        models.Group.create_table()
+    
+    with db.database.transaction():
+        models.Group.insert(name='admin', permissions=['admin'], tables=['*']).execute()
+        models.Group.insert(name='default', permissions=models.Group.DEFAULT_PERMISSIONS, tables=['*']).execute()
+        
+        migrator.drop_column(models.User, 'permissions')
+        migrator.add_column(models.User, models.User.groups, 'groups')
+        
+        models.User.update(groups=['admin', 'default']).where(models.User.is_admin == True).execute()
+        models.User.update(groups=['default']).where(models.User.is_admin == False).execute()
+
+    db.close_db(None)

rd_ui/app/index.html

@@ -14,6 +14,7 @@
     <link rel="stylesheet" href="/bower_components/gridster/dist/jquery.gridster.css">
     <link rel="stylesheet" href="/bower_components/pivottable/examples/pivot.css">
     <link rel="stylesheet" href="/bower_components/cornelius/src/cornelius.css">
+    <link rel="stylesheet" href="/bower_components/select2/select2.css">
     <link rel="stylesheet" href="/styles/redash.css">
     <!-- endbuild -->
 </head>
@@ -35,7 +36,7 @@
     <div class="collapse navbar-collapse navbar-ex1-collapse">
         <ul class="nav navbar-nav">
             <li class="active" ng-show="pageTitle"><a class="page-title" ng-bind="pageTitle"></a></li>
-            <li class="dropdown">
+            <li class="dropdown" ng-show="groupedDashboards.length > 0 || otherDashboards.length > 0 || currentUser.hasPermission('create_dashboard')">
                 <a href="#" class="dropdown-toggle" data-toggle="dropdown"><span class="glyphicon glyphicon-th-large"></span> <b class="caret"></b></a>
                 <ul class="dropdown-menu">
                     <span ng-repeat="(name, group) in groupedDashboards">
@@ -51,11 +52,11 @@
                     <li ng-repeat="dashboard in otherDashboards">
                         <a role="menu-item" ng-href="/dashboard/{{dashboard.slug}}" ng-bind="dashboard.name"></a>
                     </li>
-                    <li class="divider" ng-show="currentUser.hasPermission('create_dashboard')"></li>
+                    <li class="divider" ng-show="currentUser.hasPermission('create_dashboard') && (groupedDashboards.length > 0 || otherDashboards.length > 0)"></li>
                     <li><a data-toggle="modal" href="#new_dashboard_dialog" ng-show="currentUser.hasPermission('create_dashboard')">New Dashboard</a></li>
                 </ul>
             </li>
-            <li class="dropdown">
+            <li class="dropdown" ng-show="currentUser.hasPermission('view_query')">
                 <a href="#" class="dropdown-toggle" data-toggle="dropdown">Queries <b class="caret"></b></a>
                 <ul class="dropdown-menu">
                     <li ng-show="currentUser.hasPermission('create_query')"><a href="/queries/new">New Query</a></li>
@@ -106,6 +107,8 @@
 <script src="/bower_components/cornelius/src/cornelius.js"></script>
 <script src="/bower_components/mousetrap/mousetrap.js"></script>
 <script src="/bower_components/mousetrap/plugins/global-bind/mousetrap-global-bind.js"></script>
+<script src="/bower_components/select2/select2.js"></script>
+<script src="/bower_components/angular-ui-select2/src/select2.js"></script>
 
 <script src="/scripts/ng_highchart.js"></script>
 <script src="/scripts/ng_smart_table.js"></script>
@@ -116,17 +119,22 @@
 <!-- build:js({.tmp,app}) /scripts/scripts.js -->
 <script src="/scripts/app.js"></script>
 <script src="/scripts/services/services.js"></script>
+<script src="/scripts/services/resources.js"></script>
 <script src="/scripts/services/notifications.js"></script>
 <script src="/scripts/services/dashboards.js"></script>
 <script src="/scripts/controllers/controllers.js"></script>
+<script src="/scripts/controllers/dashboard.js"></script>
 <script src="/scripts/controllers/admin_controllers.js"></script>
 <script src="/scripts/controllers/query_view.js"></script>
+<script src="/scripts/controllers/query_source.js"></script>
 <script src="/scripts/visualizations/base.js"></script>
 <script src="/scripts/visualizations/chart.js"></script>
 <script src="/scripts/visualizations/cohort.js"></script>
 <script src="/scripts/visualizations/table.js"></script>
 <script src="/scripts/visualizations/pivot.js"></script>
-<script src="/scripts/directives.js"></script>
+<script src="/scripts/directives/directives.js"></script>
+<script src="/scripts/directives/query_directives.js"></script>
+<script src="/scripts/directives/dashboard_directives.js"></script>
 <script src="/scripts/filters.js"></script>
 <!-- endbuild -->
 

rd_ui/app/scripts/app.js

@@ -8,6 +8,7 @@ angular.module('redash', [
     'redash.visualization',
     'ui.codemirror',
     'highchart',
+    'ui.select2',
     'angular-growl',
     'angularMoment',
     'ui.bootstrap',
@@ -17,16 +18,9 @@ angular.module('redash', [
 ]).config(['$routeProvider', '$locationProvider', '$compileProvider', 'growlProvider',
     function($routeProvider, $locationProvider, $compileProvider, growlProvider) {
 
-        function getQuery(Query, $q, $route) {
-            var defer = $q.defer();
-
-            Query.get({
-                'id': $route.current.params.queryId
-            }, function(query) {
-                defer.resolve(query);
-            });
-
-            return defer.promise;
+        function getQuery(Query, $route) {
+            var query = Query.get({'id': $route.current.params.queryId });
+            return query.$promise;
         };
 
         $compileProvider.aHrefSanitizationWhitelist(/^\s*(https?|http|data):/);
@@ -43,39 +37,29 @@ angular.module('redash', [
             reloadOnSearch: false
         });
         $routeProvider.when('/queries/new', {
-            templateUrl: '/views/queryview.html',
-            controller: 'QueryViewCtrl',
+            templateUrl: '/views/query.html',
+            controller: 'QuerySourceCtrl',
             reloadOnSearch: false,
             resolve: {
-                'viewSource': function isViewSource() {
-                    return true;
-                }
+                'query': ['Query', function newQuery(Query) {
+                    return Query.newQuery();
+                }]
             }
         });
-        // TODO
-        // we should have 2 controllers: queryViewCtrl and queryEditCtrl
         $routeProvider.when('/queries/:queryId', {
-            templateUrl: '/views/queryview.html',
+            templateUrl: '/views/query.html',
             controller: 'QueryViewCtrl',
             reloadOnSearch: false,
             resolve: {
-                'query': ['Query', '$q', '$route', getQuery]
+                'query': ['Query', '$route', getQuery]
             }
         });
-        $routeProvider.when('/queries/:queryId/fiddle', {
-            templateUrl: '/views/queryfiddle.html',
-            controller: 'QueryFiddleCtrl',
-            reloadOnSearch: false
-        });
         $routeProvider.when('/queries/:queryId/source', {
-            templateUrl: '/views/queryview.html',
-            controller: 'QueryViewCtrl',
+            templateUrl: '/views/query.html',
+            controller: 'QuerySourceCtrl',
             reloadOnSearch: false,
             resolve: {
-                'query': ['Query', '$q', '$route', getQuery],
-                'viewSource': function isViewSource() {
-                    return true;
-                }
+                'query': ['Query', '$route', getQuery]
             }
         });
         $routeProvider.when('/admin/status', {
@@ -90,9 +74,6 @@ angular.module('redash', [
             redirectTo: '/'
         });
 
-        Highcharts.setOptions({
-            colors: ["#4572A7", "#AA4643", "#89A54E", "#80699B", "#3D96AE",
-                        "#DB843D", "#92A8CD", "#A47D7C", "#B5CA92"]
-        });
+
     }
 ]);

rd_ui/app/scripts/controllers/controllers.js

@@ -1,292 +1,4 @@
 (function () {
-    var DashboardCtrl = function ($scope, $routeParams, $http, $timeout, Dashboard) {
-        $scope.refreshEnabled = false;
-        $scope.refreshRate = 60;
-        $scope.dashboard = Dashboard.get({slug: $routeParams.dashboardSlug}, function(dashboard) {
-            $scope.$parent.pageTitle = dashboard.name;
-        });
-
-        var autoRefresh = function() {
-            if ($scope.refreshEnabled) {
-                $timeout(function() {
-                    Dashboard.get({slug: $routeParams.dashboardSlug}, function(dashboard) {
-                        var newWidgets = _.groupBy(_.flatten(dashboard.widgets), 'id');
-
-                        _.each($scope.dashboard.widgets, function(row) {
-                            _.each(row, function(widget, i) {
-                                var newWidget = newWidgets[widget.id];
-                                if (newWidget && newWidget[0].visualization.query.latest_query_data_id != widget.visualization.query.latest_query_data_id ) {
-                                    row[i] = newWidget[0];
-                                }
-                            });
-                        });
-
-                        autoRefresh();
-                    });
-
-                }, $scope.refreshRate);
-            };
-        }
-
-        $scope.triggerRefresh = function(){
-            $scope.refreshEnabled = !$scope.refreshEnabled;
-
-            if ($scope.refreshEnabled) {
-                var refreshRate = _.min(_.flatten($scope.dashboard.widgets), function(widget) {
-                    return widget.visualization.query.ttl;
-                }).visualization.query.ttl;
-
-                $scope.refreshRate = _.max([120, refreshRate * 2])*1000;
-
-                autoRefresh();
-            }
-        };
-    };
-
-    var WidgetCtrl = function ($scope, $http, $location, Query) {
-        $scope.deleteWidget = function() {
-            if (!confirm('Are you sure you want to remove "' + $scope.widget.visualization.name + '" from the dashboard?')) {
-                return;
-            }
-
-            $http.delete('/api/widgets/' + $scope.widget.id).success(function() {
-                $scope.dashboard.widgets = _.map($scope.dashboard.widgets, function(row) {
-                    return _.filter(row, function(widget) {
-                        return widget.id != $scope.widget.id;
-                    })
-                });
-            });
-        };
-
-        $scope.open = function(query, visualization) {
-            $location.path('/queries/' + query.id);
-            $location.hash(visualization.id);
-        }
-
-        $scope.query = new Query($scope.widget.visualization.query);
-        $scope.queryResult = $scope.query.getQueryResult();
-
-        $scope.updateTime = (new Date($scope.queryResult.getUpdatedAt())).toISOString();
-        $scope.nextUpdateTime = moment(new Date(($scope.query.updated_at + $scope.query.ttl + $scope.query.runtime + 300) * 1000)).fromNow();
-
-        $scope.updateTime = '';
-    }
-
-    var QueryFiddleCtrl = function ($scope, $window, $location, $routeParams, $http, $location, growl, notifications, Query, Visualization) {
-        var DEFAULT_TAB = 'table';
-        var pristineHash = null;
-        var leavingPageText = "You will lose your changes if you leave";
-
-        $scope.dirty = undefined;
-        $scope.newVisualization = undefined;
-
-        $window.onbeforeunload = function(){
-            if (currentUser.canEdit($scope.query) && $scope.dirty) {
-                return leavingPageText;
-            }
-        }
-
-        Mousetrap.bindGlobal("meta+s", function(e) {
-            e.preventDefault();
-
-            if (currentUser.canEdit($scope.query)) {
-                $scope.saveQuery();
-            }
-        });
-
-        $scope.$on('$locationChangeStart', function(event, next, current) {
-            if (next.split("#")[0] == current.split("#")[0]) {
-                return;
-            }
-
-            if (!currentUser.canEdit($scope.query)) {
-                return;
-            }
-
-            if($scope.dirty &&
-                !confirm(leavingPageText + "\n\nAre you sure you want to leave this page?")) {
-                event.preventDefault();
-            } else {
-                Mousetrap.unbind("meta+s");
-            }
-        });
-
-        $scope.$parent.pageTitle = "Query Fiddle";
-
-        $scope.$watch(function() {return $location.hash()}, function(hash) {
-            $scope.selectedTab = hash || DEFAULT_TAB;
-        });
-
-        $scope.lockButton = function (lock) {
-            $scope.queryExecuting = lock;
-        };
-
-        $scope.formatQuery = function() {
-            $scope.editorOptions.readOnly = 'nocursor';
-
-            $http.post('/api/queries/format', {'query': $scope.query.query}).success(function(response) {
-                $scope.query.query = response;
-                $scope.editorOptions.readOnly = false;
-            })
-        }
-
-        $scope.saveQuery = function (duplicate, oldId) {
-            if (!oldId) {
-                oldId = $scope.query.id;
-            }
-            
-            delete $scope.query.latest_query_data;
-            $scope.query.$save(function (q) {
-                pristineHash = q.getHash();
-                $scope.dirty = false;
-
-                if (duplicate) {
-                    growl.addInfoMessage("Query duplicated.", {ttl: 2000});
-                } else{
-                    growl.addSuccessMessage("Query saved.", {ttl: 2000});
-                }
-
-                if (oldId != q.id) {
-                    if (oldId == undefined) {
-                        $location.path($location.path().replace('new', q.id)).replace();
-                    } else {
-                        // TODO: replace this with a safer method
-                        $location.path($location.path().replace(oldId, q.id)).replace();
-
-                        // Reset visualizations tab to table after duplicating a query:
-                        $location.hash('table');
-                    }
-                }
-            }, function(httpResponse) {
-                growl.addErrorMessage("Query could not be saved");
-            });
-        };
-
-        $scope.duplicateQuery = function () {
-            var oldId = $scope.query.id;
-            $scope.query.id = null;
-            $scope.query.ttl = -1;
-
-            $scope.saveQuery(true, oldId);
-        };
-
-        // Query Editor:
-        $scope.editorOptions = {
-            mode: 'text/x-sql',
-            lineWrapping: true,
-            lineNumbers: true,
-            readOnly: false,
-            matchBrackets: true,
-            autoCloseBrackets: true
-        };
-
-        $scope.refreshOptions = [
-            {value: -1, name: 'No Refresh'},
-            {value: 60, name: 'Every minute'},
-        ]
-
-        _.each(_.range(1, 13), function(i) {
-            $scope.refreshOptions.push({value: i*3600, name: 'Every ' + i + 'h'});
-        })
-
-        $scope.refreshOptions.push({value: 24*3600, name: 'Every 24h'});
-        $scope.refreshOptions.push({value: 7*24*3600, name: 'Once a week'});
-
-        $scope.$watch('queryResult && queryResult.getError()', function (newError, oldError) {
-            if (newError == undefined) {
-                return;
-            }
-
-            if (oldError == undefined && newError != undefined) {
-                $scope.lockButton(false);
-            }
-        });
-
-        $scope.$watch('queryResult && queryResult.getData()', function (data, oldData) {
-            if (!data) {
-                return;
-            }
-
-            if ($scope.queryResult.getId() == null) {
-                $scope.dataUri = "";
-            } else {
-                $scope.dataUri = '/api/queries/' + $scope.query.id + '/results/' + $scope.queryResult.getId() + '.csv';
-                $scope.dataFilename = $scope.query.name.replace(" ", "_") + moment($scope.queryResult.getUpdatedAt()).format("_YYYY_MM_DD") + ".csv";
-            }
-        });
-
-        $scope.$watch("queryResult && queryResult.getStatus()", function (status) {
-            if (!status) {
-                return;
-            }
-
-            if (status == "done") {
-                if ($scope.query.id && $scope.query.latest_query_data_id != $scope.queryResult.getId() &&
-                    $scope.query.query_hash == $scope.queryResult.query_result.query_hash) {
-                    Query.save({'id': $scope.query.id, 'latest_query_data_id': $scope.queryResult.getId()})
-                }
-                $scope.query.latest_query_data_id = $scope.queryResult.getId();
-
-                notifications.showNotification("re:dash", $scope.query.name + " updated.");
-
-                $scope.lockButton(false);
-            }
-        });
-
-        if ($routeParams.queryId != undefined) {
-            $scope.query = Query.get({id: $routeParams.queryId}, function(q) {
-                pristineHash = q.getHash();
-                $scope.dirty = false;
-                $scope.queryResult = $scope.query.getQueryResult();
-            });
-        } else {
-            $scope.query = new Query({query: "", name: "New Query", ttl: -1, user: currentUser});
-            $scope.lockButton(false);
-        }
-
-        $scope.$watch('query.name', function() {
-            $scope.$parent.pageTitle = $scope.query.name;
-        });
-
-        $scope.$watch(function() {
-            return $scope.query.getHash();
-        }, function(newHash) {
-            $scope.dirty = (newHash !== pristineHash);
-        });
-
-        $scope.executeQuery = function() {
-            $scope.queryResult = $scope.query.getQueryResult(0);
-            $scope.lockButton(true);
-            $scope.cancelling = false;
-        };
-
-        $scope.cancelExecution = function() {
-            $scope.cancelling = true;
-            $scope.queryResult.cancelExecution();
-        };
-
-        $scope.deleteVisualization = function($e, vis) {
-            $e.preventDefault();
-            if (confirm('Are you sure you want to delete ' + vis.name + ' ?')) {
-                Visualization.delete(vis);
-                if ($scope.selectedTab == vis.id) {
-                    $scope.selectedTab = DEFAULT_TAB;
-                }
-                $scope.query.visualizations =
-                    $scope.query.visualizations.filter(function(v) {
-                        return vis.id !== v.id;
-                    });
-            }
-        };
-        
-        unbind = $scope.$watch('selectedTab == "add"', function(newPanel) {
-            if (newPanel && $routeParams.queryId == undefined) {
-                unbind();
-                $scope.saveQuery();
-            }
-        });
-    }
-
     var QueriesCtrl = function($scope, $http, $location, $filter, Query) {
         $scope.$parent.pageTitle = "All Queries";
         $scope.gridConfig = {
@@ -437,10 +149,7 @@
     }
 
     angular.module('redash.controllers', [])
-        .controller('DashboardCtrl', ['$scope', '$routeParams', '$http', '$timeout', 'Dashboard', DashboardCtrl])
-        .controller('WidgetCtrl', ['$scope', '$http', '$location', 'Query', WidgetCtrl])
         .controller('QueriesCtrl', ['$scope', '$http', '$location', '$filter', 'Query', QueriesCtrl])
-        .controller('QueryFiddleCtrl', ['$scope', '$window', '$location', '$routeParams', '$http', '$location', 'growl', 'notifications', 'Query', 'Visualization', QueryFiddleCtrl])
         .controller('IndexCtrl', ['$scope', 'Dashboard', IndexCtrl])
         .controller('MainCtrl', ['$scope', 'Dashboard', 'notifications', MainCtrl]);
 })();

rd_ui/app/scripts/controllers/dashboard.js

@@ -0,0 +1,78 @@
+(function() {
+  var DashboardCtrl = function($scope, $routeParams, $http, $timeout, Dashboard) {
+    $scope.refreshEnabled = false;
+    $scope.refreshRate = 60;
+    $scope.dashboard = Dashboard.get({
+      slug: $routeParams.dashboardSlug
+    }, function(dashboard) {
+      $scope.$parent.pageTitle = dashboard.name;
+    });
+
+    var autoRefresh = function() {
+      if ($scope.refreshEnabled) {
+        $timeout(function() {
+          Dashboard.get({
+            slug: $routeParams.dashboardSlug
+          }, function(dashboard) {
+            var newWidgets = _.groupBy(_.flatten(dashboard.widgets), 'id');
+
+            _.each($scope.dashboard.widgets, function(row) {
+              _.each(row, function(widget, i) {
+                var newWidget = newWidgets[widget.id];
+                if (newWidget && newWidget[0].visualization.query.latest_query_data_id != widget.visualization.query.latest_query_data_id) {
+                  row[i] = newWidget[0];
+                }
+              });
+            });
+
+            autoRefresh();
+          });
+
+        }, $scope.refreshRate);
+      };
+    }
+
+    $scope.triggerRefresh = function() {
+      $scope.refreshEnabled = !$scope.refreshEnabled;
+
+      if ($scope.refreshEnabled) {
+        var refreshRate = _.min(_.flatten($scope.dashboard.widgets), function(widget) {
+          return widget.visualization.query.ttl;
+        }).visualization.query.ttl;
+
+        $scope.refreshRate = _.max([120, refreshRate * 2]) * 1000;
+
+        autoRefresh();
+      }
+    };
+  };
+
+  var WidgetCtrl = function($scope, $http, $location, Query) {
+    $scope.deleteWidget = function() {
+      if (!confirm('Are you sure you want to remove "' + $scope.widget.visualization.name + '" from the dashboard?')) {
+        return;
+      }
+
+      $http.delete('/api/widgets/' + $scope.widget.id).success(function() {
+        $scope.dashboard.widgets = _.map($scope.dashboard.widgets, function(row) {
+          return _.filter(row, function(widget) {
+            return widget.id != $scope.widget.id;
+          })
+        });
+      });
+    };
+
+    $scope.query = new Query($scope.widget.visualization.query);
+    $scope.queryResult = $scope.query.getQueryResult();
+
+    $scope.updateTime = (new Date($scope.queryResult.getUpdatedAt())).toISOString();
+    $scope.nextUpdateTime = moment(new Date(($scope.query.updated_at + $scope.query.ttl + $scope.query.runtime + 300) * 1000)).fromNow();
+
+    $scope.updateTime = '';
+  };
+
+  angular.module('redash.controllers')
+    .controller('DashboardCtrl', ['$scope', '$routeParams', '$http', '$timeout', 'Dashboard', DashboardCtrl])
+    .controller('WidgetCtrl', ['$scope', '$http', '$location', 'Query', WidgetCtrl])
+
+})();

rd_ui/app/scripts/controllers/query_source.js

@@ -0,0 +1,100 @@
+(function() {
+  'use strict';
+
+  function QuerySourceCtrl($controller, $scope, $location, Query, Visualization, KeyboardShortcuts) {
+    // extends QueryViewCtrl
+    $controller('QueryViewCtrl', {$scope: $scope});
+    // TODO:
+    // This doesn't get inherited. Setting it on this didn't work either (which is weird).
+    // Obviously it shouldn't be repeated, but we got bigger fish to fry.
+    var DEFAULT_TAB = 'table';
+
+    var isNewQuery = !$scope.query.id,
+        queryText = $scope.query.query,
+        // ref to QueryViewCtrl.saveQuery
+        saveQuery = $scope.saveQuery,
+        shortcuts = {
+          'meta+s': function () {
+            if ($scope.canEdit) {
+              $scope.saveQuery();
+            }
+          }
+        };
+
+    $scope.sourceMode = true;
+    $scope.canEdit = currentUser.canEdit($scope.query);
+    $scope.isDirty = false;
+
+    $scope.newVisualization = undefined;
+
+    KeyboardShortcuts.bind(shortcuts);
+
+    // @override
+    $scope.saveQuery = function(options, data) {
+      var savePromise = saveQuery(options, data);
+
+      savePromise.then(function(savedQuery) {
+        queryText = savedQuery.query;
+        $scope.isDirty = $scope.query.query !== queryText;
+
+        if (isNewQuery) {
+          // redirect to new created query (keep hash)
+          $location.path(savedQuery.getSourceLink()).replace();
+        }
+      });
+
+      return savePromise;
+    };
+
+    $scope.duplicateQuery = function() {
+      $scope.query.id = null;
+      $scope.query.ttl = -1;
+
+      $scope.saveQuery({
+        successMessage: 'Query forked',
+        errorMessage: 'Query could not be forked'
+      }).then(function redirect(savedQuery) {
+        // redirect to forked query (clear hash)
+        $location.url(savedQuery.getSourceLink()).replace()
+      });
+    };
+
+    $scope.deleteVisualization = function($e, vis) {
+      $e.preventDefault();
+      if (confirm('Are you sure you want to delete ' + vis.name + ' ?')) {
+        Visualization.delete(vis);
+        if ($scope.selectedTab == vis.id) {
+          $scope.selectedTab = DEFAULT_TAB;
+          $location.hash($scope.selectedTab);
+        }
+        $scope.query.visualizations =
+          $scope.query.visualizations.filter(function(v) {
+            return vis.id !== v.id;
+          });
+      }
+    };
+
+    $scope.$watch('query.query', function(newQueryText) {
+      $scope.isDirty = (newQueryText !== queryText);
+    });
+
+    $scope.$on('$destroy', function destroy() {
+      KeyboardShortcuts.unbind(shortcuts);
+    });
+
+    if (isNewQuery) {
+      // save new query when creating a visualization
+      var unbind = $scope.$watch('selectedTab == "add"', function(triggerSave) {
+        if (triggerSave) {
+          unbind();
+          $scope.saveQuery();
+        }
+      });
+    }
+  }
+
+  angular.module('redash.controllers').controller('QuerySourceCtrl', [
+    '$controller', '$scope', '$location', 'Query',
+    'Visualization', 'KeyboardShortcuts', QuerySourceCtrl
+    ]);
+})();

rd_ui/app/scripts/controllers/query_view.js

@@ -1,159 +1,88 @@
-(function () {
+(function() {
   'use strict';
 
-    var QueryViewCtrl = function ($scope, $window, $route, $http, $location, growl, notifications, Query, Visualization) {
+  function QueryViewCtrl($scope, $route, $location, notifications, growl, Query, DataSource) {
     var DEFAULT_TAB = 'table';
-        var pristineHash = "";
-        var leavingPageText = "You will lose your changes if you leave";
-        var route = $route.current;
-
-        $scope.dirty = undefined;
-        $scope.isNewQuery = false;
-        $scope.isOwner = false;
-        $scope.canEdit = false;
-        $scope.canFork = false;
-
-        $scope.isSourceVisible = route.locals.viewSource;
 
+    $scope.query = $route.current.locals.query;
+    $scope.queryResult = $scope.query.getQueryResult();
     $scope.queryExecuting = false;
 
-        $scope.newVisualization = undefined;
+    $scope.isQueryOwner = currentUser.id === $scope.query.user.id;
+    $scope.canViewSource = currentUser.hasPermission('view_source');
 
-        updateSourceHref();
-
-        $window.onbeforeunload = function () {
-            if ($scope.canEdit && $scope.dirty) {
-                return leavingPageText;
-            }
-        }
-
-        function updateSourceHref() {
-            if ($scope.query && $scope.query.id) {
-                $scope.sourceHref = $scope.isSourceVisible ?
-                    $location.url().replace('source', '') : getQuerySourceUrl($scope.query.id);
-            }
-        };
-
-        function getQuerySourceUrl(queryId) {
-            return '/queries/' + queryId + '/source#' + $location.hash();
-        };
-
-        Mousetrap.bindGlobal("meta+s", function (e) {
-            e.preventDefault();
-
-            if ($scope.canEdit) {
-                $scope.saveQuery();
-            }
+    $scope.dataSources = DataSource.get(function(dataSources) {
+      $scope.query.data_source_id = $scope.query.data_source_id || dataSources[0].id;
     });
 
-        $scope.$on('$locationChangeStart', function (event, next, current) {
-            if (next.split("#")[0] == current.split("#")[0]) {
-                return;
-            }
-
-            if (!$scope.canEdit) {
-                return;
-            }
-
-            if ($scope.dirty && !confirm(leavingPageText + "\n\nAre you sure you want to leave this page?")) {
-                event.preventDefault();
-            } else {
-                Mousetrap.unbind("meta+s");
-            }
-        });
-
-        $scope.$watch(function () {
-            return $location.hash()
-        }, function (hash) {
-            $scope.selectedTab = hash || DEFAULT_TAB;
-            updateSourceHref();
-        });
-
-        $scope.lockButton = function (lock) {
+    $scope.lockButton = function(lock) {
       $scope.queryExecuting = lock;
     };
 
-        $scope.formatQuery = function () {
-            $scope.editorOptions.readOnly = 'nocursor';
-
-            $http.post('/api/queries/format', {
-                'query': $scope.query.query
-            }).success(function (response) {
-                $scope.query.query = response;
-                $scope.editorOptions.readOnly = false;
-            })
+    $scope.saveQuery = function(options, data) {
+      if (data) {
+        data.id = $scope.query.id;
+      } else {
+        data = $scope.query;
       }
 
-        $scope.saveQuery = function (duplicate, oldId) {
-            if (!oldId) {
-                oldId = $scope.query.id;
-            }
+      options = _.extend({}, {
+        successMessage: 'Query saved',
+        errorMessage: 'Query could not be saved'
+      }, options);
 
       delete $scope.query.latest_query_data;
-            $scope.query.$save(function (q) {
-                pristineHash = q.getHash();
-                $scope.dirty = false;
 
-                if (duplicate) {
-                    growl.addSuccessMessage("Query forked");
-                } else {
-                    growl.addSuccessMessage("Query saved");
-                }
-
-                if (oldId != q.id) {
-                    $location.url(getQuerySourceUrl(q.id)).replace();
-                }
-            }, function (httpResponse) {
-                growl.addErrorMessage("Query could not be saved");
-            });
-        };
-
-        $scope.duplicateQuery = function () {
-            var oldId = $scope.query.id;
-            $scope.query.id = null;
-            $scope.query.ttl = -1;
-
-            $scope.saveQuery(true, oldId);
-        };
-
-        // Query Editor:
-        $scope.editorOptions = {
-            mode: 'text/x-sql',
-            lineWrapping: true,
-            lineNumbers: true,
-            readOnly: false,
-            matchBrackets: true,
-            autoCloseBrackets: true
-        };
-
-        $scope.refreshOptions = [
-            {
-                value: -1,
-                name: 'No Refresh'
-            },
-            {
-                value: 60,
-                name: 'Every minute'
-            },
-        ]
-
-        _.each(_.range(1, 13), function (i) {
-            $scope.refreshOptions.push({
-                value: i * 3600,
-                name: 'Every ' + i + 'h'
-            });
+      return Query.save(data, function() {
+        growl.addSuccessMessage(options.successMessage);
+      }, function(httpResponse) {
+        growl.addErrorMessage(options.errorMessage);
       })
+      .$promise;
+    }
 
-        $scope.refreshOptions.push({
-            value: 24 * 3600,
-            name: 'Every 24h'
-        });
-        $scope.refreshOptions.push({
-            value: 7 * 24 * 3600,
-            name: 'Once a week'
+    $scope.saveDescription = function() {
+      $scope.saveQuery(undefined, {'description': $scope.query.description});
+    };
+
+    $scope.saveName = function() {
+      $scope.saveQuery(undefined, {'name': $scope.query.name});
+    };
+
+    $scope.executeQuery = function() {
+      $scope.queryResult = $scope.query.getQueryResult(0);
+      $scope.lockButton(true);
+      $scope.cancelling = false;
+    };
+
+    $scope.cancelExecution = function() {
+      $scope.cancelling = true;
+      $scope.queryResult.cancelExecution();
+    };
+
+    $scope.updateDataSource = function() {
+        $scope.query.latest_query_data = null;
+        $scope.query.latest_query_data_id = null;
+        Query.save({
+            'id': $scope.query.id,
+            'data_source_id': $scope.query.data_source_id,
+            'latest_query_data_id': null
         });
 
-        $scope.$watch('queryResult && queryResult.getError()', function (newError, oldError) {
+        $scope.executeQuery();
+    };
+
+    $scope.setVisualizationTab = function (visualization) {
+      $scope.selectedTab = visualization.id;
+      $location.hash(visualization.id);
+    };
+
+    $scope.$watch('query.name', function() {
+      $scope.$parent.pageTitle = $scope.query.name;
+    });
+
+    $scope.$watch('queryResult && queryResult.getError()',
+      function(newError, oldError) {
         if (newError == undefined) {
           return;
         }
@@ -163,26 +92,36 @@
         }
       });
 
-        $scope.$watch('queryResult && queryResult.getData()', function (data, oldData) {
+    $scope.$watch('queryResult && queryResult.getData()',
+      function(data, oldData) {
         if (!data) {
           return;
         }
 
+        $scope.filters = $scope.queryResult.getFilters();
+
         if ($scope.queryResult.getId() == null) {
           $scope.dataUri = "";
         } else {
-                $scope.dataUri = '/api/queries/' + $scope.query.id + '/results/' + $scope.queryResult.getId() + '.csv';
-                $scope.dataFilename = $scope.query.name.replace(" ", "_") + moment($scope.queryResult.getUpdatedAt()).format("_YYYY_MM_DD") + ".csv";
+          $scope.dataUri =
+            '/api/queries/' + $scope.query.id + '/results/' +
+            $scope.queryResult.getId() + '.csv';
+
+          $scope.dataFilename =
+            $scope.query.name.replace(" ", "_") +
+            moment($scope.queryResult.getUpdatedAt()).format("_YYYY_MM_DD") +
+            ".csv";
         }
       });
 
-        $scope.$watch("queryResult && queryResult.getStatus()", function (status) {
+    $scope.$watch("queryResult && queryResult.getStatus()", function(status) {
       if (!status) {
         return;
       }
 
       if (status == "done") {
-                if ($scope.query.id && $scope.query.latest_query_data_id != $scope.queryResult.getId() &&
+        if ($scope.query.id &&
+          $scope.query.latest_query_data_id != $scope.queryResult.getId() &&
           $scope.query.query_hash == $scope.queryResult.query_result.query_hash) {
           Query.save({
             'id': $scope.query.id,
@@ -197,86 +136,14 @@
       }
     });
 
-        // view or source pages: controller is instantiated with a query
-        if (route.locals.query) {
-            $scope.query = route.locals.query;
-            pristineHash = $scope.query.getHash();
-            $scope.dirty = false;
-            $scope.queryResult = $scope.query.getQueryResult();
-
-            $scope.isOwner = currentUser.canEdit($scope.query);
-            $scope.canEdit = $scope.isSourceVisible && $scope.isOwner;
-            $scope.canFork = true;
-
-        } else {
-            // new query
-            $scope.query = new Query({
-                query: "",
-                name: "New Query",
-                ttl: -1,
-                user: currentUser
-            });
-            $scope.lockButton(false);
-            $scope.isOwner = $scope.canEdit = true;
-            $scope.isNewQuery = true;
-        }
-
-        $scope.$watch('query.name', function () {
-            $scope.$parent.pageTitle = $scope.query.name;
-        });
-
-        $scope.$watch(function () {
-            return $scope.query.getHash();
-        }, function (newHash) {
-            $scope.dirty = (newHash !== pristineHash);
-        });
-
-        $scope.executeQuery = function () {
-            $scope.queryResult = $scope.query.getQueryResult(0);
-            $scope.lockButton(true);
-            $scope.cancelling = false;
-        };
-
-        $scope.cancelExecution = function () {
-            $scope.cancelling = true;
-            $scope.queryResult.cancelExecution();
-        };
-
-        $scope.deleteVisualization = function ($e, vis) {
-            $e.preventDefault();
-            if (confirm('Are you sure you want to delete ' + vis.name + ' ?')) {
-                Visualization.delete(vis);
-                if ($scope.selectedTab == vis.id) {
-                    $scope.selectedTab = DEFAULT_TAB;
-                    $location.hash($scope.selectedTab);
-                }
-                $scope.query.visualizations =
-                    $scope.query.visualizations.filter(function (v) {
-                        return vis.id !== v.id;
-                    });
-            }
-        };
-
-        var unbind = $scope.$watch('selectedTab == "add"', function (newPanel) {
-            if (newPanel && route.params.queryId == undefined) {
-                unbind();
-                $scope.saveQuery();
-            }
+    $scope.$watch(function() {
+      return $location.hash()
+    }, function(hash) {
+      $scope.selectedTab = hash || DEFAULT_TAB;
     });
   };
 
-    angular.module('redash.controllers').controller('QueryViewCtrl',
-        [
-            '$scope',
-            '$window',
-            '$route',
-            '$http',
-            '$location',
-            'growl',
-            'notifications',
-            'Query',
-            'Visualization',
-            QueryViewCtrl
-        ]);
-
+  angular.module('redash.controllers')
+    .controller('QueryViewCtrl',
+      ['$scope', '$route', '$location', 'notifications', 'growl', 'Query', 'DataSource', QueryViewCtrl]);
 })();

rd_ui/app/scripts/directives.js

@@ -1,347 +0,0 @@
-(function() {
-    'use strict';
-
-    var directives = angular.module('redash.directives', []);
-
-    directives.directive('rdTab', function() {
-        return {
-            restrict: 'E',
-            scope: {
-                'tabId': '@',
-                'name': '@'
-            },
-            transclude: true,
-            template: '<li class="rd-tab" ng-class="{active: tabId==selectedTab}"><a href="#{{tabId}}">{{name}}<span ng-transclude></span></a></li>',
-            replace: true,
-            link: function(scope) {
-                scope.$watch(function(){return scope.$parent.selectedTab}, function(tab) {
-                    scope.selectedTab = tab;
-                });
-            }
-        }
-    });
-
-    directives.directive('rdTabs', ['$location', function($location) {
-        return {
-            restrict: 'E',
-            scope: {
-                tabsCollection: '=',
-                selectedTab: '='
-            },
-            template: '<ul class="nav nav-tabs"><li ng-class="{active: tab==selectedTab}" ng-repeat="tab in tabsCollection"><a href="#{{tab.key}}">{{tab.name}}</a></li></ul>',
-            replace: true,
-            link: function($scope, element, attrs) {
-                $scope.selectTab = function(tabKey) {
-                    $scope.selectedTab  = _.find($scope.tabsCollection, function(tab) { return tab.key == tabKey; });
-                }
-
-                $scope.$watch(function() { return $location.hash()}, function(hash) {
-                    if (hash) {
-                        $scope.selectTab($location.hash());
-                    } else {
-                        $scope.selectTab($scope.tabsCollection[0].key);
-                    }
-                });
-            }
-        }
-    }]);
-
-    directives.directive('editDashboardForm', ['$http', '$location', '$timeout', 'Dashboard', function($http, $location, $timeout, Dashboard) {
-        return {
-            restrict: 'E',
-            scope: {
-                dashboard: '='
-            },
-            templateUrl: '/views/edit_dashboard.html',
-            replace: true,
-            link: function($scope, element, attrs) {
-                var gridster = element.find(".gridster ul").gridster({
-                    widget_margins: [5, 5],
-                    widget_base_dimensions: [260, 100],
-                    min_cols: 2,
-                    max_cols: 2,
-                    serialize_params: function($w, wgd) {
-                        return {
-                            col: wgd.col,
-                            row: wgd.row,
-                            id: $w.data('widget-id')
-                        }
-                    }
-                }).data('gridster');
-
-                var gsItemTemplate = '<li data-widget-id="{id}" class="widget panel panel-default gs-w">' +
-                                 '<div class="panel-heading">{name}' +
-                                 '</div></li>';
-
-                $scope.$watch('dashboard.widgets', function(widgets) {
-                    $timeout(function () {
-                        gridster.remove_all_widgets();
-
-                        if (widgets && widgets.length) {
-                            var layout = [];
-
-                            _.each(widgets, function(row, rowIndex) {
-                                _.each(row, function(widget, colIndex) {
-                                    layout.push({
-                                        id: widget.id,
-                                        col: colIndex+1,
-                                        row: rowIndex+1,
-                                        ySize: 1,
-                                        xSize: widget.width,
-                                        name: widget.visualization.query.name
-                                    });
-                                });
-                            });
-
-                            _.each(layout, function(item) {
-                                var el = gsItemTemplate.replace('{id}', item.id).replace('{name}', item.name);
-                                gridster.add_widget(el, item.xSize, item.ySize, item.col, item.row);
-
-                            });
-                        }
-                    });
-                }, true);
-
-                $scope.saveDashboard = function() {
-                    $scope.saveInProgress = true;
-                    // TODO: we should use the dashboard service here.
-                    if ($scope.dashboard.id) {
-                        var positions = $(element).find('.gridster ul').data('gridster').serialize();
-                        var layout = [];
-                        _.each(_.sortBy(positions, function (pos) {
-                            return pos.row * 10 + pos.col;
-                        }), function (pos) {
-                            var row = pos.row - 1;
-                            var col = pos.col - 1;
-                            layout[row] = layout[row] || [];
-                            if (col > 0 && layout[row][col - 1] == undefined) {
-                                layout[row][col - 1] = pos.id;
-                            } else {
-                                layout[row][col] = pos.id;
-                            }
-
-                        });
-                        $scope.dashboard.layout = layout;
-
-                        layout = JSON.stringify(layout);
-                        $http.post('/api/dashboards/' + $scope.dashboard.id, {'name': $scope.dashboard.name, 'layout': layout}).success(function(response) {
-                            $scope.dashboard = new Dashboard(response);
-                            $scope.saveInProgress = false;
-                            $(element).modal('hide');
-                        })
-                    } else {
-                        $http.post('/api/dashboards', {'name': $scope.dashboard.name}).success(function(response) {
-                            $(element).modal('hide');
-                            $location.path('/dashboard/' + response.slug).replace();
-                        })
-                    }
-                }
-
-            }
-        }
-    }]);
-
-    directives.directive('newWidgetForm', ['$http', 'Query', function($http, Query) {
-        return {
-            restrict: 'E',
-            scope: {
-                dashboard: '='
-            },
-            templateUrl: '/views/new_widget_form.html',
-            replace: true,
-            link: function($scope, element, attrs) {
-                $scope.widgetSizes = [{name: 'Regular', value: 1}, {name: 'Double', value: 2}];
-
-                var reset = function() {
-                    $scope.saveInProgress = false;
-                    $scope.widgetSize = 1;
-                    $scope.queryId = null;
-                    $scope.selectedVis = null;
-                    $scope.query = null;
-
-                }
-
-                reset();
-
-                $scope.loadVisualizations = function() {
-                    if (!$scope.queryId) {
-                        return;
-                    }
-
-                    Query.get({
-                        id: $scope.queryId
-                    }, function(query) {
-                        if (query) {
-                            $scope.query = query;
-                            if(query.visualizations.length) {
-                                $scope.selectedVis = query.visualizations[0];
-                            }
-                        }
-                    });
-                };
-
-                $scope.saveWidget = function() {
-                    $scope.saveInProgress = true;
-
-                    var widget = {
-                        'visualization_id': $scope.selectedVis.id,
-                        'dashboard_id': $scope.dashboard.id,
-                        'options': {},
-                        'width': $scope.widgetSize
-                    }
-
-                    $http.post('/api/widgets', widget).success(function(response) {
-                        // update dashboard layout
-                        $scope.dashboard.layout = response['layout'];
-                        if (response['new_row']) {
-                            $scope.dashboard.widgets.push([response['widget']]);
-                        } else {
-                            $scope.dashboard.widgets[$scope.dashboard.widgets.length-1].push(response['widget']);
-                        }
-
-                        // close the dialog
-                        $('#add_query_dialog').modal('hide');
-                        reset();
-                    })
-                }
-
-            }
-        }
-    }])
-
-    // From: http://jsfiddle.net/joshdmiller/NDFHg/
-    directives.directive('editInPlace', function () {
-        return {
-            restrict: 'E',
-            scope: {
-                value: '=',
-                ignoreBlanks: '=',
-                editable: '=',
-                done: '='
-            },
-            template: function(tElement, tAttrs) {
-                var elType = tAttrs.editor || 'input';
-                var placeholder = tAttrs.placeholder || 'Click to edit';
-                return '<span ng-click="editable && edit()" ng-bind="value" ng-class="{editable: editable}"></span>' +
-                       '<span ng-click="editable && edit()" ng-show="editable && !value" ng-class="{editable: editable}">' + placeholder + '</span>' +
-                       '<{elType} ng-model="value" class="rd-form-control"></{elType}>'.replace('{elType}', elType);
-            },
-            link: function ($scope, element, attrs) {
-                // Let's get a reference to the input element, as we'll want to reference it.
-                var inputElement = angular.element(element.children()[2]);
-
-                // This directive should have a set class so we can style it.
-                element.addClass('edit-in-place');
-
-                // Initially, we're not editing.
-                $scope.editing = false;
-
-                // ng-click handler to activate edit-in-place
-                $scope.edit = function () {
-                    $scope.oldValue = $scope.value;
-
-                    $scope.editing = true;
-
-                    // We control display through a class on the directive itself. See the CSS.
-                    element.addClass('active');
-
-                    // And we must focus the element.
-                    // `angular.element()` provides a chainable array, like jQuery so to access a native DOM function,
-                    // we have to reference the first element in the array.
-                    inputElement[0].focus();
-                };
-
-                function save() {
-                    if ($scope.editing) {
-                        if ($scope.ignoreBlanks && _.isEmpty($scope.value)) {
-                            $scope.value = $scope.oldValue;
-                        }
-                        $scope.editing = false;
-                        element.removeClass('active');
-
-                        if ($scope.value !== $scope.oldValue) {
-                            $scope.done && $scope.done();
-                        }
-                    }
-                }
-
-                $(inputElement).keydown(function(e) {
-                    // 'return' or 'enter' key pressed
-                    // allow 'shift' to break lines
-                    if (e.which === 13 && !e.shiftKey) {
-                        save();
-                    } else if (e.which === 27) {
-                        $scope.value = $scope.oldValue;
-                        $scope.$apply(function() {
-                            $(inputElement[0]).blur();
-                        });
-                    }
-                }).blur(function() {
-                    save();
-                });
-            }
-        };
-    });
-
-    // http://stackoverflow.com/a/17904092/1559840
-    directives.directive('jsonText', function() {
-        return {
-            restrict: 'A',
-            require: 'ngModel',
-            link: function(scope, element, attr, ngModel) {
-              function into(input) {
-                return JSON.parse(input);
-              }
-              function out(data) {
-                return JSON.stringify(data, undefined, 2);
-              }
-              ngModel.$parsers.push(into);
-              ngModel.$formatters.push(out);
-
-            }
-        };
-    });
-
-    directives.directive('rdTimer', ['$timeout', function ($timeout) {
-        return {
-            restrict: 'E',
-            scope: { timestamp: '=' },
-            template: '{{currentTime}}',
-            controller: ['$scope' ,function ($scope) {
-                $scope.currentTime = "00:00:00";
-                var currentTimeout = null;
-
-                var updateTime = function() {
-                    $scope.currentTime = moment(moment() - moment($scope.timestamp)).utc().format("HH:mm:ss")
-                    currentTimeout = $timeout(updateTime, 1000);
-                }
-
-                var cancelTimer = function() {
-                    if (currentTimeout) {
-                        $timeout.cancel(currentTimeout);
-                        currentTimeout = null;
-                    }
-                }
-
-                updateTime();
-
-                $scope.$on('$destroy', function () {
-                    cancelTimer();
-                });
-            }]
-        };
-    }]);
-
-    directives.directive('rdTimeAgo', function() {
-        return {
-            restrict: 'E',
-            scope: {
-                value: '='
-            },
-            template: '<span>' +
-                        '<span ng-show="value" am-time-ago="value"></span>' +
-                        '<span ng-hide="value">-</span>' +
-                      '</span>'
-        }
-    });
-})();

rd_ui/app/scripts/directives/dashboard_directives.js

@@ -0,0 +1,187 @@
+(function() {
+  'use strict'
+
+  var directives = angular.module('redash.directives');
+
+  directives.directive('editDashboardForm', ['$http', '$location', '$timeout', 'Dashboard',
+    function($http, $location, $timeout, Dashboard) {
+      return {
+        restrict: 'E',
+        scope: {
+          dashboard: '='
+        },
+        templateUrl: '/views/edit_dashboard.html',
+        replace: true,
+        link: function($scope, element, attrs) {
+          var gridster = element.find(".gridster ul").gridster({
+            widget_margins: [5, 5],
+            widget_base_dimensions: [260, 100],
+            min_cols: 2,
+            max_cols: 2,
+            serialize_params: function($w, wgd) {
+              return {
+                col: wgd.col,
+                row: wgd.row,
+                id: $w.data('widget-id')
+              }
+            }
+          }).data('gridster');
+
+          var gsItemTemplate = '<li data-widget-id="{id}" class="widget panel panel-default gs-w">' +
+            '<div class="panel-heading">{name}' +
+            '</div></li>';
+
+          $scope.$watch('dashboard.widgets && dashboard.widgets.length', function(widgets_length) {
+            $timeout(function() {
+              gridster.remove_all_widgets();
+
+              if ($scope.dashboard.widgets && $scope.dashboard.widgets.length) {
+                var layout = [];
+
+                _.each($scope.dashboard.widgets, function(row, rowIndex) {
+                  _.each(row, function(widget, colIndex) {
+                    layout.push({
+                      id: widget.id,
+                      col: colIndex + 1,
+                      row: rowIndex + 1,
+                      ySize: 1,
+                      xSize: widget.width,
+                      name: widget.visualization.query.name
+                    });
+                  });
+                });
+
+                _.each(layout, function(item) {
+                  var el = gsItemTemplate.replace('{id}', item.id).replace('{name}', item.name);
+                  gridster.add_widget(el, item.xSize, item.ySize, item.col, item.row);
+
+                });
+              }
+            });
+          });
+
+          $scope.saveDashboard = function() {
+            $scope.saveInProgress = true;
+            // TODO: we should use the dashboard service here.
+            if ($scope.dashboard.id) {
+              var positions = $(element).find('.gridster ul').data('gridster').serialize();
+              var layout = [];
+              _.each(_.sortBy(positions, function(pos) {
+                return pos.row * 10 + pos.col;
+              }), function(pos) {
+                var row = pos.row - 1;
+                var col = pos.col - 1;
+                layout[row] = layout[row] || [];
+                if (col > 0 && layout[row][col - 1] == undefined) {
+                  layout[row][col - 1] = pos.id;
+                } else {
+                  layout[row][col] = pos.id;
+                }
+
+              });
+              $scope.dashboard.layout = layout;
+
+              layout = JSON.stringify(layout);
+              $http.post('/api/dashboards/' + $scope.dashboard.id, {
+                'name': $scope.dashboard.name,
+                'layout': layout
+              }).success(function(response) {
+                $scope.dashboard = new Dashboard(response);
+                $scope.saveInProgress = false;
+                $(element).modal('hide');
+              })
+            } else {
+              $http.post('/api/dashboards', {
+                'name': $scope.dashboard.name
+              }).success(function(response) {
+                $(element).modal('hide');
+                $location.path('/dashboard/' + response.slug).replace();
+              })
+            }
+          }
+
+        }
+      }
+    }
+  ]);
+
+  directives.directive('newWidgetForm', ['Query', 'Widget', 'growl',
+    function(Query, Widget, growl) {
+      return {
+        restrict: 'E',
+        scope: {
+          dashboard: '='
+        },
+        templateUrl: '/views/new_widget_form.html',
+        replace: true,
+        link: function($scope, element, attrs) {
+          $scope.widgetSizes = [{
+            name: 'Regular',
+            value: 1
+          }, {
+            name: 'Double',
+            value: 2
+          }];
+
+          var reset = function() {
+            $scope.saveInProgress = false;
+            $scope.widgetSize = 1;
+            $scope.queryId = null;
+            $scope.selectedVis = null;
+            $scope.query = null;
+
+          }
+
+          reset();
+
+          $scope.loadVisualizations = function() {
+            if (!$scope.queryId) {
+              return;
+            }
+
+            Query.get({
+              id: $scope.queryId
+            }, function(query) {
+              if (query) {
+                $scope.query = query;
+                if (query.visualizations.length) {
+                  $scope.selectedVis = query.visualizations[0];
+                }
+              }
+            });
+          };
+
+          $scope.saveWidget = function() {
+            $scope.saveInProgress = true;
+
+            var widget = new Widget({
+              'visualization_id': $scope.selectedVis.id,
+              'dashboard_id': $scope.dashboard.id,
+              'options': {},
+              'width': $scope.widgetSize
+            });
+
+            widget.$save().then(function(response) {
+              // update dashboard layout
+              $scope.dashboard.layout = response['layout'];
+              if (response['new_row']) {
+                $scope.dashboard.widgets.push([response['widget']]);
+              } else {
+                $scope.dashboard.widgets[$scope.dashboard.widgets.length - 1].push(response['widget']);
+              }
+
+              // close the dialog
+              $('#add_query_dialog').modal('hide');
+              reset();
+            }).catch(function() {
+              growl.addErrorMessage("Widget can not be added");
+            }).finally(function() {
+              $scope.saveInProgress = false;
+            });
+          }
+
+        }
+      }
+    }
+  ])
+})();

rd_ui/app/scripts/directives/directives.js

@@ -0,0 +1,219 @@
+(function() {
+    'use strict';
+
+    var directives = angular.module('redash.directives', []);
+
+    directives.directive('alertUnsavedChanges', ['$window', function($window) {
+        return {
+            restrict: 'E',
+            replace: true,
+            scope: {
+                'isDirty': '='
+            },
+            link: function($scope) {
+                var
+
+                unloadMessage = "You will lose your changes if you leave",
+                confirmMessage = unloadMessage + "\n\nAre you sure you want to leave this page?",
+
+                // store original handler (if any)
+                _onbeforeunload = $window.onbeforeunload;
+
+                $window.onbeforeunload = function() {
+                    return $scope.isDirty ? unloadMessage : null;
+                }
+
+                $scope.$on('$locationChangeStart', function(event, next, current) {
+                  if (next.split("#")[0] == current.split("#")[0]) {
+                    return;
+                  }
+
+                  if ($scope.isDirty && !confirm(confirmMessage)) {
+                    event.preventDefault();
+                  }
+                });
+
+                $scope.$on('$destroy', function() {
+                    $window.onbeforeunload = _onbeforeunload;
+                });
+            }
+        }
+    }]);
+
+    directives.directive('rdTab', function() {
+        return {
+            restrict: 'E',
+            scope: {
+                'tabId': '@',
+                'name': '@'
+            },
+            transclude: true,
+            template: '<li class="rd-tab" ng-class="{active: tabId==selectedTab}"><a href="#{{tabId}}">{{name}}<span ng-transclude></span></a></li>',
+            replace: true,
+            link: function(scope) {
+                scope.$watch(function(){return scope.$parent.selectedTab}, function(tab) {
+                    scope.selectedTab = tab;
+                });
+            }
+        }
+    });
+
+    directives.directive('rdTabs', ['$location', function($location) {
+        return {
+            restrict: 'E',
+            scope: {
+                tabsCollection: '=',
+                selectedTab: '='
+            },
+            template: '<ul class="nav nav-tabs"><li ng-class="{active: tab==selectedTab}" ng-repeat="tab in tabsCollection"><a href="#{{tab.key}}">{{tab.name}}</a></li></ul>',
+            replace: true,
+            link: function($scope, element, attrs) {
+                $scope.selectTab = function(tabKey) {
+                    $scope.selectedTab  = _.find($scope.tabsCollection, function(tab) { return tab.key == tabKey; });
+                }
+
+                $scope.$watch(function() { return $location.hash()}, function(hash) {
+                    if (hash) {
+                        $scope.selectTab($location.hash());
+                    } else {
+                        $scope.selectTab($scope.tabsCollection[0].key);
+                    }
+                });
+            }
+        }
+    }]);
+
+    // From: http://jsfiddle.net/joshdmiller/NDFHg/
+    directives.directive('editInPlace', function () {
+        return {
+            restrict: 'E',
+            scope: {
+                value: '=',
+                ignoreBlanks: '=',
+                editable: '=',
+                done: '='
+            },
+            template: function(tElement, tAttrs) {
+                var elType = tAttrs.editor || 'input';
+                var placeholder = tAttrs.placeholder || 'Click to edit';
+                return '<span ng-click="editable && edit()" ng-bind="value" ng-class="{editable: editable}"></span>' +
+                       '<span ng-click="editable && edit()" ng-show="editable && !value" ng-class="{editable: editable}">' + placeholder + '</span>' +
+                       '<{elType} ng-model="value" class="rd-form-control"></{elType}>'.replace('{elType}', elType);
+            },
+            link: function ($scope, element, attrs) {
+                // Let's get a reference to the input element, as we'll want to reference it.
+                var inputElement = angular.element(element.children()[2]);
+
+                // This directive should have a set class so we can style it.
+                element.addClass('edit-in-place');
+
+                // Initially, we're not editing.
+                $scope.editing = false;
+
+                // ng-click handler to activate edit-in-place
+                $scope.edit = function () {
+                    $scope.oldValue = $scope.value;
+
+                    $scope.editing = true;
+
+                    // We control display through a class on the directive itself. See the CSS.
+                    element.addClass('active');
+
+                    // And we must focus the element.
+                    // `angular.element()` provides a chainable array, like jQuery so to access a native DOM function,
+                    // we have to reference the first element in the array.
+                    inputElement[0].focus();
+                };
+
+                function save() {
+                    if ($scope.editing) {
+                        if ($scope.ignoreBlanks && _.isEmpty($scope.value)) {
+                            $scope.value = $scope.oldValue;
+                        }
+                        $scope.editing = false;
+                        element.removeClass('active');
+
+                        if ($scope.value !== $scope.oldValue) {
+                            $scope.done && $scope.done();
+                        }
+                    }
+                }
+
+                $(inputElement).keydown(function(e) {
+                    // 'return' or 'enter' key pressed
+                    // allow 'shift' to break lines
+                    if (e.which === 13 && !e.shiftKey) {
+                        save();
+                    } else if (e.which === 27) {
+                        $scope.value = $scope.oldValue;
+                        $scope.$apply(function() {
+                            $(inputElement[0]).blur();
+                        });
+                    }
+                }).blur(function() {
+                    save();
+                });
+            }
+        };
+    });
+
+    // http://stackoverflow.com/a/17904092/1559840
+    directives.directive('jsonText', function() {
+        return {
+            restrict: 'A',
+            require: 'ngModel',
+            link: function(scope, element, attr, ngModel) {
+              function into(input) {
+                return JSON.parse(input);
+              }
+              function out(data) {
+                return JSON.stringify(data, undefined, 2);
+              }
+              ngModel.$parsers.push(into);
+              ngModel.$formatters.push(out);
+
+              scope.$watch(attr.ngModel, function(newValue) {
+                element[0].value = out(newValue);
+              }, true);
+            }
+        };
+    });
+
+    directives.directive('rdTimer', [function () {
+        return {
+            restrict: 'E',
+            scope: { timestamp: '=' },
+            template: '{{currentTime}}',
+            controller: ['$scope' ,function ($scope) {
+                $scope.currentTime = "00:00:00";
+
+                // We're using setInterval directly instead of $timeout, to avoid using $apply, to
+                // prevent the digest loop being run every second.
+                var currentTimer = setInterval(function() {
+                    $scope.currentTime = moment(moment() - moment($scope.timestamp)).utc().format("HH:mm:ss");
+                    $scope.$digest();
+                }, 1000);
+
+                $scope.$on('$destroy', function () {
+                    if (currentTimer) {
+                        clearInterval(currentTimer);
+                        currentTimer = null;
+                    }
+                });
+            }]
+        };
+    }]);
+
+    directives.directive('rdTimeAgo', function() {
+        return {
+            restrict: 'E',
+            scope: {
+                value: '='
+            },
+            template: '<span>' +
+                        '<span ng-show="value" am-time-ago="value"></span>' +
+                        '<span ng-hide="value">-</span>' +
+                      '</span>'
+        }
+    });
+})();

rd_ui/app/scripts/directives/query_directives.js

@@ -0,0 +1,141 @@
+(function() {
+  'use strict'
+
+  function queryLink() {
+    return {
+      restrict: 'E',
+      scope: {
+        'query': '=',
+        'visualization': '=?'
+      },
+      template: '<a ng-href="{{link}}" class="query-link">{{query.name}}</a>',
+      link: function(scope, element) {
+        scope.link = '/queries/' + scope.query.id;
+        if (scope.visualization) {
+          if (scope.visualization.type === 'TABLE') {
+            // link to hard-coded table tab instead of the (hidden) visualization tab
+            scope.link += '#table';
+          } else {
+            scope.link += '#' + scope.visualization.id;
+          }
+        }
+        // element.find('a').attr('href', link);
+      }
+    }
+  }
+
+  function querySourceLink() {
+    return {
+      restrict: 'E',
+      template: '<span ng-show="query.id && canViewSource">\
+                    <a ng-show="!sourceMode"\
+                      ng-href="{{query.id}}/source#{{selectedTab}}">Show Source\
+                    </a>\
+                    <a ng-show="sourceMode"\
+                      ng-href="/queries/{{query.id}}#{{selectedTab}}">Hide Source\
+                    </a>\
+                </span>'
+    }
+  }
+
+  function queryEditor() {
+    return {
+      restrict: 'E',
+      scope: {
+        'query': '=',
+        'lock': '='
+      },
+      template: '<textarea\
+                  ui-codemirror="editorOptions"\
+                  ng-model="query.query">',
+      link: function($scope) {
+        $scope.editorOptions = {
+            mode: 'text/x-sql',
+            lineWrapping: true,
+            lineNumbers: true,
+            readOnly: false,
+            matchBrackets: true,
+            autoCloseBrackets: true
+        };
+
+        $scope.$watch('lock', function(locked) {
+          $scope.editorOptions.readOnly = locked ? 'nocursor' : false;
+        });
+      }
+    }
+  }
+
+  function queryFormatter($http) {
+    return {
+      restrict: 'E',
+      // don't create new scope to avoid ui-codemirror bug
+      // seehttps://github.com/angular-ui/ui-codemirror/pull/37
+      scope: false,
+      template: '<button type="button" class="btn btn-default btn-xs"\
+                   ng-click="formatQuery()">\
+                    <span class="glyphicon glyphicon-indent-left"></span>\
+                     Format SQL\
+                </button>',
+      link: function($scope) {
+        $scope.formatQuery = function formatQuery() {
+            $scope.queryExecuting = true;
+            $http.post('/api/queries/format', {
+                'query': $scope.query.query
+            }).success(function (response) {
+                $scope.query.query = response;
+            }).finally(function () {
+              $scope.queryExecuting = false;
+            });
+        };
+      }
+    }
+  }
+
+  function queryRefreshSelect() {
+    return {
+      restrict: 'E',
+      template: '<select\
+                  ng-disabled="!isQueryOwner"\
+                  ng-model="query.ttl"\
+                  ng-change="saveQuery()"\
+                  ng-options="c.value as c.name for c in refreshOptions">\
+                  </select>',
+      link: function($scope) {
+        $scope.refreshOptions = [
+            {
+                value: -1,
+                name: 'No Refresh'
+            },
+            {
+                value: 60,
+                name: 'Every minute'
+            },
+        ]
+
+        _.each(_.range(1, 13), function (i) {
+            $scope.refreshOptions.push({
+                value: i * 3600,
+                name: 'Every ' + i + 'h'
+            });
+        })
+
+        $scope.refreshOptions.push({
+            value: 24 * 3600,
+            name: 'Every 24h'
+        });
+        $scope.refreshOptions.push({
+            value: 7 * 24 * 3600,
+            name: 'Once a week'
+        });
+      }
+
+    }
+  }
+
+  angular.module('redash.directives')
+  .directive('queryLink', queryLink)
+  .directive('querySourceLink', querySourceLink)
+  .directive('queryEditor', queryEditor)
+  .directive('queryRefreshSelect', queryRefreshSelect)
+  .directive('queryFormatter', ['$http', queryFormatter]);
+})();

rd_ui/app/scripts/ng_highchart.js

@@ -1,6 +1,11 @@
 (function () {
   'use strict';
 
+  Highcharts.setOptions({
+    colors: ["#4572A7", "#AA4643", "#89A54E", "#80699B", "#3D96AE",
+      "#DB843D", "#92A8CD", "#A47D7C", "#B5CA92"]
+  });
+
   var defaultOptions = {
     title: {
       "text": null
@@ -18,7 +23,8 @@
       formatter: function () {
         if (!this.points) {
           this.points = [this.point];
-                };
+        }
+        ;
 
         if (moment.isMoment(this.x)) {
           var s = '<b>' + moment(this.x).format("DD/MM/YY HH:mm") + '</b>',
@@ -44,7 +50,7 @@
             } else {
               s += ": " + Highcharts.numberFormat(point.y);
               if (point.percentage < 100) {
-                                s += ' (' +Highcharts.numberFormat(point.percentage) +  '%)';
+                s += ' (' + Highcharts.numberFormat(point.percentage) + '%)';
               }
             }
           });
@@ -174,30 +180,28 @@
             // invoked after the DOM is ready, we see first an empty HighCharts objects and later
             // they get filled up. Which gives the feeling that the charts loading faster (otherwise
             // we stare at an empty screen until the HighCharts object is ready).
-                    $timeout(function(){
+            $timeout(function () {
               // Update when options change
               scope.$watch('options', function (newOptions) {
                 initChart(newOptions);
               }, true);
 
               //Update when charts data changes
-                        scope.$watch(function () {
-                            // TODO: this might be an issue in case the series change, but they stay
-                            // with the same length
-                            return (scope.series && scope.series.length) || 0;
-                        }, function (length) {
-                            if (!length || length == 0) {
+              scope.$watchCollection('series', function (series) {
+                if (!series || series.length == 0) {
                   scope.chart.showLoading();
                 } else {
                   drawChart();
-                            };
-                        }, true);
+                }
+                ;
+              });
             });
 
             function initChart(options) {
               if (scope.chart) {
                 scope.chart.destroy();
-                        };
+              }
+              ;
 
               $.extend(true, chartOptions, options);
 
@@ -210,11 +214,19 @@
                 scope.chart.series[0].remove(false);
               };
 
+              if (!('xAxis' in chartOptions && 'type' in chartOptions['xAxis'])) {
                 if (scope.series.length > 0 && _.some(scope.series[0].data, function (p) {
                   return (angular.isString(p.x) || angular.isDefined(p.name));
                 })) {
-                            scope.chart.xAxis[0].update({type: 'category'});
+                  chartOptions['xAxis'] = chartOptions['xAxis'] || {};
+                  chartOptions['xAxis']['type'] = 'category';
+                } else {
+                  chartOptions['xAxis'] = chartOptions['xAxis'] || {};
+                  chartOptions['xAxis']['type'] = 'datetime';
+                }
+              }
 
+              if (chartOptions['xAxis']['type'] == 'category' || chartOptions['series']['type']=='pie') {
                 if (!angular.isDefined(scope.series[0].data[0].name)) {
                   // We need to make sure that for each category, each series has a value.
                   var categories = _.union.apply(this, _.map(scope.series, function (s) {
@@ -228,19 +240,18 @@
                     var newData = _.map(categories, function (category) {
                       return {
                         name: category,
-                                            y: yValues[category] && yValues[category][0].y
+                        y: (yValues[category] && yValues[category][0].y) || 0
                       }
                     });
 
                     if (categories.length == 1) {
                       newData = _.sortBy(newData, 'y').reverse();
-                                    };
+                    }
+                    ;
 
                     s.data = newData;
                   });
                 }
-                        } else {
-                            scope.chart.xAxis[0].update({type: 'datetime'});
               }
 
               scope.chart.counters.color = 0;
@@ -262,7 +273,8 @@
                       };
                     }
                   });
-                            };
+                }
+                ;
 
                 scope.chart.addSeries(s, false);
               });

rd_ui/app/scripts/services/resources.js

@@ -0,0 +1,367 @@
+(function () {
+  var QueryResult = function ($resource, $timeout) {
+    var QueryResultResource = $resource('/api/query_results/:id', {id: '@id'}, {'post': {'method': 'POST'}});
+    var Job = $resource('/api/jobs/:id', {id: '@id'});
+
+    var updateFunction = function (props) {
+      angular.extend(this, props);
+      if ('query_result' in props) {
+        this.status = "done";
+        this.filters = undefined;
+        this.filterFreeze = undefined;
+
+        _.each(this.query_result.data.rows, function (row) {
+          _.each(row, function (v, k) {
+            if (_.isString(v) && v.match(/^\d{4}-\d{2}-\d{2}/)) {
+              row[k] = moment(v);
+            }
+          });
+        });
+      } else if (this.job.status == 3) {
+        this.status = "processing";
+      } else {
+        this.status = undefined;
+      }
+    }
+
+    function QueryResult(props) {
+      this.job = {};
+      this.query_result = {};
+      this.status = "waiting";
+      this.filters = undefined;
+      this.filterFreeze = undefined;
+
+      this.updatedAt = moment();
+
+      if (props) {
+        updateFunction.apply(this, [props]);
+      }
+    }
+
+    var statuses = {
+      1: "waiting",
+      2: "processing",
+      3: "done",
+      4: "failed"
+    }
+
+    QueryResult.prototype.update = updateFunction;
+
+    QueryResult.prototype.getId = function () {
+      var id = null;
+      if ('query_result' in this) {
+        id = this.query_result.id;
+      }
+      return id;
+    }
+
+    QueryResult.prototype.cancelExecution = function () {
+      Job.delete({id: this.job.id});
+    }
+
+    QueryResult.prototype.getStatus = function () {
+      return this.status || statuses[this.job.status];
+    }
+
+    QueryResult.prototype.getError = function () {
+      // TODO: move this logic to the server...
+      if (this.job.error == "None") {
+        return undefined;
+      }
+
+      return this.job.error;
+    }
+
+    QueryResult.prototype.getUpdatedAt = function () {
+      return this.query_result.retrieved_at || this.job.updated_at * 1000.0 || this.updatedAt;
+    }
+
+    QueryResult.prototype.getRuntime = function () {
+      return this.query_result.runtime;
+    }
+
+    QueryResult.prototype.getRawData = function () {
+      if (!this.query_result.data) {
+        return null;
+      }
+
+      var data = this.query_result.data.rows;
+
+      return data;
+    }
+
+    QueryResult.prototype.getData = function () {
+      if (!this.query_result.data) {
+        return null;
+      }
+
+      var filterValues = function (filters) {
+        if (!filters) {
+          return null;
+        }
+
+        return _.reduce(filters, function (str, filter) {
+          return str + filter.current;
+        }, "")
+      }
+
+      var filters = this.getFilters();
+      var filterFreeze = filterValues(filters);
+
+      if (this.filterFreeze != filterFreeze) {
+        this.filterFreeze = filterFreeze;
+
+        if (filters) {
+          this.filteredData = _.filter(this.query_result.data.rows, function (row) {
+            return _.reduce(filters, function (memo, filter) {
+              if (!_.isArray(filter.current)) {
+                filter.current = [filter.current];
+              };
+              return (memo && _.some(filter.current, function(v) { return v == row[filter.name] }));
+            }, true);
+          });
+        } else {
+          this.filteredData = this.query_result.data.rows;
+        }
+      }
+
+      return this.filteredData;
+    }
+
+    QueryResult.prototype.getChartData = function () {
+      var series = {};
+
+      _.each(this.getData(), function (row) {
+        var point = {};
+        var seriesName = undefined;
+        var xValue = 0;
+        var yValues = {};
+
+        _.each(row, function (value, definition) {
+          var type = definition.split("::")[1];
+          var name = definition.split("::")[0];
+
+          if (type == 'x') {
+            xValue = value;
+            point[type] = value;
+          }
+
+          if (type == 'y') {
+            yValues[name] = value;
+            point[type] = value;
+          }
+
+          if (type == 'series') {
+            seriesName = String(value);
+          }
+
+          if (type == 'multi-filter') {
+            seriesName = String(value);
+          }
+        });
+
+        var addPointToSeries = function (seriesName, point) {
+          if (series[seriesName] == undefined) {
+            series[seriesName] = {
+              name: seriesName,
+              type: 'column',
+              data: []
+            }
+          }
+
+          series[seriesName]['data'].push(point);
+        }
+
+        if (seriesName === undefined) {
+          _.each(yValues, function (yValue, seriesName) {
+            addPointToSeries(seriesName, {'x': xValue, 'y': yValue});
+          });
+        } else {
+          addPointToSeries(seriesName, point);
+        }
+      });
+
+      _.each(series, function (series) {
+        series.data = _.sortBy(series.data, 'x');
+      });
+
+      return _.values(series);
+    };
+
+    QueryResult.prototype.getColumns = function () {
+      if (this.columns == undefined && this.query_result.data) {
+        this.columns = _.map(this.query_result.data.columns, function (v) {
+          return v.name;
+        });
+      }
+
+      return this.columns;
+    }
+
+    QueryResult.prototype.getColumnCleanName = function (column) {
+      var parts = column.split('::');
+      var name = parts[1];
+      if (parts[0] != '') {
+        // TODO: it's probably time to generalize this.
+        // see also getColumnFriendlyName
+        name = parts[0].replace(/%/g, '__pct').replace(/ /g, '_').replace(/\?/g, '');
+      }
+      return name;
+    }
+
+    QueryResult.prototype.getColumnFriendlyName = function (column) {
+      return this.getColumnCleanName(column).replace('__pct', '%').replace(/_/g, ' ').replace(/(?:^|\s)\S/g, function (a) {
+        return a.toUpperCase();
+      });
+    }
+
+    QueryResult.prototype.getColumnCleanNames = function () {
+      return _.map(this.getColumns(), function (col) {
+        return this.getColumnCleanName(col);
+      }, this);
+    }
+
+    QueryResult.prototype.getColumnFriendlyNames = function () {
+      return _.map(this.getColumns(), function (col) {
+        return this.getColumnFriendlyName(col);
+      }, this);
+    }
+
+    QueryResult.prototype.getFilters = function () {
+      if (!this.filters) {
+        this.prepareFilters();
+      }
+
+      return this.filters;
+    };
+
+    QueryResult.prototype.prepareFilters = function () {
+      var filters = [];
+      var filterTypes = ['filter', 'multi-filter'];
+      _.each(this.getColumns(), function (col) {
+        var type = col.split('::')[1]
+        if (_.contains(filterTypes, type)) {
+          // filter found
+          var filter = {
+            name: col,
+            friendlyName: this.getColumnFriendlyName(col),
+            values: [],
+            multiple: (type=='multi-filter')
+          }
+          filters.push(filter);
+        }
+      }, this);
+
+      _.each(this.getRawData(), function (row) {
+        _.each(filters, function (filter) {
+          filter.values.push(row[filter.name]);
+          if (filter.values.length == 1) {
+            filter.current = row[filter.name];
+          }
+        })
+      });
+
+      _.each(filters, function(filter) {
+        filter.values = _.uniq(filter.values);
+      });
+
+      this.filters = filters;
+    }
+
+    var refreshStatus = function (queryResult, query, ttl) {
+      Job.get({'id': queryResult.job.id}, function (response) {
+        queryResult.update(response);
+
+        if (queryResult.getStatus() == "processing" && queryResult.job.query_result_id && queryResult.job.query_result_id != "None") {
+          QueryResultResource.get({'id': queryResult.job.query_result_id}, function (response) {
+            queryResult.update(response);
+          });
+        } else if (queryResult.getStatus() != "failed") {
+          $timeout(function () {
+            refreshStatus(queryResult, query, ttl);
+          }, 3000);
+        }
+      })
+    }
+
+    QueryResult.getById = function (id) {
+      var queryResult = new QueryResult();
+
+      QueryResultResource.get({'id': id}, function (response) {
+        queryResult.update(response);
+      });
+
+      return queryResult;
+    }
+
+    QueryResult.get = function (data_source_id, query, ttl) {
+      var queryResult = new QueryResult();
+
+      QueryResultResource.post({'data_source_id': data_source_id, 'query': query, 'ttl': ttl}, function (response) {
+        queryResult.update(response);
+
+        if ('job' in response) {
+          refreshStatus(queryResult, query, ttl);
+        }
+      });
+
+      return queryResult;
+    }
+
+    return QueryResult;
+  };
+
+  var Query = function ($resource, QueryResult, DataSource) {
+    var Query = $resource('/api/queries/:id', {id: '@id'});
+
+    Query.newQuery = function () {
+      return new Query({
+        query: "",
+        name: "New Query",
+        ttl: -1,
+        user: currentUser
+      });
+    };
+
+    Query.prototype.getSourceLink = function () {
+      return '/queries/' + this.id + '/source';
+    };
+
+    Query.prototype.getQueryResult = function (ttl) {
+      if (ttl == undefined) {
+        ttl = this.ttl;
+      }
+
+      var queryResult = null;
+      if (this.latest_query_data && ttl != 0) {
+        queryResult = new QueryResult({'query_result': this.latest_query_data});
+      } else if (this.latest_query_data_id && ttl != 0) {
+        queryResult = QueryResult.getById(this.latest_query_data_id);
+      } else if (this.data_source_id) {
+        queryResult = QueryResult.get(this.data_source_id, this.query, ttl);
+      }
+
+      return queryResult;
+    };
+
+    return Query;
+  };
+
+  var DataSource = function ($resource) {
+    var DataSourceResource = $resource('/api/data_sources/:id', {id: '@id'}, {'get': {'method': 'GET', 'cache': true, 'isArray': true}});
+
+    return DataSourceResource;
+  }
+
+  var Widget = function ($resource) {
+    var WidgetResource = $resource('/api/widgets/:id', {id: '@id'});
+
+    return WidgetResource;
+  }
+
+  angular.module('redash.services')
+      .factory('QueryResult', ['$resource', '$timeout', QueryResult])
+      .factory('Query', ['$resource', 'QueryResult', 'DataSource', Query])
+      .factory('DataSource', ['$resource', DataSource])
+      .factory('Widget', ['$resource', Widget]);
+})();

rd_ui/app/scripts/services/services.js

@@ -1,289 +1,24 @@
-(function () {
-    var QueryResult = function($resource, $timeout) {
-        var QueryResultResource = $resource('/api/query_results/:id', {id: '@id'}, {'post': {'method': 'POST'}});
-        var Job = $resource('/api/jobs/:id', {id: '@id'});
+(function() {
+  'use strict'
 
-        var updateFunction = function (props) {
-            angular.extend(this, props);
-            if ('query_result' in props) {
-                this.status = "done";
-
-                _.each(this.query_result.data.rows, function (row) {
-                    _.each(row, function (v, k) {
-                        if (_.isString(v) && v.match(/^\d{4}-\d{2}-\d{2}/)) {
-                            row[k] = moment(v);
-                        }
+  function KeyboardShortcuts() {
+    this.bind = function bind(keymap) {
+      _.forEach(keymap, function(fn, key) {
+        Mousetrap.bindGlobal(key, function(e) {
+            e.preventDefault();
+            fn();
         });
       });
-            } else if (this.job.status == 3) {
-                this.status = "processing";
-            } else {
-                this.status = undefined;
-            }
+
     }
 
-        function QueryResult(props) {
-            this.job = {};
-            this.query_result = {};
-            this.status = "waiting";
-
-            this.updatedAt = moment();
-
-            if (props) {
-                updateFunction.apply(this, [props]);
-            }
-        }
-
-        var statuses = {
-            1: "waiting",
-            2: "processing",
-            3: "done",
-            4: "failed"
-        }
-
-        QueryResult.prototype.update = updateFunction;
-
-        QueryResult.prototype.getId = function() {
-            var id = null;
-            if ('query_result' in this) {
-                id = this.query_result.id;
-            }
-            return id;
-        }
-
-        QueryResult.prototype.cancelExecution = function() {
-            Job.delete({id: this.job.id});
-        }
-
-        QueryResult.prototype.getStatus = function() {
-            return this.status || statuses[this.job.status];
-        }
-
-        QueryResult.prototype.getError = function() {
-            // TODO: move this logic to the server...
-            if (this.job.error == "None") {
-                return undefined;
-            }
-
-            return this.job.error;
-        }
-
-        QueryResult.prototype.getUpdatedAt = function() {
-            return this.query_result.retrieved_at || this.job.updated_at*1000.0 || this.updatedAt;
-        }
-
-        QueryResult.prototype.getRuntime = function() {
-            return this.query_result.runtime;
-        }
-
-        QueryResult.prototype.getData = function() {
-            if (!this.query_result.data) {
-                return null;
-            }
-
-            var data = this.query_result.data.rows;
-            return data;
-        }
-
-        QueryResult.prototype.getChartData = function () {
-            var series = {};
-
-            _.each(this.getData(), function (row) {
-                var point = {};
-                var seriesName = undefined;
-                var xValue = 0;
-                var yValues = {};
-
-                _.each(row, function (value, definition) {
-                    var type = definition.split("::")[1];
-                    var name = definition.split("::")[0];
-
-                    if (type == 'x') {
-                        xValue = value;
-                        point[type] = value;
-                    }
-
-                    if (type == 'y') {
-                        yValues[name] = value;
-                        point[type] = value;
-                    }
-
-                    if (type == 'series') {
-                        seriesName = String(value);
-                    }
-                });
-
-                var addPointToSeries = function(seriesName, point) {
-                    if (series[seriesName] == undefined) {
-                        series[seriesName] = {
-                            name: seriesName,
-                            type: 'column',
-                            data: []
-                        }
-                    }
-
-                    series[seriesName]['data'].push(point);
-                }
-
-                if (seriesName === undefined) {
-                    _.each(yValues, function(yValue, seriesName) {
-                        addPointToSeries(seriesName, {'x': xValue, 'y': yValue});
-                    });
-                } else {
-                    addPointToSeries(seriesName, point);
-                }
-            });
-
-            _.each(series, function(series) {
-                series.data = _.sortBy(series.data, 'x');
-            });
-
-            return _.values(series);
-        };
-
-        QueryResult.prototype.getColumns = function () {
-            if (this.columns == undefined) {
-                this.columns = _.map(this.query_result.data.columns, function(v) {
-                    return v.name;
-                })
-            }
-
-            return this.columns;
-        }
-
-        QueryResult.prototype.getColumnCleanName = function (column) {
-            var parts = column.split('::');
-            var name = parts[1];
-            if (parts[0] != '') {
-                // TODO: it's probably time to generalize this.
-                // see also getColumnFriendlyName
-                name = parts[0].replace(/%/g, '__pct').replace(/ /g, '_').replace(/\?/g,'');
-            }
-            return name;
-        }
-
-        QueryResult.prototype.getColumnFriendlyName = function (column) {
-            return this.getColumnCleanName(column).replace('__pct', '%').replace(/_/g, ' ').replace(/(?:^|\s)\S/g, function (a) {
-                return a.toUpperCase();
+    this.unbind = function unbind(keymap) {
+      _.forEach(keymap, function(fn, key) {
+        Mousetrap.unbind(key);
       });
     }
-
-        QueryResult.prototype.getColumnCleanNames = function () {
-            return _.map(this.getColumns(), function (col) {
-                return this.getColumnCleanName(col);
-            }, this);
   }
 
-        QueryResult.prototype.getColumnFriendlyNames = function () {
-            return _.map(this.getColumns(), function (col) {
-                return this.getColumnFriendlyName(col);
-            }, this);
-        }
-
-        QueryResult.prototype.getFilters = function () {
-            var filterNames = [];
-            _.each(this.getColumns(), function (col) {
-                if (col.split('::')[1] == 'filter') {
-                    filterNames.push(col);
-                }
-            });
-
-            var filterValues = [];
-            _.each(this.getData(), function (row) {
-                _.each(filterNames, function (filter, i) {
-                    if (filterValues[i] == undefined) {
-                        filterValues[i] = [];
-                    }
-                    filterValues[i].push(row[filter]);
-                })
-            });
-
-            var filters = _.map(filterNames, function (filter, i) {
-                var f = {
-                    name: filter,
-                    friendlyName: this.getColumnFriendlyName(filter),
-                    values: _.uniq(filterValues[i])
-                };
-
-                f.current = f.values[0];
-                return f;
-            }, this);
-
-            return filters;
-        };
-
-        var refreshStatus = function(queryResult, query, ttl) {
-            Job.get({'id': queryResult.job.id}, function(response) {
-                queryResult.update(response);
-
-                if (queryResult.getStatus() == "processing" && queryResult.job.query_result_id && queryResult.job.query_result_id != "None") {
-                    QueryResultResource.get({'id': queryResult.job.query_result_id}, function(response) {
-                        queryResult.update(response);
-                    });
-                } else if (queryResult.getStatus() != "failed") {
-                    $timeout(function () {
-                        refreshStatus(queryResult, query, ttl);
-                    }, 3000);
-                }
-            })
-        }
-
-        QueryResult.getById = function (id) {
-            var queryResult = new QueryResult();
-
-            QueryResultResource.get({'id': id}, function (response) {
-                queryResult.update(response);
-            });
-
-            return queryResult;
-        }
-
-        QueryResult.get = function (query, ttl) {
-            var queryResult = new QueryResult();
-
-            QueryResultResource.post({'query': query, 'ttl': ttl}, function (response) {
-                queryResult.update(response);
-
-                if ('job' in response) {
-                    refreshStatus(queryResult, query, ttl);
-                }
-            });
-
-            return queryResult;
-        }
-
-        return QueryResult;
-    };
-
-    var Query = function ($resource, QueryResult) {
-        var Query = $resource('/api/queries/:id', {id: '@id'});
-
-        Query.prototype.getQueryResult = function(ttl) {
-            if (ttl == undefined) {
-                ttl = this.ttl;
-            }
-
-
-            var queryResult = null;
-            if (this.latest_query_data && ttl != 0) {
-                queryResult = new QueryResult({'query_result': this.latest_query_data});
-            } else if (this.latest_query_data_id && ttl != 0) {
-                queryResult = QueryResult.getById(this.latest_query_data_id);
-            } else {
-                queryResult = QueryResult.get(this.query, ttl);
-            }
-
-            return queryResult;
-        };
-
-        Query.prototype.getHash = function() {
-            return this.query;
-        };
-
-        return Query;
-    };
-
   angular.module('redash.services', [])
-        .factory('QueryResult', ['$resource', '$timeout', QueryResult])
-        .factory('Query', ['$resource', 'QueryResult', Query])
+    .service('KeyboardShortcuts', [KeyboardShortcuts])
 })();

rd_ui/app/scripts/visualizations/base.js

@@ -64,8 +64,15 @@
             // TODO: using switch here (and in the options editor) might introduce errors and bad
             // performance wise. It's better to eventually show the correct template based on the
             // visualization type and not make the browser render all of them.
-            template: Visualization.renderVisualizationsTemplate,
-            replace: false
+            template: '<filters></filters>\n' + Visualization.renderVisualizationsTemplate,
+            replace: false,
+            link: function(scope) {
+                scope.$watch('queryResult && queryResult.getFilters()', function(filters) {
+                    if (filters) {
+                        scope.filters = filters;
+                    }
+                });
+            }
         }
     };
 
@@ -77,6 +84,13 @@
         }
     };
 
+    var Filters = function() {
+        return {
+            restrict: 'E',
+            templateUrl: '/views/visualizations/filters.html'
+        }
+    }
+
     var EditVisualizationForm = function(Visualization, growl) {
         return {
             restrict: 'E',
@@ -85,9 +99,11 @@
             scope: {
                 query: '=',
                 queryResult: '=',
-                visualization: '=?'
+                visualization: '=?',
+                onNewSuccess: '=?'
             },
             link: function (scope, element, attrs) {
+                scope.editRawOptions = currentUser.hasPermission('edit_raw_chart');
                 scope.visTypes = Visualization.visualizationTypes;
 
                 scope.newVisualization = function(q) {
@@ -131,7 +147,9 @@
                         if (index > -1) {
                             scope.query.visualizations[index] = result;
                         } else {
+                            // new visualization
                             scope.query.visualizations.push(result);
+                            scope.onNewSuccess && scope.onNewSuccess(result);
                         }
                     }, function error() {
                         growl.addErrorMessage("Visualization could not be saved");
@@ -141,9 +159,12 @@
         }
     };
 
+
+
     angular.module('redash.visualization', [])
         .provider('Visualization', VisualizationProvider)
         .directive('visualizationRenderer', ['Visualization', VisualizationRenderer])
         .directive('visualizationOptionsEditor', ['Visualization', VisualizationOptionsEditor])
+        .directive('filters', Filters)
         .directive('editVisulatizationForm', ['Visualization', 'growl', EditVisualizationForm])
 })();

rd_ui/app/scripts/visualizations/chart.js

@@ -1,7 +1,7 @@
 (function () {
   var chartVisualization = angular.module('redash.visualization');
 
-    chartVisualization.config(['VisualizationProvider', function(VisualizationProvider) {
+  chartVisualization.config(['VisualizationProvider', function (VisualizationProvider) {
     var renderTemplate = '<chart-renderer options="visualization.options" query-result="queryResult"></chart-renderer>';
     var editTemplate = '<chart-editor></chart-editor>';
     var defaultOptions = {
@@ -33,7 +33,7 @@
         $scope.chartSeries = [];
         $scope.chartOptions = {};
 
-                $scope.$watch('options', function(chartOptions) {
+        $scope.$watch('options', function (chartOptions) {
           if (chartOptions) {
             $scope.chartOptions = chartOptions;
           }
@@ -72,6 +72,13 @@
           "Percent": "percent"
         };
 
+        scope.xAxisOptions = {
+          "Date/Time": "datetime",
+          "Linear": "linear",
+          "Category": "category"
+        };
+
+        scope.xAxisType = "datetime";
         scope.stacking = "none";
 
         var chartOptionsUnwatch = null;
@@ -93,11 +100,21 @@
                 scope.visualization.options.series.stacking = stacking;
               }
             });
+
+            xAxisUnwatch = scope.$watch("xAxisType", function (xAxisType) {
+              scope.visualization.options.xAxis = scope.visualization.options.xAxis || {};
+              scope.visualization.options.xAxis.type = xAxisType;
+            });
           } else {
             if (chartOptionsUnwatch) {
               chartOptionsUnwatch();
               chartOptionsUnwatch = null;
             }
+
+            if (xAxisUnwatch) {
+              xAxisUnwatch();
+              xAxisUnwatch = null;
+            }
           }
         });
       }

rd_ui/app/scripts/visualizations/table.js

@@ -38,11 +38,10 @@
                         $scope.gridData = [];
                         $scope.filters = [];
                     } else {
-
-
                         $scope.filters = $scope.queryResult.getFilters();
 
-                        var gridData = _.map($scope.queryResult.getData(), function (row) {
+                        var prepareGridData = function(data) {
+                            var gridData = _.map(data, function (row) {
                                 var newRow = {};
                                 _.each(row, function (val, key) {
                                     newRow[$scope.queryResult.getColumnCleanName(key)] = val;
@@ -50,14 +49,21 @@
                                 return newRow;
                             });
 
+                            return gridData;
+                        };
+
+                        $scope.gridData = prepareGridData($scope.queryResult.getData());
+
                         $scope.gridColumns = _.map($scope.queryResult.getColumnCleanNames(), function (col, i) {
                             var columnDefinition = {
                                 'label': $scope.queryResult.getColumnFriendlyNames()[i],
                                 'map': col
                             };
 
-                            if (gridData.length > 0) {
-                                var exampleData = gridData[0][col];
+                            var rawData = $scope.queryResult.getRawData();
+
+                            if (rawData.length > 0) {
+                                var exampleData = rawData[0][col];
                                 if (angular.isNumber(exampleData)) {
                                     columnDefinition['formatFunction'] = 'number';
                                     columnDefinition['formatParameter'] = 2;
@@ -76,20 +82,6 @@
 
                             return columnDefinition;
                         });
-
-                        $scope.gridData = _.clone(gridData);
-
-                        $scope.$watch('filters', function (filters) {
-                            $scope.gridData = _.filter(gridData, function (row) {
-                                return _.reduce(filters, function (memo, filter) {
-                                    if (filter.current == 'All') {
-                                        return memo && true;
-                                    }
-
-                                    return (memo && row[$scope.queryResult.getColumnCleanName(filter.name)] == filter.current);
-                                }, true);
-                            });
-                        }, true);
                     }
                 });
             }]

rd_ui/app/styles/redash.css

@@ -37,6 +37,18 @@ a.navbar-brand {
     bottom: -11px;
 }
 
+.details-toggle {
+    cursor: pointer;
+}
+.details-toggle::before {
+    content: '▸';
+    margin-right: 5px;
+}
+.details-toggle.open::before {
+    content: '▾';
+    margin-right: 5px;
+}
+
 .edit-in-place span {
     white-space: pre-line;
 }
@@ -74,10 +86,15 @@ a.navbar-brand {
     margin-bottom: 0px;
 }
 
-.panel-heading > a {
+.panel-heading > a,
+.panel-heading .query-link {
     color: inherit;
 }
 
+.panel-heading .query-link:hover {
+    text-decoration: none;
+}
+
 /* angular-growl */
 .growl {
     position: fixed;

rd_ui/app/views/dashboard.html

@@ -23,9 +23,10 @@
 
             <div class="panel panel-default">
                 <div class="panel-heading">
-                    <h3 class="panel-title" style="cursor: pointer;" ng-click="open(query, widget.visualization)">
+                    <h3 class="panel-title">
                         <p>
-                            <span ng-bind="query.name"></span>
+                            <span ng-hide="currentUser.hasPermission('view_query')">{{query.name}}</span>
+                            <query-link query="query" visualization="widget.visualization" ng-show="currentUser.hasPermission('view_query')"></query-link>
                         </p>
                         <div class="text-muted" ng-bind="query.description"></div>
                     </h3>
@@ -39,7 +40,7 @@
                           tooltip-placement="bottom">Updated: <span am-time-ago="queryResult.getUpdatedAt()"></span></span>
 
                     <span class="pull-right">
-                        <a class="btn btn-default btn-xs" ng-href="/queries/{{query.id}}#{{widget.visualization.id}}"><span class="glyphicon glyphicon-link"></span></a>
+                        <a class="btn btn-default btn-xs" ng-href="/queries/{{query.id}}#{{widget.visualization.id}}" ng-show="currentUser.hasPermission('view_query')"><span class="glyphicon glyphicon-link"></span></a>
                         <button type="button" class="btn btn-default btn-xs" ng-show="dashboard.canEdit()" ng-click="deleteWidget()" title="Remove Widget"><span class="glyphicon glyphicon-trash"></span></button>
                     </span>
 

rd_ui/app/views/grid_renderer.html

@@ -1,17 +1,4 @@
 <div>
-    <div class="btn-group pull-right" ng-repeat="filter in filters">
-        <button type="button" class="btn btn-default dropdown-toggle" data-toggle="dropdown">
-            {{filter.friendlyName}}: {{filter.current}}<span class="caret"></span>
-        </button>
-        <ul class="dropdown-menu" role="menu">
-            <li ng-repeat="value in filter.values">
-                <a href="#" ng-click="filter.current = value">{{value}}</a>
-            </li>
-            <li class="divider"></li>
-            <li><a href="#" ng-click="filter.current = 'All'">All</a></li>
-        </ul>
-    </div>
-
     <smart-table rows="gridData" columns="gridColumns"
                  config="gridConfig"
                  class="table table-condensed table-hover"></smart-table>

rd_ui/app/views/new_widget_form.html

@@ -12,7 +12,7 @@
                         <input class="form-control" placeholder="Query Id" ng-model="queryId">
                       </div>
                       <button type="submit" class="btn btn-primary" ng-disabled="!queryId">
-                        <span class="glyphicon glyphicon-refresh"></span> Load
+                        Load visualizations
                       </button>
                     </form>
                 </p>
@@ -29,7 +29,7 @@
                   </div>
                 </div>
             </div>
-            <div class="modal-footer">
+            <div class="modal-footer" ng-if="selectedVis">
                 <button type="button" class="btn btn-default" ng-disabled="saveInProgress" data-dismiss="modal">Close</button>
                 <button type="button" class="btn btn-primary" ng-disabled="saveInProgress" ng-click="saveWidget()">Add to Dashboard</button>
             </div>

rd_ui/app/views/query.html

@@ -1,27 +1,28 @@
 
 <div class="container">
+
+    <alert-unsaved-changes ng-if="canEdit" is-dirty="isDirty"></alert-unsaved-changes>
+
     <div class="row">
         <div class="col-lg-12">
             <div class="row">
                 <div class="col-lg-10">
                     <h2>
-                        <edit-in-place editable="isOwner" done="saveQuery" ignore-blanks='true' value="query.name"></edit-in-place>
+                        <edit-in-place editable="isQueryOwner" done="saveName" ignore-blanks='true' value="query.name"></edit-in-place>
                     </h2>
                     <p>
                         <em>
-                            <edit-in-place editable="isOwner" done="saveQuery" editor="textarea" placeholder="No description" ignore-blanks='false' value="query.description"></edit-in-place>
+                            <edit-in-place editable="isQueryOwner" done="saveDescription" editor="textarea" placeholder="No description" ignore-blanks='false' value="query.description"></edit-in-place>
                         </em>
                     </p>
                 </div>
-                <div class="col-lg-2" ng-hide="isNewQuery || !currentUser.hasPermission('view_source')">
-                    <a ng-href="{{sourceHref}}" ng-click="toggleSource()" class="hidden-xs pull-right">
-                        <span ng-show="isSourceVisible">Hide Source</span>
-                        <span ng-show="!isSourceVisible">View Source</span>
-                    </a>
+
+                <div class="col-lg-2">
+                    <div class="rd-hidden-xs pull-right">
+                        <query-source-link></query-source-link>
+                    </div>
                 </div>
             </div>
-
-
 
             <div class="visible-xs">
                 <p>
@@ -31,7 +32,8 @@
                     </strong>
                     &nbsp;
                     <span class="text-muted">Created By </span>
-                    <strong>{{query.user.name}}</strong>
+                    <strong ng-hide="isQueryOwner">{{query.user.name}}</strong>
+                    <strong ng-show="isQueryOwner">You</strong>
                     &nbsp;
                     <span class="text-muted">Runtime </span>
                     <strong ng-show="!queryExecuting">{{queryResult.getRuntime() | durationHumanize}}</strong>
@@ -41,10 +43,7 @@
                     <strong>{{queryResult.getData().length}}</strong>
                 </p>
                 <p>
-                    <a ng-href="{{sourceHref}}" ng-click="toggleSource()">
-                        <span ng-show="isSourceVisible">Hide Source</span>
-                        <span ng-show="!isSourceVisible">View Source</span>
-                    </a>
+                    <query-source-link></query-source-link>
                 </p>
             </div>
         </div>
@@ -54,31 +53,31 @@
 
     <div class="row">
         <div class="col-lg-12">
-            <div ng-show="isSourceVisible">
+            <div ng-show="sourceMode">
                 <p>
                     <button type="button" class="btn btn-primary btn-xs" ng-disabled="queryExecuting" ng-click="executeQuery()">
                         <span class="glyphicon glyphicon-play"></span> Execute
                     </button>
-                    <button type="button" class="btn btn-default btn-xs" ng-click="formatQuery()">
-                        <span class="glyphicon glyphicon-indent-left"></span> Format SQL
-                    </button>
+                    <query-formatter></query-formatter>
                     <span class="pull-right">
-                        <button class="btn btn-xs btn-default rd-hidden-xs" ng-show="canFork" ng-click="duplicateQuery()">
+                        <button class="btn btn-xs btn-default rd-hidden-xs" ng-click="duplicateQuery()">
                             <span class="glyphicon glyphicon-share-alt"></span> Fork
                         </button>
 
                         <button class="btn btn-success btn-xs" ng-show="canEdit" ng-click="saveQuery()">
-                            <span class="glyphicon glyphicon-floppy-disk"> </span> Save<span ng-show="dirty">&#42;</span>
+                            <span class="glyphicon glyphicon-floppy-disk"> </span> Save<span ng-show="isDirty">&#42;</span>
                         </button>
                     </span>
                 </p>
             </div>
 
             <!-- code editor -->
-            <p ng-show="isSourceVisible">
-                <textarea ui-codemirror="editorOptions" ui-refresh="isSourceVisible" ng-model="query.query"></textarea>
+            <div ng-show="sourceMode">
+                <p>
+                    <query-editor query="query" lock="queryExecuting"></query-editor>
                 </p>
-            <hr ng-show="isSourceVisible">
+                <hr>
+            </div>
         </div>
 
     </div>
@@ -94,7 +93,9 @@
             </p>
             <p>
                 <span class="glyphicon glyphicon-user"></span>
-                <span class="text-muted">Created By </span><strong>{{query.user.name}}</strong>
+                <span class="text-muted">Created By </span>
+                <strong ng-hide="isQueryOwner">{{query.user.name}}</strong>
+                <strong ng-show="isQueryOwner">You</strong>
             </p>
             <p>
                 <span class="glyphicon glyphicon-play"></span>
@@ -109,7 +110,13 @@
             <p>
                 <span class="glyphicon glyphicon-refresh"></span>
                 <span class="text-muted">Refresh Interval</span>
-                <select ng-disabled="!isOwner" ng-model="query.ttl" ng-change="saveQuery()" ng-options="c.value as c.name for c in refreshOptions"></select>
+                <query-refresh-select></query-refresh-select>
+            </p>
+
+            <p>
+                <span class="glyphicon glyphicon-hdd"></span>
+                <span class="text-muted">Data Source</span>
+                <select ng-disabled="!isQueryOwner" ng-model="query.data_source_id" ng-change="updateDataSource()" ng-options="ds.id as ds.name for ds in dataSources"></select>
             </p>
 
             <hr>
@@ -141,7 +148,7 @@
                         <ul class="nav nav-tabs">
                             <rd-tab tab-id="table" name="Table"></rd-tab>
                             <rd-tab tab-id="pivot" name="Pivot Table"></rd-tab>
-                            <rd-tab tab-id="{{vis.id}}" name="{{vis.name}}" ng-hide="vis.type=='TABLE'" ng-repeat="vis in query.visualizations">
+                            <rd-tab tab-id="{{vis.id}}" name="{{vis.name}}" ng-if="vis.type!='TABLE'" ng-repeat="vis in query.visualizations">
                                 <span class="remove" ng-click="deleteVisualization($event, vis)" ng-show="canEdit"> &times;</span>
                             </rd-tab>
                             <rd-tab tab-id="add" name="&plus; New" removeable="true" ng-show="canEdit"></rd-tab>
@@ -150,17 +157,22 @@
                 </div>
                 <div class="row">
                     <div class="col-lg-12">
-                        <grid-renderer ng-show="selectedTab == 'table'" query-result="queryResult" items-per-page="50"></grid-renderer>
+                        <div ng-show="selectedTab == 'table'" >
+                            <filters></filters>
+                            <grid-renderer query-result="queryResult" items-per-page="50"></grid-renderer>
+                        </div>
+
                         <pivot-table-renderer ng-show="selectedTab == 'pivot'" query-result="queryResult"></pivot-table-renderer>
 
                         <div ng-show="selectedTab == vis.id" ng-repeat="vis in query.visualizations">
                             <visualization-renderer visualization="vis" query-result="queryResult"></visualization-renderer>
+
                             <edit-visulatization-form visualization="vis" query="query" query-result="queryResult" ng-show="canEdit"></edit-visulatization-form>
                         </div>
 
                         <div ng-show="selectedTab == 'add'">
                             <visualization-renderer visualization="newVisualization" query-result="queryResult"></visualization-renderer>
-                            <edit-visulatization-form visualization="newVisualization" query="query" ng-show="canEdit"></edit-visulatization-form>
+                            <edit-visulatization-form visualization="newVisualization" query="query" ng-show="canEdit" on-new-success="setVisualizationTab"></edit-visulatization-form>
                         </div>
                     </div>
                 </div>

rd_ui/app/views/queryfiddle.html

@@ -1,102 +0,0 @@
-<div class="container">
-    <div class="row">
-        <div class="panel panel-default">
-            <div class="panel-heading">
-                <h3 class="panel-title">
-                    <p>
-                     <edit-in-place editable="currentUser.canEdit(query)" ignore-blanks='true' value="query.name"></edit-in-place>
-                 </p>
-                </h3>
-                <p>
-                    <edit-in-place editable="currentUser.canEdit(query)" editor="textarea" placeholder="No description" ignore-blanks='false' value="query.description" class="text-muted"></edit-in-place>
-                </p>
-            </div>
-            <div class="panel-body">
-                <textarea ui-codemirror="editorOptions" ng-model="query.query"></textarea>
-
-                <div>
-                    <a class="btn btn-default" ng-disabled="queryExecuting || !queryResult.getData()" ng-href="{{dataUri}}" download="{{dataFilename}}" target="_self">
-                        <span class="glyphicon glyphicon-floppy-disk"></span> Download Data Set
-                    </a>
-                    <button type="button" class="btn btn-default center-x" ng-click="formatQuery()"><span class="glyphicon glyphicon-ok"></span> Format SQL</button>
-
-                    <div class="btn-group pull-right">
-                        <button type="button" class="btn btn-default" ng-click="duplicateQuery()">Duplicate</button>
-                        <button type="button" class="btn btn-default" ng-disabled="!currentUser.canEdit(query)" ng-click="saveQuery()">Save
-                            <span ng-show="dirty">&#42;</span>
-                        </button>
-                        <button type="button" class="btn btn-primary" ng-disabled="queryExecuting" ng-click="executeQuery()">Execute</button>
-                    </div>
-                </div>
-
-            </div>
-            <div class="panel-footer">
-                <span ng-show="queryResult.getRuntime()>=0">Query runtime: {{queryResult.getRuntime() | durationHumanize}} | </span>
-                <span ng-show="queryResult.query_result.retrieved_at">Last update time: <span am-time-ago="queryResult.query_result.retrieved_at"></span> | </span>
-                <span ng-show="queryResult.getStatus() == 'done'">Rows: {{queryResult.getData().length}} | </span>
-                Created by: {{query.user.name}}
-                <div class="pull-right">Refresh query: <select ng-model="query.ttl" ng-options="c.value as c.name for c in refreshOptions"></select><br></div>
-            </div>
-        </div>
-
-        <div class="alert alert-info" ng-show="queryResult.getStatus() == 'processing'">
-            Executing query... <rd-timer timestamp="queryResult.getUpdatedAt()"></rd-timer>
-            <button type="button" class="btn btn-warning btn-xs pull-right" ng-disabled="cancelling" ng-click="cancelExecution()">Cancel</button>
-        </div>
-        <div class="alert alert-info" ng-show="queryResult.getStatus() == 'waiting'">
-            Query in queue... <rd-timer timestamp="queryResult.getUpdatedAt()"></rd-timer>
-            <button type="button" class="btn btn-warning btn-xs pull-right" ng-disabled="cancelling" ng-click="cancelExecution()">Cancel</button>
-        </div>
-        <div class="alert alert-danger" ng-show="queryResult.getError()">Error running query: <strong>{{queryResult.getError()}}</strong></div>
-    </div>
-
-    <div class="row" ng-show="queryResult.getStatus() == 'done'">
-        <ul class="nav nav-tabs">
-            <rd-tab tab-id="table" name="Table"></rd-tab>
-            <rd-tab tab-id="pivot" name="Pivot Table"></rd-tab>
-            <!-- hide the table visualization -->
-            <rd-tab tab-id="{{vis.id}}" name="{{vis.name}}" ng-hide="vis.type=='TABLE'" ng-repeat="vis in query.visualizations">
-                <span class="remove" ng-click="deleteVisualization($event, vis)" ng-show="currentUser.canEdit(query)"> &times;</span>
-            </rd-tab>
-            <rd-tab tab-id="add" name="&plus; New" removeable="true" ng-show="currentUser.canEdit(query)"></rd-tab>
-        </ul>
-
-        <div class="col-lg-12" ng-show="selectedTab == 'table'">
-            <grid-renderer query-result="queryResult" items-per-page="50"></grid-renderer>
-        </div>
-
-        <div class="col-lg-12" ng-show="selectedTab == 'pivot'">
-            <pivot-table-renderer query-result="queryResult"></pivot-table-renderer>
-        </div>
-
-        <div class="col-lg-12" ng-show="selectedTab == vis.id" ng-repeat="vis in query.visualizations">
-            <div class="row" ng-show="currentUser.canEdit(query)">
-                <p>
-                    <div class="col-lg-12">
-                        <edit-visulatization-form visualization="vis" query="query" query-result="queryResult"></edit-visulatization-form>
-                    </div>
-                </p>
-            </div>
-            <div class="row">
-                <p>
-                    <div class="col-lg-12">
-                        <visualization-renderer visualization="vis" query-result="queryResult"></visualization-renderer>
-                    </div>
-                </p>
-            </div>
-        </div>
-
-        <div class="col-lg-12" ng-show="selectedTab == 'add'">
-            <div class="row">
-                 <p>
-                    <div class="col-lg-6">
-                        <edit-visulatization-form visualization="newVisualization" query="query"></edit-visulatization-form>
-                    </div>
-                    <div class="col-lg-6">
-                        <visualization-renderer visualization="newVisualization" query-result="queryResult"></visualization-renderer>
-                    </div>
-                </p>
-            </div>
-        </div>
-    </div>
-</div>

rd_ui/app/views/visualizations/chart_editor.html

@@ -1,7 +1,14 @@
-<div class="form-group">
+<div>
+  <div class="form-group">
       <label class="control-label">Chart Type</label>
       <select required ng-model="visualization.options.series.type" ng-options="value as key for (key, value) in seriesTypes" class="form-control"></select>
+  </div>
 
+  <div class="form-group">
       <label class="control-label">Stacking</label>
       <select required ng-model="stacking" ng-options="value as key for (key, value) in stackingOptions" class="form-control"></select>
+
+      <label class="control-label">X Axis Type</label>
+      <select required ng-model="xAxisType" ng-options="value as key for (key, value) in xAxisOptions" class="form-control"></select>
+  </div>
 </div>

rd_ui/app/views/visualizations/edit_visualization.html

@@ -1,4 +1,7 @@
-<form role="form" name="visForm" ng-submit="submit()">
+<div>
+  <span ng-click="visEdit=!visEdit" class="details-toggle" ng-class="{open: visEdit}">Edit</span>
+
+  <form ng-if="visEdit" role="form" name="visForm" ng-submit="submit()">
     <div class="form-group">
       <label class="control-label">Name</label>
       <input name="name" type="text" class="form-control" ng-model="visualization.name" placeholder="{{visualization.type | capitalize}}">
@@ -11,8 +14,14 @@
 
     <visualization-options-editor></visualization-options-editor>
 
+    <div class="form-group" ng-if="editRawOptions">
+      <label class="control-label">Advanced</label>
+      <textarea json-text ng-model="visualization.options" class="form-control" rows="10"></textarea>
+    </div>
+
     <div class="form-group">
       <button type="submit" class="btn btn-primary">Save</button>
     </div>
 
-</form>
+  </form>
+</div>

rd_ui/app/views/visualizations/filters.html

@@ -0,0 +1,8 @@
+<div class="well well-sm" ng-show="filters">
+    <div ng-repeat="filter in filters">
+        {{filter.friendlyName}}:
+        <select ui-select2='{width:"50%"}' ng-model="filter.current" ng-multiple="{{filter.multiple}}">
+            <option ng-repeat="value in filter.values" value="{{value}}">{{value}}</option>
+        </select>
+    </div>
+</div>

rd_ui/bower.json

@@ -13,19 +13,21 @@
     "angular-ui-codemirror": "0.0.5",
     "highcharts": "3.0.1",
     "underscore": "1.5.1",
-    "angular-resource": "1.0.7",
+    "angular-resource": "1.2.15",
     "angular-growl": "0.3.1",
     "angular-route": "1.2.7",
     "pivottable": "https://github.com/arikfr/pivottable.git",
     "cornelius": "https://github.com/restorando/cornelius.git",
     "gridster": "0.2.0",
-    "mousetrap": "~1.4.6"
+    "mousetrap": "~1.4.6",
+    "angular-ui-select2": "~0.0.5",
+    "jquery-ui": "~1.10.4"
   },
   "devDependencies": {
     "angular-mocks": "~1.0.7",
     "angular-scenario": "~1.0.7"
   },
   "resolutions": {
-    "angular": "~1.2.7"
+    "angular": "1.2.7"
   }
 }

redash/__init__.py

@@ -1,5 +1,6 @@
 import json
 import urlparse
+import logging
 from flask import Flask, make_response
 from flask.ext.restful import Api
 from flask_peewee.db import Database
@@ -10,6 +11,9 @@ from redash import settings, utils
 
 __version__ = '0.3.5'
 
+logging.getLogger().addHandler(logging.StreamHandler())
+logging.getLogger().setLevel(settings.LOG_LEVEL)
+
 app = Flask(__name__,
             template_folder=settings.STATIC_ASSETS_PATH,
             static_folder=settings.STATIC_ASSETS_PATH,
@@ -42,6 +46,6 @@ redis_connection = redis.StrictRedis(host=redis_url.hostname, port=redis_url.por
 statsd_client = StatsClient(host=settings.STATSD_HOST, port=settings.STATSD_PORT, prefix=settings.STATSD_PREFIX)
 
 from redash import data
-data_manager = data.Manager(redis_connection, db, statsd_client)
+data_manager = data.Manager(redis_connection, statsd_client)
 
 from redash import controllers

redash/authentication.py

@@ -40,6 +40,7 @@ class HMACAuthentication(object):
             calculated_signature = sign(query.api_key, request.path, expires)
 
             if query.api_key and signature == calculated_signature:
+                login_user(models.ApiUser(query.api_key), remember=False)
                 return True
 
         return False
@@ -78,7 +79,8 @@ def create_and_login_user(app, user):
     except models.User.DoesNotExist:
         logger.debug("Creating user object (%r)", user.name)
         user_object = models.User.create(name=user.name, email=user.email,
-                                  is_admin=(user.email in settings.ADMINS))
+                                  is_admin=(user.email in settings.ADMINS), 
+                                  groups = ['admin', 'default'] if (user.email in settings.ADMINS) else ['default'])
 
     login_user(user_object, remember=True)
 

redash/controllers.py

@@ -10,6 +10,7 @@ import json
 import numbers
 import cStringIO
 import datetime
+import itertools
 
 from flask import render_template, send_from_directory, make_response, request, jsonify, redirect, \
     session, url_for
@@ -47,7 +48,8 @@ def index(**kwargs):
         'id': current_user.id,
         'name': current_user.name,
         'email': current_user.email,
-        'permissions': current_user.permissions
+        'groups': current_user.groups,
+        'permissions': list(itertools.chain(*[g.permissions for g in models.Group.select().where(models.Group.name << current_user.groups)]))
     }
 
     return render_template("index.html", user=json.dumps(user), name=settings.NAME,
@@ -129,6 +131,14 @@ class BaseResource(Resource):
         return current_user._get_current_object()
 
 
+class DataSourceListAPI(BaseResource):
+    def get(self):
+        data_sources = [ds.to_dict() for ds in models.DataSource.select()]
+        return data_sources
+
+api.add_resource(DataSourceListAPI, '/api/data_sources', endpoint='data_sources')
+
+
 class DashboardListAPI(BaseResource):
     def get(self):
         dashboards = [d.to_dict() for d in
@@ -229,11 +239,11 @@ class QueryListAPI(BaseResource):
     @require_permission('create_query')
     def post(self):
         query_def = request.get_json(force=True)
-        # id, created_at, api_key
         for field in ['id', 'created_at', 'api_key', 'visualizations', 'latest_query_data']:
             query_def.pop(field, None)
 
         query_def['user'] = self.current_user
+        query_def['data_source'] = query_def.pop('data_source_id')
         query = models.Query(**query_def)
         query.save()
 
@@ -241,6 +251,7 @@ class QueryListAPI(BaseResource):
 
         return query.to_dict(with_result=False)
 
+    @require_permission('view_query')
     def get(self):
         return [q.to_dict(with_result=False, with_stats=True) for q in models.Query.all_queries()]
 
@@ -255,12 +266,16 @@ class QueryAPI(BaseResource):
         if 'latest_query_data_id' in query_def:
             query_def['latest_query_data'] = query_def.pop('latest_query_data_id')
 
+        if 'data_source_id' in query_def:
+            query_def['data_source'] = query_def.pop('data_source_id')
+
         models.Query.update_instance(query_id, **query_def)
 
         query = models.Query.get_by_id(query_id)
 
         return query.to_dict(with_result=False, with_visualizations=True)
 
+    @require_permission('view_query')
     def get(self, query_id):
         q = models.Query.get(models.Query.id == query_id)
         if q:
@@ -313,6 +328,44 @@ class QueryResultListAPI(BaseResource):
     @require_permission('execute_query')
     def post(self):
         params = request.json
+        parsedQuery = sqlparse.parse(params['query'])
+        
+        if len(parsedQuery) > 1:
+            return {
+                'job': {
+                    'error': 'Please, execute only one statement at a time'
+                }
+            }
+        
+        parsedQuery = parsedQuery[0]
+        
+        if len([x for x in parsedQuery.flatten() if x.ttype == sqlparse.tokens.DDL]):
+            return {
+                'job': {
+                    'error': 'Only SELECT statements are allowed'
+                }
+            }
+        
+        # Check the type of queries executed
+        for dml in [x for x in parsedQuery.flatten() if x.ttype == sqlparse.tokens.DML]:
+            if dml.normalized != 'SELECT':
+                return {
+                    'job': {
+                        'error': 'Only SELECT statements are allowed'
+                    }
+                }
+        
+        # Get table identifier
+        parsedTables = utils.extract_table_names(parsedQuery.tokens)
+        allowedTables = list(set(itertools.chain(*[g.tables for g in models.Group.select().where(models.Group.name << self.current_user.groups)])))
+        
+        for table in parsedTables:
+            if table not in allowedTables and '*' not in allowedTables:
+                return {
+                    'job': {
+                        'error': 'Access denied for table %s' % (table)
+                    }
+                }
         
         models.ActivityLog(
             user=self.current_user,
@@ -323,32 +376,35 @@ class QueryResultListAPI(BaseResource):
         if params['ttl'] == 0:
             query_result = None
         else:
-            query_result = data_manager.get_query_result(params['query'], int(params['ttl']))
+            query_result = models.QueryResult.get_latest(params['data_source_id'], params['query'], int(params['ttl']))
 
         if query_result:
-            return {'query_result': query_result.to_dict(parse_data=True)}
+            return {'query_result': query_result.to_dict()}
         else:
-            job = data_manager.add_job(params['query'], data.Job.HIGH_PRIORITY)
+            data_source = models.DataSource.get_by_id(params['data_source_id'])
+            job = data_manager.add_job(params['query'], data.Job.HIGH_PRIORITY, data_source)
             return {'job': job.to_dict()}
 
 
 class QueryResultAPI(BaseResource):
+    @require_permission('view_query')
     def get(self, query_result_id):
-        query_result = data_manager.get_query_result_by_id(query_result_id)
+        query_result = models.QueryResult.get_by_id(query_result_id)
         if query_result:
-            return {'query_result': query_result.to_dict(parse_data=True)}
+            return {'query_result': query_result.to_dict()}
         else:
             abort(404)
 
 
 class CsvQueryResultsAPI(BaseResource):
+    @require_permission('view_query')
     def get(self, query_id, query_result_id=None):
         if not query_result_id:
             query = models.Query.get(models.Query.id == query_id)
             if query:
                 query_result_id = query._data['latest_query_data']
 
-        query_result = query_result_id and data_manager.get_query_result_by_id(query_result_id)
+        query_result = query_result_id and models.QueryResult.get_by_id(query_result_id)
         if query_result:
             s = cStringIO.StringIO()
 

redash/data/manager.py

@@ -1,34 +1,30 @@
 """
 Data manager. Used to manage and coordinate execution of queries.
 """
-from contextlib import contextmanager
-import collections
-import json
 import time
 import logging
-import psycopg2
+import peewee
 import qr
 import redis
+import json
+from redash import models
 from redash.data import worker
 from redash.utils import gen_query_hash
 
-class QueryResult(collections.namedtuple('QueryData', 'id query data runtime retrieved_at query_hash')):
-    def to_dict(self, parse_data=False):
-        d = self._asdict()
 
-        if parse_data and d['data']:
-            d['data'] = json.loads(d['data'])
-
-        return d
+class JSONPriorityQueue(qr.PriorityQueue):
+    """ Use a JSON serializer to help with cross language support """
+    def __init__(self, key, **kwargs):
+        super(qr.PriorityQueue, self).__init__(key, **kwargs)
+        self.serializer = json
 
 
 class Manager(object):
-    def __init__(self, redis_connection, db, statsd_client):
+    def __init__(self, redis_connection, statsd_client):
         self.statsd_client = statsd_client
         self.redis_connection = redis_connection
-        self.db = db
         self.workers = []
-        self.queue = qr.PriorityQueue("jobs", **self.redis_connection.connection_pool.connection_kwargs)
+        self.queue = JSONPriorityQueue("jobs", **self.redis_connection.connection_pool.connection_kwargs)
         self.max_retries = 5
         self.status = {
             'last_refresh_at': 0,
@@ -37,36 +33,7 @@ class Manager(object):
 
         self._save_status()
 
-    # TODO: Use our Django Models
-    def get_query_result_by_id(self, query_result_id):
-        with self.db_transaction() as cursor:
-            sql = "SELECT id, query, data, runtime, retrieved_at, query_hash FROM query_results " \
-                  "WHERE id=%s LIMIT 1"
-            cursor.execute(sql, (query_result_id,))
-            query_result = cursor.fetchone()
-
-        if query_result:
-            query_result = QueryResult(*query_result)
-
-        return query_result
-
-    def get_query_result(self, query, ttl=0):
-        query_hash = gen_query_hash(query)
-
-        with self.db_transaction() as cursor:
-            sql = "SELECT id, query, data, runtime, retrieved_at, query_hash FROM query_results " \
-                  "WHERE query_hash=%s " \
-                  "AND retrieved_at < now() at time zone 'utc' - interval '%s second'" \
-                  "ORDER BY retrieved_at DESC LIMIT 1"
-            cursor.execute(sql, (query_hash, psycopg2.extensions.AsIs(ttl)))
-            query_result = cursor.fetchone()
-
-        if query_result:
-            query_result = QueryResult(*query_result)
-
-        return query_result
-
-    def add_job(self, query, priority):
+    def add_job(self, query, priority, data_source):
         query_hash = gen_query_hash(query)
         logging.info("[Manager][%s] Inserting job with priority=%s", query_hash, priority)
         try_count = 0
@@ -83,7 +50,11 @@ class Manager(object):
                     logging.info("[Manager][%s] Found existing job: %s", query_hash, job_id)
                     job = worker.Job.load(self.redis_connection, job_id)
                 else:
-                    job = worker.Job(self.redis_connection, query, priority)
+                    job = worker.Job(self.redis_connection, query=query, priority=priority,
+                                     data_source_id=data_source.id,
+                                     data_source_name=data_source.name,
+                                     data_source_type=data_source.type,
+                                     data_source_options=data_source.options)
                     pipe.multi()
                     job.save(pipe)
                     logging.info("[Manager][%s] Created new job: %s", query_hash, job.id)
@@ -113,85 +84,64 @@ class Manager(object):
                                  time.time() - float(manager_status['last_refresh_at']))
 
     def refresh_queries(self):
-        sql = """SELECT first_value(t1."query") over(partition by t1.query_hash)
-                 FROM "queries" AS t1
-                 INNER JOIN "query_results" AS t2 ON (t1."latest_query_data_id" = t2."id")
-                 WHERE ((t1."ttl" > 0) AND ((t2."retrieved_at" + t1.ttl * interval '1 second') <
-                                           now() at time zone 'utc'));
-        """
+        # TODO: this will only execute scheduled queries that were executed before. I think this is
+        # a reasonable assumption, but worth revisiting.
+
+        # TODO: move this logic to the model.
+        outdated_queries = models.Query.select(peewee.Func('first_value', models.Query.id)\
+            .over(partition_by=[models.Query.query_hash, models.Query.data_source]))\
+            .join(models.QueryResult)\
+            .where(models.Query.ttl > 0,
+                   (models.QueryResult.retrieved_at +
+                    (models.Query.ttl * peewee.SQL("interval '1 second'"))) <
+                   peewee.SQL("(now() at time zone 'utc')"))
+
+        queries = models.Query.select(models.Query, models.DataSource).join(models.DataSource)\
+            .where(models.Query.id << outdated_queries)
 
         self.status['last_refresh_at'] = time.time()
         self._save_status()
 
         logging.info("Refreshing queries...")
-        queries = self.run_query(sql)
 
+        outdated_queries_count = 0
         for query in queries:
-            self.add_job(query[0], worker.Job.LOW_PRIORITY)
+            self.add_job(query.query, worker.Job.LOW_PRIORITY, query.data_source)
+            outdated_queries_count += 1
 
-        self.statsd_client.gauge('manager.outdated_queries', len(queries))
+        self.statsd_client.gauge('manager.outdated_queries', outdated_queries_count)
         self.statsd_client.gauge('manager.queue_size', self.redis_connection.zcard('jobs'))
 
-        logging.info("Done refreshing queries... %d" % len(queries))
+        logging.info("Done refreshing queries... %d" % outdated_queries_count)
 
-    def store_query_result(self, query, data, run_time, retrieved_at):
-        query_result_id = None
+    def store_query_result(self, data_source_id, query, data, run_time, retrieved_at):
         query_hash = gen_query_hash(query)
-        sql = "INSERT INTO query_results (query_hash, query, data, runtime, retrieved_at) " \
-              "VALUES (%s, %s, %s, %s, %s) RETURNING id"
-        with self.db_transaction() as cursor:
-            cursor.execute(sql, (query_hash, query, data, run_time, retrieved_at))
-            if cursor.rowcount == 1:
-                query_result_id = cursor.fetchone()[0]
-                logging.info("[Manager][%s] Inserted query data; id=%s", query_hash, query_result_id)
 
-                sql = "UPDATE queries SET latest_query_data_id=%s WHERE query_hash=%s"
-                cursor.execute(sql, (query_result_id, query_hash))
+        query_result = models.QueryResult.create(query_hash=query_hash,
+                                                 query=query,
+                                                 runtime=run_time,
+                                                 data_source=data_source_id,
+                                                 retrieved_at=retrieved_at,
+                                                 data=data)
 
-                logging.info("[Manager][%s] Updated %s queries.", query_hash, cursor.rowcount)
-            else:
-                logging.error("[Manager][%s] Failed inserting query data.", query_hash)
-        return query_result_id
+        logging.info("[Manager][%s] Inserted query data; id=%s", query_hash, query_result.id)
 
-    def run_query(self, *args):
-        sql = args[0]
-        logging.debug("running query: %s %s", sql, args[1:])
+        # TODO: move this logic to the model?
+        updated_count = models.Query.update(latest_query_data=query_result).\
+            where(models.Query.query_hash==query_hash, models.Query.data_source==data_source_id).\
+            execute()
 
-        with self.db_transaction() as cursor:
-            cursor.execute(sql, args[1:])
-            if cursor.description:
-                data = list(cursor)
-            else:
-                data = cursor.rowcount
+        logging.info("[Manager][%s] Updated %s queries.", query_hash, updated_count)
 
-        return data
+        return query_result.id
 
-    def start_workers(self, workers_count, connection_type, connection_string):
+    def start_workers(self, workers_count):
         if self.workers:
             return self.workers
 
-        if connection_type == 'mysql':
-            from redash.data import query_runner_mysql
-            runner = query_runner_mysql.mysql(connection_string)
-        elif connection_type == 'graphite':
-            from redash.data import query_runner_graphite
-            connection_params = json.loads(connection_string)
-            if connection_params['auth']:
-                connection_params['auth'] = tuple(connection_params['auth'])
-            else:
-                connection_params['auth'] = None
-            runner = query_runner_graphite.graphite(connection_params)
-        elif connection_type == 'bigquery':
-            from redash.data import query_runner_bigquery
-            connection_params = json.loads(connection_string)
-            runner = query_runner_bigquery.bigquery(connection_params)
-        else:
-            from redash.data import query_runner
-            runner = query_runner.redshift(connection_string)
-
         redis_connection_params = self.redis_connection.connection_pool.connection_kwargs
-        self.workers = [worker.Worker(worker_id, self, redis_connection_params, runner)
-                        for worker_id in range(workers_count)]
+        self.workers = [worker.Worker(worker_id, self, redis_connection_params)
+                        for worker_id in xrange(workers_count)]
         for w in self.workers:
             w.start()
 
@@ -202,20 +152,5 @@ class Manager(object):
             w.continue_working = False
             w.join()
 
-    @contextmanager
-    def db_transaction(self):
-        self.db.connect_db()
-
-        cursor = self.db.database.get_cursor()
-        try:
-            yield cursor
-        except:
-            self.db.database.rollback()
-            raise
-        else:
-            self.db.database.commit()
-        finally:
-            self.db.close_db(None)
-
     def _save_status(self):
         self.redis_connection.hmset('manager:status', self.status)

redash/data/query_runner.py

@@ -1,69 +1,30 @@
-"""
-QueryRunner is the function that the workers use, to execute queries. This is the Redshift
-(PostgreSQL in fact) version, but easily we can write another to support additional databases
-(MySQL and others).
-
-Because the worker just pass the query, this can be used with any data store that has some sort of
-query language (for example: HiveQL).
-"""
-import logging
 import json
-import sys
-import select
 
-import psycopg2
 
-from redash.utils import JSONEncoder
-
-def redshift(connection_string):
-    def column_friendly_name(column_name):
-        return column_name
- 
-    def wait(conn):
-        while 1:
-            state = conn.poll()
-            if state == psycopg2.extensions.POLL_OK:
-                break
-            elif state == psycopg2.extensions.POLL_WRITE:
-                select.select([], [conn.fileno()], [])
-            elif state == psycopg2.extensions.POLL_READ:
-                select.select([conn.fileno()], [], [])
+def get_query_runner(connection_type, connection_string):
+    if connection_type == 'mysql':
+        from redash.data import query_runner_mysql
+        runner = query_runner_mysql.mysql(connection_string)
+    elif connection_type == 'graphite':
+        from redash.data import query_runner_graphite
+        connection_params = json.loads(connection_string)
+        if connection_params['auth']:
+            connection_params['auth'] = tuple(connection_params['auth'])
         else:
-                raise psycopg2.OperationalError("poll() returned %s" % state)
+            connection_params['auth'] = None
+        runner = query_runner_graphite.graphite(connection_params)
+    elif connection_type == 'bigquery':
+        from redash.data import query_runner_bigquery
+        connection_params = json.loads(connection_string)
+        runner = query_runner_bigquery.bigquery(connection_params)
+    elif connection_type == 'script':
+        from redash.data import query_runner_script
+        runner = query_runner_script.script(connection_string)
+    elif connection_type == 'url':
+        from redash.data import query_runner_url
+        runner = query_runner_url.url(connection_string)
+    else:
+        from redash.data import query_runner_pg
+        runner = query_runner_pg.pg(connection_string)
 
-    def query_runner(query):
-        connection = psycopg2.connect(connection_string, async=True)
-        wait(connection)
- 
-        cursor = connection.cursor()
- 
-        try:
-            cursor.execute(query)
-            wait(connection)
- 
-            column_names = [col.name for col in cursor.description]
- 
-            rows = [dict(zip(column_names, row)) for row in cursor]
-            columns = [{'name': col.name,
-                        'friendly_name': column_friendly_name(col.name),
-                        'type': None} for col in cursor.description]
- 
-            data = {'columns': columns, 'rows': rows}
-            json_data = json.dumps(data, cls=JSONEncoder)
-            error = None
-            cursor.close()
-        except psycopg2.DatabaseError as e:
-            json_data = None
-            error = e.message
-        except KeyboardInterrupt:
-            connection.cancel()
-            error = "Query cancelled by user."
-            json_data = None
-        except Exception as e:
-            raise sys.exc_info()[1], None, sys.exc_info()[2]
-        finally:
-            connection.close()
- 
-        return json_data, error
- 
-    return query_runner
+    return runner

redash/data/query_runner_pg.py

@@ -0,0 +1,68 @@
+"""
+QueryRunner is the function that the workers use, to execute queries. This is the PostgreSQL
+version, but easily we can write another to support additional databases (MySQL and others).
+
+Because the worker just pass the query, this can be used with any data store that has some sort of
+query language (for example: HiveQL).
+"""
+import json
+import sys
+import select
+
+import psycopg2
+
+from redash.utils import JSONEncoder
+
+
+def pg(connection_string):
+    def column_friendly_name(column_name):
+        return column_name
+
+    def wait(conn):
+        while 1:
+            state = conn.poll()
+            if state == psycopg2.extensions.POLL_OK:
+                break
+            elif state == psycopg2.extensions.POLL_WRITE:
+                select.select([], [conn.fileno()], [])
+            elif state == psycopg2.extensions.POLL_READ:
+                select.select([conn.fileno()], [], [])
+            else:
+                raise psycopg2.OperationalError("poll() returned %s" % state)
+
+    def query_runner(query):
+        connection = psycopg2.connect(connection_string, async=True)
+        wait(connection)
+
+        cursor = connection.cursor()
+
+        try:
+            cursor.execute(query)
+            wait(connection)
+
+            column_names = [col.name for col in cursor.description]
+
+            rows = [dict(zip(column_names, row)) for row in cursor]
+            columns = [{'name': col.name,
+                        'friendly_name': column_friendly_name(col.name),
+                        'type': None} for col in cursor.description]
+
+            data = {'columns': columns, 'rows': rows}
+            json_data = json.dumps(data, cls=JSONEncoder)
+            error = None
+            cursor.close()
+        except psycopg2.DatabaseError as e:
+            json_data = None
+            error = e.message
+        except KeyboardInterrupt:
+            connection.cancel()
+            error = "Query cancelled by user."
+            json_data = None
+        except Exception as e:
+            raise sys.exc_info()[1], None, sys.exc_info()[2]
+        finally:
+            connection.close()
+
+        return json_data, error
+
+    return query_runner

redash/data/query_runner_script.py

@@ -0,0 +1,48 @@
+import json
+import logging
+import sys
+import os
+import subprocess
+
+# We use subprocess.check_output because we are lazy.
+# If someone will really want to run this on Python < 2.7 they can easily update the code to run
+# Popen, check the retcodes and other things and read the standard output to a variable.
+if not "check_output" in subprocess.__dict__:
+    print "ERROR: This runner uses subprocess.check_output function which exists in Python 2.7"
+
+def script(connection_string):
+
+    def query_runner(query):
+        try:
+            json_data = None
+            error = None
+
+            # Poor man's protection against running scripts from output the scripts directory
+            if connection_string.find("../") > -1:
+                return None, "Scripts can only be run from the configured scripts directory"
+
+            query = query.strip()
+
+            script = os.path.join(connection_string, query)
+            if not os.path.exists(script):
+                return None, "Script '%s' not found in script directory" % query
+
+            output = subprocess.check_output(script, shell=False)
+            if output != None:
+                output = output.strip()
+                if output != "":
+                    return output, None
+
+            error = "Error reading output"
+        except subprocess.CalledProcessError as e:
+            return None, str(e)
+        except KeyboardInterrupt:
+            error = "Query cancelled by user."
+            json_data = None
+        except Exception as e:
+            raise sys.exc_info()[1], None, sys.exc_info()[2]
+
+        return json_data, error
+
+    query_runner.annotate_query = False
+    return query_runner

redash/data/query_runner_url.py

@@ -0,0 +1,45 @@
+import json
+import logging
+import sys
+import os
+import urllib2
+
+def url(connection_string):
+
+    def query_runner(query):
+        base_url = connection_string
+
+        try:
+            json_data = None
+            error = None
+
+            query = query.strip()
+
+            if base_url is not None and base_url != "":
+                if query.find("://") > -1:
+                    return None, "Accepting only relative URLs to '%s'" % base_url
+
+            if base_url is None:
+                base_url = ""
+
+            url = base_url + query
+
+            json_data = urllib2.urlopen(url).read().strip()
+
+            if not json_data:
+                error = "Error reading data from '%s'" % url
+
+            return json_data, error
+
+        except urllib2.URLError as e:
+            return None, str(e)
+        except KeyboardInterrupt:
+            error = "Query cancelled by user."
+            json_data = None
+        except Exception as e:
+            raise sys.exc_info()[1], None, sys.exc_info()[2]
+
+        return json_data, error
+
+    query_runner.annotate_query = False
+    return query_runner

redash/data/worker.py

@@ -13,9 +13,81 @@ import setproctitle
 import redis
 from statsd import StatsClient
 from redash.utils import gen_query_hash
+from redash.data.query_runner import get_query_runner
 from redash import settings
 
-class Job(object):
+
+class RedisObject(object):
+    # The following should be overriden in the inheriting class:
+    fields = {}
+    conversions = {}
+    id_field = ''
+    name = ''
+
+    def __init__(self, redis_connection, **kwargs):
+        self.redis_connection = redis_connection
+        self.values = {}
+
+        if not self.fields:
+            raise ValueError("You must set the fields dictionary, before using RedisObject.")
+
+        if not self.name:
+            raise ValueError("You must set the name, before using RedisObject")
+
+        self.update(**kwargs)
+
+    def __getattr__(self, name):
+        if name in self.values:
+            return self.values[name]
+        else:
+            raise AttributeError
+
+    def update(self, **kwargs):
+        for field, default_value in self.fields.iteritems():
+            value = kwargs.get(field, self.values.get(field, default_value))
+            if callable(value):
+                value = value()
+
+            if value == 'None':
+                value = None
+
+            if field in self.conversions and value:
+                value = self.conversions[field](value)
+
+            self.values[field] = value
+
+    @classmethod
+    def _redis_key(cls, object_id):
+        return '{}:{}'.format(cls.name, object_id)
+
+    def save(self, pipe):
+        if not pipe:
+            pipe = self.redis_connection.pipeline()
+
+        pipe.sadd('{}_set'.format(self.name), self.id)
+        pipe.hmset(self._redis_key(self.id), self.values)
+        pipe.publish(self._redis_key(self.id), json.dumps(self.to_dict()))
+
+        pipe.execute()
+
+    @classmethod
+    def load(cls, redis_connection, object_id):
+        object_dict = redis_connection.hgetall(cls._redis_key(object_id))
+        obj = None
+        if object_dict:
+            obj = cls(redis_connection, **object_dict)
+
+        return obj
+
+
+def fix_unicode(string):
+    if isinstance(string, unicode):
+        return string
+
+    return string.decode('utf-8')
+
+
+class Job(RedisObject):
     HIGH_PRIORITY = 1
     LOW_PRIORITY = 2
 
@@ -24,37 +96,43 @@ class Job(object):
     DONE = 3
     FAILED = 4
 
-    def __init__(self, redis_connection, query, priority,
-                 job_id=None,
-                 wait_time=None, query_time=None,
-                 updated_at=None, status=None, error=None, query_result_id=None,
-                 process_id=0):
-        self.redis_connection = redis_connection
-        self.query = query
-        self.priority = priority
-        self.query_hash = gen_query_hash(self.query)
-        self.query_result_id = query_result_id
-        if process_id == 'None':
-            self.process_id = None
-        else:
-            self.process_id = int(process_id)
+    fields = {
+        'id': lambda: str(uuid.uuid1()),
+        'query': None,
+        'priority': None,
+        'query_hash': None,
+        'wait_time': 0,
+        'query_time': 0,
+        'error': None,
+        'updated_at': time.time,
+        'status': WAITING,
+        'process_id': None,
+        'query_result_id': None,
+        'data_source_id': None,
+        'data_source_name': None,
+        'data_source_type': None,
+        'data_source_options': None
+    }
 
-        if job_id is None:
-            self.id = str(uuid.uuid1())
-            self.new_job = True
-            self.wait_time = 0
-            self.query_time = 0
-            self.error = None
-            self.updated_at = time.time() # job_dict.get('updated_at', time.time())
-            self.status = self.WAITING # int(job_dict.get('status', self.WAITING))
-        else:
-            self.id = job_id
-            self.new_job = False
-            self.error = error
-            self.wait_time = wait_time
-            self.query_time = query_time
-            self.updated_at = updated_at
-            self.status = status
+    conversions = {
+        'query': fix_unicode,
+        'priority': int,
+        'updated_at': float,
+        'status': int,
+        'wait_time': float,
+        'query_time': float,
+        'process_id': int,
+        'query_result_id': int
+    }
+
+    name = 'job'
+
+    def __init__(self, redis_connection, query, priority, **kwargs):
+        kwargs['query'] = fix_unicode(query)
+        kwargs['priority'] = priority
+        kwargs['query_hash'] = gen_query_hash(kwargs['query'])
+        self.new_job = 'id' not in kwargs
+        super(Job, self).__init__(redis_connection, **kwargs)
 
     def to_dict(self):
         return {
@@ -67,13 +145,11 @@ class Job(object):
             'status': self.status,
             'error': self.error,
             'query_result_id': self.query_result_id,
-            'process_id': self.process_id
+            'process_id': self.process_id,
+            'data_source_name': self.data_source_name,
+            'data_source_type': self.data_source_type
         }
 
-    @staticmethod
-    def _redis_key(job_id):
-        return 'job:%s' % job_id
-
     def cancel(self):
         # TODO: Race condition:
         # it's possible that it will be picked up by worker while processing the cancel order
@@ -95,16 +171,14 @@ class Job(object):
         if self.is_finished():
             pipe.delete('query_hash_job:%s' % self.query_hash)
 
-        pipe.sadd('jobs_set', self.id)
-        pipe.hmset(self._redis_key(self.id), self.to_dict())
-        pipe.publish(self._redis_key(self.id), json.dumps(self.to_dict()))
-        pipe.execute()
+        super(Job, self).save(pipe)
 
     def processing(self, process_id):
-        self.status = self.PROCESSING
-        self.process_id = process_id
-        self.wait_time = time.time() - self.updated_at
-        self.updated_at = time.time()
+        self.update(status=self.PROCESSING,
+                    process_id=process_id,
+                    wait_time=time.time() - self.updated_at,
+                    updated_at=time.time())
+
         self.save()
 
     def is_finished(self):
@@ -112,48 +186,32 @@ class Job(object):
 
     def done(self, query_result_id, error):
         if error:
-            self.status = self.FAILED
+            new_status = self.FAILED
         else:
-            self.status = self.DONE
+            new_status = self.DONE
+
+        self.update(status=new_status,
+                    query_result_id=query_result_id,
+                    error=error,
+                    query_time=time.time() - self.updated_at,
+                    updated_at=time.time())
 
-        self.query_result_id = query_result_id
-        self.error = error
-        self.query_time = time.time() - self.updated_at
-        self.updated_at = time.time()
         self.save()
 
     def __str__(self):
         return "<Job:%s,priority:%d,status:%d>" % (self.id, self.priority, self.status)
 
-    @classmethod
-    def _load(cls, redis_connection, job_id):
-        return redis_connection.hgetall(cls._redis_key(job_id))
-
-    @classmethod
-    def load(cls, redis_connection, job_id):
-        job_dict = cls._load(redis_connection, job_id)
-        job = None
-        if job_dict:
-            job = Job(redis_connection, job_id=job_dict['id'], query=job_dict['query'].decode('utf-8'),
-                      priority=int(job_dict['priority']), updated_at=float(job_dict['updated_at']),
-                      status=int(job_dict['status']), wait_time=float(job_dict['wait_time']),
-                      query_time=float(job_dict['query_time']), error=job_dict['error'],
-                      query_result_id=job_dict['query_result_id'],
-                      process_id=job_dict['process_id'])
-
-        return job
-
 
 class Worker(threading.Thread):
-    def __init__(self, worker_id, manager, redis_connection_params, query_runner, sleep_time=0.1):
+    def __init__(self, worker_id, manager, redis_connection_params, sleep_time=0.1):
         self.manager = manager
 
         self.statsd_client = StatsClient(host=settings.STATSD_HOST, port=settings.STATSD_PORT,
                                          prefix=settings.STATSD_PREFIX)
         self.redis_connection_params = {k: v for k, v in redis_connection_params.iteritems()
                                         if k in ('host', 'db', 'password', 'port')}
+
         self.continue_working = True
-        self.query_runner = query_runner
         self.sleep_time = sleep_time
         self.child_pid = None
         self.worker_id = worker_id
@@ -238,15 +296,22 @@ class Worker(threading.Thread):
         start_time = time.time()
         self.set_title("running query %s" % job_id)
 
-        if getattr(self.query_runner, 'annotate_query', True):
+        logging.info("[%s][%s] Loading query runner (%s, %s)...", self.name, job.id,
+                     job.data_source_name, job.data_source_type)
+
+        query_runner = get_query_runner(job.data_source_type, job.data_source_options)
+
+        if getattr(query_runner, 'annotate_query', True):
             annotated_query = "/* Pid: %s, Job Id: %s, Query hash: %s, Priority: %s */ %s" % \
                               (pid, job.id, job.query_hash, job.priority, job.query)
         else:
             annotated_query = job.query
 
         # TODO: here's the part that needs to be forked, not all of the worker process...
-        with self.statsd_client.timer('worker_{}.query_runner.run_time'.format(self.worker_id)):
-            data, error = self.query_runner(annotated_query)
+        with self.statsd_client.timer('worker_{}.query_runner.{}.{}.run_time'.format(self.worker_id,
+                                                                                     job.data_source_type,
+                                                                                     job.data_source_name)):
+            data, error = query_runner(annotated_query)
 
         run_time = time.time() - start_time
         logging.info("[%s][%s] query finished... data length=%s, error=%s",
@@ -257,7 +322,8 @@ class Worker(threading.Thread):
         query_result_id = None
         if not error:
             self.set_title("storing results %s" % job_id)
-            query_result_id = self.manager.store_query_result(job.query, data, run_time,
+            query_result_id = self.manager.store_query_result(job.data_source_id,
+                                                              job.query, data, run_time,
                                                               datetime.datetime.utcnow())
 
         self.set_title("marking job as done %s" % job_id)

redash/import_export.py

@@ -1,16 +1,19 @@
+import contextlib
 import json
 from redash import models
 from flask.ext.script import Manager
 
 
 class Importer(object):
-    def __init__(self, object_mapping=None):
+    def __init__(self, object_mapping=None, data_source=None):
         if object_mapping is None:
             object_mapping = {}
         self.object_mapping = object_mapping
+        self.data_source = data_source
 
     def import_query_result(self, query_result):
         query_result = self._get_or_create(models.QueryResult, query_result['id'],
+                                           data_source=self.data_source,
                                            data=json.dumps(query_result['data']),
                                            query_hash=query_result['query_hash'],
                                            retrieved_at=query_result['retrieved_at'],
@@ -29,7 +32,8 @@ class Importer(object):
                                         query=query['query'],
                                         query_hash=query['query_hash'],
                                         description=query['description'],
-                                        latest_query_data=query_result)
+                                        latest_query_data=query_result,
+                                        data_source=self.data_source)
 
         return new_query
 
@@ -108,18 +112,47 @@ class Importer(object):
 import_manager = Manager(help="import utilities")
 export_manager = Manager(help="export utilities")
 
-@import_manager.command
-def dashboard(mapping_filename, dashboard_filename, user_id):
+
+@contextlib.contextmanager
+def importer_with_mapping_file(mapping_filename):
     with open(mapping_filename) as f:
         mapping = json.loads(f.read())
 
-    user = models.User.get_by_id(user_id)
-    with open(dashboard_filename) as f:
-        dashboard = json.loads(f.read())
-
-        importer = Importer(object_mapping=mapping)
-        importer.import_dashboard(user, dashboard)
+    importer = Importer(object_mapping=mapping, data_source=get_data_source())
+    yield importer
 
     with open(mapping_filename, 'w') as f:
         f.write(json.dumps(importer.object_mapping, indent=2))
 
+
+def get_data_source():
+    try:
+        data_source = models.DataSource.get(models.DataSource.name=="Import")
+    except models.DataSource.DoestNotExist:
+        data_source = models.DataSource.create(name="Import", type="import", options='{}')
+
+    return data_source
+
+@import_manager.command
+def query(mapping_filename, query_filename, user_id):
+    user = models.User.get_by_id(user_id)
+    with open(query_filename) as f:
+        query = json.loads(f.read())
+
+    with importer_with_mapping_file(mapping_filename) as importer:
+        imported_query = importer.import_query(user, query)
+
+        print "New query id: {}".format(imported_query.id)
+
+
+@import_manager.command
+def dashboard(mapping_filename, dashboard_filename, user_id):
+    user = models.User.get_by_id(user_id)
+    with open(dashboard_filename) as f:
+        dashboard = json.loads(f.read())
+
+    with importer_with_mapping_file(mapping_filename) as importer:
+        importer.import_dashboard(user, dashboard)
+
+
+

redash/models.py

@@ -22,16 +22,48 @@ class AnonymousUser(AnonymousUserMixin):
         return []
 
 
-class User(BaseModel, UserMixin):
-    DEFAULT_PERMISSIONS = ['create_dashboard', 'create_query', 'edit_dashboard', 'edit_query',
-                           'view_source', 'execute_query']
+class ApiUser(UserMixin):
+    def __init__(self, api_key):
+        self.id = api_key
 
+    @property
+    def permissions(self):
+        return ['view_query']
+
+
+class Group(BaseModel):
+    DEFAULT_PERMISSIONS = ['create_dashboard', 'create_query', 'edit_dashboard', 'edit_query',
+                           'view_query', 'view_source', 'execute_query']
+    
+    id = peewee.PrimaryKeyField()
+    name = peewee.CharField(max_length=100)
+    permissions = ArrayField(peewee.CharField, default=DEFAULT_PERMISSIONS)
+    tables = ArrayField(peewee.CharField)
+    created_at = peewee.DateTimeField(default=datetime.datetime.now)
+
+    class Meta:
+        db_table = 'groups'
+
+    def to_dict(self):
+        return {
+            'id': self.id,
+            'name': self.name,
+            'permissions': self.permissions,
+            'tables': self.tables,
+            'created_at': self.created_at
+        }
+
+    def __unicode__(self):
+        return unicode(self.id)
+
+
+class User(BaseModel, UserMixin):
     id = peewee.PrimaryKeyField()
     name = peewee.CharField(max_length=320)
     email = peewee.CharField(max_length=320, index=True, unique=True)
     password_hash = peewee.CharField(max_length=128, null=True)
     is_admin = peewee.BooleanField(default=False)
-    permissions = ArrayField(peewee.CharField, default=DEFAULT_PERMISSIONS)
+    groups = ArrayField(peewee.CharField, default=['default'])
 
     class Meta:
         db_table = 'users'
@@ -59,7 +91,7 @@ class ActivityLog(BaseModel):
     
     id = peewee.PrimaryKeyField()
     user = peewee.ForeignKeyField(User)
-    type = peewee.IntegerField() # 1 for query execution
+    type = peewee.IntegerField()
     activity = peewee.TextField()
     created_at = peewee.DateTimeField(default=datetime.datetime.now)
 
@@ -78,9 +110,27 @@ class ActivityLog(BaseModel):
     def __unicode__(self):
         return unicode(self.id)
 
+class DataSource(BaseModel):
+    id = peewee.PrimaryKeyField()
+    name = peewee.CharField()
+    type = peewee.CharField()
+    options = peewee.TextField()
+    created_at = peewee.DateTimeField(default=datetime.datetime.now)
+
+    class Meta:
+        db_table = 'data_sources'
+
+    def to_dict(self):
+        return {
+            'id': self.id,
+            'name': self.name,
+            'type': self.type
+        }
+
 
 class QueryResult(BaseModel):
     id = peewee.PrimaryKeyField()
+    data_source = peewee.ForeignKeyField(DataSource)
     query_hash = peewee.CharField(max_length=32, index=True)
     query = peewee.TextField()
     data = peewee.TextField()
@@ -96,16 +146,27 @@ class QueryResult(BaseModel):
             'query_hash': self.query_hash,
             'query': self.query,
             'data': json.loads(self.data),
+            'data_source_id': self._data.get('data_source', None),
             'runtime': self.runtime,
             'retrieved_at': self.retrieved_at
         }
 
+    @classmethod
+    def get_latest(cls, data_source, query, ttl=0):
+        query_hash = utils.gen_query_hash(query)
+
+        query = cls.select().where(cls.query_hash == query_hash, cls.data_source == data_source,
+                                   peewee.SQL("retrieved_at + interval '%s second' >= now() at time zone 'utc'", ttl)).order_by(cls.retrieved_at.desc())
+
+        return query.first()
+
     def __unicode__(self):
         return u"%d | %s | %s" % (self.id, self.query_hash, self.retrieved_at)
 
 
 class Query(BaseModel):
     id = peewee.PrimaryKeyField()
+    data_source = peewee.ForeignKeyField(DataSource)
     latest_query_data = peewee.ForeignKeyField(QueryResult, null=True)
     name = peewee.CharField(max_length=255)
     description = peewee.CharField(max_length=4096, null=True)
@@ -137,6 +198,7 @@ class Query(BaseModel):
             'ttl': self.ttl,
             'api_key': self.api_key,
             'created_at': self.created_at,
+            'data_source_id': self._data.get('data_source', None)
         }
 
         if with_user:
@@ -310,7 +372,7 @@ class Widget(BaseModel):
     def __unicode__(self):
         return u"%s" % self.id
 
-all_models = (User, QueryResult, Query, Dashboard, Visualization, Widget, ActivityLog)
+all_models = (DataSource, User, QueryResult, Query, Dashboard, Visualization, Widget, ActivityLog, Group)
 
 
 def create_db(create_tables, drop_tables):

redash/permissions.py

@@ -1,4 +1,6 @@
 import functools
+import itertools
+import models
 from flask.ext.login import current_user
 from flask.ext.restful import abort
 
@@ -10,8 +12,10 @@ class require_permissions(object):
     def __call__(self, fn):
         @functools.wraps(fn)
         def decorated(*args, **kwargs):
+            permissions = list(itertools.chain(*[g.permissions for g in models.Group.select().where(models.Group.name << current_user.groups)]))
+            
             has_permissions = reduce(lambda a, b: a and b,
-                                      map(lambda permission: permission in current_user.permissions,
+                                      map(lambda permission: permission in permissions,
                                           self.permissions),
                                       True)
 

redash/settings.py

@@ -46,14 +46,8 @@ STATSD_PREFIX = os.environ.get('REDASH_STATSD_PREFIX', "redash")
 
 NAME = os.environ.get('REDASH_NAME', 're:dash')
 
-# "pg", "graphite" or "mysql"
+# The following is kept for backward compatability, and shouldn't be used any more.
 CONNECTION_ADAPTER = os.environ.get("REDASH_CONNECTION_ADAPTER", "pg")
-# Connection string for the database that is used to run queries against. Examples:
-# -- mysql:    CONNECTION_STRING = "Server=;User=;Pwd=;Database="
-# -- pg:       CONNECTION_STRING = "user= password= host= port=5439 dbname="
-# -- graphite: CONNECTION_STRING = {"url": "https://graphite.yourcompany.com", "auth": ["user", "password"], "verify": true}
-# -- bigquery: CONNECTION_STRING = {"serviceAccount" : "43242343247-fjdfakljr3r2@developer.gserviceaccount.com", "privateKey" : "/somewhere/23fjkfjdsfj21312-privatekey.p12", "projectId" : "myproject-123" }
-#    to obtain bigquery credentials follow the guidelines at https://developers.google.com/bigquery/authorization#service-accounts
 CONNECTION_STRING = os.environ.get("REDASH_CONNECTION_STRING", "user= password= host= port=5439 dbname=")
 
 # Connection settings for re:dash's own database (where we store the queries, results, etc)

redash/utils.py

@@ -6,9 +6,22 @@ import datetime
 import json
 import re
 import hashlib
+import sqlparse
 
 COMMENTS_REGEX = re.compile("/\*.*?\*/")
 
+def extract_table_names(tokens, tables = set()):
+    tokens = [t for t in tokens if t.ttype not in (sqlparse.tokens.Whitespace, sqlparse.tokens.Newline)]
+    
+    for i in range(len(tokens)):
+        if tokens[i].is_group():
+            tables.update(extract_table_names(tokens[i].tokens))
+        else:
+            if tokens[i].ttype == sqlparse.tokens.Keyword \
+            and tokens[i].normalized in ['FROM', 'JOIN', 'LEFT JOIN', 'FULL JOIN', 'RIGHT JOIN', 'CROSS JOIN', 'INNER JOIN', 'OUTER JOIN', 'LEFT OUTER JOIN', 'RIGHT OUTER JOIN', 'FULL OUTER JOIN'] \
+            and isinstance(tokens[i + 1], sqlparse.sql.Identifier):
+                tables.add(tokens[i + 1].value)
+    return tables
 
 def gen_query_hash(sql):
     """Returns hash of the given query after stripping all comments, line breaks and multiple

requirements.txt

@@ -12,7 +12,7 @@ atfork==0.1.2
 blinker==1.3
 flask-peewee==0.6.5
 itsdangerous==0.23
-peewee==2.2.0
+peewee==2.2.2
 psycopg2==2.5.1
 python-dateutil==2.1
 pytz==2013.9

tests/__init__.py

@@ -1,3 +1,4 @@
+import logging
 from unittest import TestCase
 from redash import settings, db, app
 import redash.models
@@ -11,6 +12,8 @@ settings.DATABASE_CONFIG = {
 app.config['DATABASE'] = settings.DATABASE_CONFIG
 db.load_database()
 
+logging.getLogger('peewee').setLevel(logging.INFO)
+
 for model in redash.models.all_models:
     model._meta.database = db.database
 

tests/factories.py

@@ -1,5 +1,6 @@
 import datetime
 import redash.models
+from redash.utils import gen_query_hash
 
 
 class ModelFactory(object):
@@ -43,6 +44,12 @@ user_factory = ModelFactory(redash.models.User,
                             is_admin=False)
 
 
+data_source_factory = ModelFactory(redash.models.DataSource,
+                                   name='Test',
+                                   type='pg',
+                                   options='')
+
+
 dashboard_factory = ModelFactory(redash.models.Dashboard,
                                  name='test', user=user_factory.create, layout='[]')
 
@@ -52,14 +59,16 @@ query_factory = ModelFactory(redash.models.Query,
                              description='',
                              query='SELECT 1',
                              ttl=-1,
-                             user=user_factory.create)
+                             user=user_factory.create,
+                             data_source=data_source_factory.create)
 
 query_result_factory = ModelFactory(redash.models.QueryResult,
                                     data='{"columns":{}, "rows":[]}',
                                     runtime=1,
-                                    retrieved_at=datetime.datetime.now(),
-                                    query=query_factory.create,
-                                    query_hash='')
+                                    retrieved_at=datetime.datetime.utcnow,
+                                    query="SELECT 1",
+                                    query_hash=gen_query_hash('SELECT 1'),
+                                    data_source=data_source_factory.create)
 
 visualization_factory = ModelFactory(redash.models.Visualization,
                                      type='CHART',

tests/test_controllers.py

@@ -7,7 +7,7 @@ from flask.ext.login import current_user
 from mock import patch
 from tests import BaseTestCase
 from tests.factories import dashboard_factory, widget_factory, visualization_factory, query_factory, \
-    query_result_factory, user_factory
+    query_result_factory, user_factory, data_source_factory
 from redash import app, models, settings
 from redash.utils import json_dumps
 from redash.authentication import sign
@@ -211,10 +211,12 @@ class QueryAPITest(BaseTestCase, AuthenticationTestMixin):
 
     def test_create_query(self):
         user = user_factory.create()
+        data_source = data_source_factory.create()
         query_data = {
             'name': 'Testing',
             'query': 'SELECT 1',
-            'ttl': 3600
+            'ttl': 3600,
+            'data_source_id': data_source.id
         }
 
         with app.test_client() as c, authenticated_user(c, user=user):
@@ -304,12 +306,13 @@ class CsvQueryResultAPITest(BaseTestCase, AuthenticationTestMixin):
         super(CsvQueryResultAPITest, self).setUp()
         self.paths = []
         self.query_result = query_result_factory.create()
-        self.path = '/api/queries/{0}/results/{1}.csv'.format(self.query_result.query.id, self.query_result.id)
+        self.query = query_factory.create()
+        self.path = '/api/queries/{0}/results/{1}.csv'.format(self.query.id, self.query_result.id)
 
     # TODO: factor out the HMAC authentication tests
 
     def signature(self, expires):
-        return sign(self.query_result.query.api_key, self.path, expires)
+        return sign(self.query.api_key, self.path, expires)
 
     def test_redirect_when_unauthenticated(self):
         with app.test_client() as c:
@@ -318,34 +321,34 @@ class CsvQueryResultAPITest(BaseTestCase, AuthenticationTestMixin):
 
     def test_redirect_for_wrong_signature(self):
         with app.test_client() as c:
-            rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query_result.query.id, self.query_result.id), query_string={'signature': 'whatever', 'expires': 0})
+            rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query.id, self.query_result.id), query_string={'signature': 'whatever', 'expires': 0})
             self.assertEquals(rv.status_code, 302)
 
     def test_redirect_for_correct_signature_and_wrong_expires(self):
         with app.test_client() as c:
-            rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query_result.query.id, self.query_result.id), query_string={'signature': self.signature(0), 'expires': 0})
+            rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query.id, self.query_result.id), query_string={'signature': self.signature(0), 'expires': 0})
             self.assertEquals(rv.status_code, 302)
 
     def test_redirect_for_correct_signature_and_no_expires(self):
         with app.test_client() as c:
-            rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query_result.query.id, self.query_result.id), query_string={'signature': self.signature(time.time()+3600)})
+            rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query.id, self.query_result.id), query_string={'signature': self.signature(time.time()+3600)})
             self.assertEquals(rv.status_code, 302)
 
     def test_redirect_for_correct_signature_and_expires_too_long(self):
         with app.test_client() as c:
             expires = time.time()+(10*3600)
-            rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query_result.query.id, self.query_result.id), query_string={'signature': self.signature(expires), 'expires': expires})
+            rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query.id, self.query_result.id), query_string={'signature': self.signature(expires), 'expires': expires})
             self.assertEquals(rv.status_code, 302)
 
     def test_returns_200_for_correct_signature(self):
         with app.test_client() as c:
             expires = time.time()+1800
-            rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query_result.query.id, self.query_result.id), query_string={'signature': self.signature(expires), 'expires': expires})
+            rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query.id, self.query_result.id), query_string={'signature': self.signature(expires), 'expires': expires})
             self.assertEquals(rv.status_code, 200)
 
     def test_returns_200_for_authenticated_user(self):
         with app.test_client() as c, authenticated_user(c):
-            rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query_result.query.id, self.query_result.id))
+            rv = c.get('/api/queries/{0}/results/{1}.csv'.format(self.query.id, self.query_result.id))
             self.assertEquals(rv.status_code, 200)
 
 

tests/test_import.py

@@ -3,7 +3,7 @@ import os.path
 from tests import BaseTestCase
 from redash import models
 from redash import import_export
-from factories import user_factory, dashboard_factory
+from factories import user_factory, dashboard_factory, data_source_factory
 
 
 class ImportTest(BaseTestCase):
@@ -15,7 +15,7 @@ class ImportTest(BaseTestCase):
             self.user = user_factory.create()
 
     def test_imports_dashboard_correctly(self):
-        importer = import_export.Importer()
+        importer = import_export.Importer(data_source=data_source_factory.create())
         dashboard = importer.import_dashboard(self.user, self.dashboard)
 
         self.assertIsNotNone(dashboard)
@@ -31,7 +31,7 @@ class ImportTest(BaseTestCase):
         self.assertEqual(models.QueryResult.select().count(), dashboard.widgets.count()-1)
 
     def test_imports_updates_existing_models(self):
-        importer = import_export.Importer()
+        importer = import_export.Importer(data_source=data_source_factory.create())
         importer.import_dashboard(self.user, self.dashboard)
 
         self.dashboard['name'] = 'Testing #2'
@@ -47,7 +47,7 @@ class ImportTest(BaseTestCase):
             }
         }
 
-        importer = import_export.Importer(object_mapping=mapping)
+        importer = import_export.Importer(object_mapping=mapping, data_source=data_source_factory.create())
         imported_dashboard = importer.import_dashboard(self.user, self.dashboard)
 
         self.assertEqual(imported_dashboard, dashboard)

tests/test_job.py

@@ -0,0 +1,100 @@
+# coding=utf-8
+
+import time
+from unittest import TestCase
+from mock import patch
+from redash.data.worker import Job
+from redash import redis_connection
+from redash.utils import gen_query_hash
+
+
+class TestJob(TestCase):
+    def setUp(self):
+        self.priority = 1
+        self.query = "SELECT 1"
+        self.query_hash = gen_query_hash(self.query)
+
+    def test_job_creation(self):
+        now = time.time()
+        with patch('time.time', return_value=now):
+            job = Job(redis_connection, query=self.query, priority=self.priority)
+        self.assertIsNotNone(job.id)
+        self.assertTrue(job.new_job)
+        self.assertEquals(0, job.wait_time)
+        self.assertEquals(0, job.query_time)
+        self.assertEquals(None, job.process_id)
+        self.assertEquals(Job.WAITING, job.status)
+        self.assertEquals(self.priority, job.priority)
+        self.assertEquals(self.query, job.query)
+        self.assertEquals(self.query_hash, job.query_hash)
+        self.assertIsNone(job.error)
+        self.assertIsNone(job.query_result_id)
+
+    def test_job_loading(self):
+        job = Job(redis_connection, query=self.query, priority=self.priority)
+        job.save()
+
+        loaded_job = Job.load(redis_connection, job.id)
+
+        self.assertFalse(loaded_job.new_job)
+
+        self.assertEquals(loaded_job.id, job.id)
+        self.assertEquals(loaded_job.wait_time, job.wait_time)
+        self.assertEquals(loaded_job.query_time, job.query_time)
+        self.assertEquals(loaded_job.process_id, job.process_id)
+        self.assertEquals(loaded_job.status, job.status)
+        self.assertEquals(loaded_job.priority, job.priority)
+        self.assertEquals(loaded_job.query_hash, job.query_hash)
+        self.assertEquals(loaded_job.query, job.query)
+        self.assertEquals(loaded_job.error, job.error)
+        self.assertEquals(loaded_job.query_result_id, job.query_result_id)
+
+    def test_update(self):
+        job = Job(redis_connection, query=self.query, priority=self.priority)
+
+        job.update(process_id=1)
+        self.assertEquals(1, job.process_id)
+        self.assertEquals(self.query, job.query)
+        self.assertEquals(self.priority, job.priority)
+
+    def test_processing(self):
+        job = Job(redis_connection, query=self.query, priority=self.priority)
+
+        updated_at = job.updated_at
+        now = time.time()+10
+        with patch('time.time', return_value=now):
+            job.processing(10)
+
+        job = Job.load(redis_connection, job.id)
+
+        self.assertEquals(10, job.process_id)
+        self.assertEquals(Job.PROCESSING, job.status)
+        self.assertEquals(now, job.updated_at)
+        self.assertEquals(now - updated_at, job.wait_time)
+
+    def test_done(self):
+        job = Job(redis_connection, query=self.query, priority=self.priority)
+
+        updated_at = job.updated_at
+        now = time.time()+10
+        with patch('time.time', return_value=now):
+            job.done(1, None)
+
+        job = Job.load(redis_connection, job.id)
+
+        self.assertEquals(Job.DONE, job.status)
+        self.assertEquals(1, job.query_result_id)
+        self.assertEquals(now, job.updated_at)
+        self.assertEquals(now - updated_at, job.query_time)
+        self.assertIsNone(job.error)
+
+    def test_unicode_serialization(self):
+        unicode_query = u"יוניקוד"
+        job = Job(redis_connection, query=unicode_query, priority=self.priority)
+
+        self.assertEquals(job.query, unicode_query)
+
+        job.save()
+        loaded_job = Job.load(redis_connection, job.id)
+        self.assertEquals(loaded_job.query, unicode_query)
+

tests/test_manager.py

@@ -0,0 +1,149 @@
+import datetime
+from mock import patch, call
+from tests import BaseTestCase
+from redash.data import worker
+from redash import data_manager, models
+from tests.factories import query_factory, query_result_factory, data_source_factory
+from redash.utils import gen_query_hash
+
+
+class TestManagerRefresh(BaseTestCase):
+    def test_enqueues_outdated_queries(self):
+        query = query_factory.create(ttl=60)
+        retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10)
+        query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query,
+                                                   query_hash=query.query_hash)
+        query.latest_query_data = query_result
+        query.save()
+
+        with patch('redash.data.Manager.add_job') as add_job_mock:
+            data_manager.refresh_queries()
+            add_job_mock.assert_called_with(query.query, worker.Job.LOW_PRIORITY, query.data_source)
+
+    def test_skips_fresh_queries(self):
+        query = query_factory.create(ttl=1200)
+        retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10)
+        query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query,
+                                                   query_hash=query.query_hash)
+
+        with patch('redash.data.Manager.add_job') as add_job_mock:
+            data_manager.refresh_queries()
+            self.assertFalse(add_job_mock.called)
+
+    def test_skips_queries_with_no_ttl(self):
+        query = query_factory.create(ttl=-1)
+        retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10)
+        query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query,
+                                                   query_hash=query.query_hash)
+
+        with patch('redash.data.Manager.add_job') as add_job_mock:
+            data_manager.refresh_queries()
+            self.assertFalse(add_job_mock.called)
+
+    def test_enqueues_query_only_once(self):
+        query = query_factory.create(ttl=60)
+        query2 = query_factory.create(ttl=60, query=query.query, query_hash=query.query_hash,
+                                      data_source=query.data_source)
+        retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10)
+        query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query,
+                                                   query_hash=query.query_hash)
+        query.latest_query_data = query_result
+        query2.latest_query_data = query_result
+        query.save()
+        query2.save()
+
+        with patch('redash.data.Manager.add_job') as add_job_mock:
+            data_manager.refresh_queries()
+            add_job_mock.assert_called_once_with(query.query, worker.Job.LOW_PRIORITY, query.data_source)
+
+    def test_enqueues_query_with_correct_data_source(self):
+        query = query_factory.create(ttl=60)
+        query2 = query_factory.create(ttl=60, query=query.query, query_hash=query.query_hash)
+        retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10)
+        query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query,
+                                                   query_hash=query.query_hash)
+        query.latest_query_data = query_result
+        query2.latest_query_data = query_result
+        query.save()
+        query2.save()
+
+        with patch('redash.data.Manager.add_job') as add_job_mock:
+            data_manager.refresh_queries()
+            add_job_mock.assert_has_calls([call(query2.query, worker.Job.LOW_PRIORITY, query2.data_source), call(query.query, worker.Job.LOW_PRIORITY, query.data_source)], any_order=True)
+            self.assertEquals(2, add_job_mock.call_count)
+
+    def test_enqueues_only_for_relevant_data_source(self):
+        query = query_factory.create(ttl=60)
+        query2 = query_factory.create(ttl=3600, query=query.query, query_hash=query.query_hash)
+        retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10)
+        query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query,
+                                                   query_hash=query.query_hash)
+        query.latest_query_data = query_result
+        query2.latest_query_data = query_result
+        query.save()
+        query2.save()
+
+        with patch('redash.data.Manager.add_job') as add_job_mock:
+            data_manager.refresh_queries()
+            add_job_mock.assert_called_once_with(query.query, worker.Job.LOW_PRIORITY, query.data_source)
+
+
+class TestManagerStoreResults(BaseTestCase):
+    def setUp(self):
+        super(TestManagerStoreResults, self).setUp()
+        self.data_source = data_source_factory.create()
+        self.query = "SELECT 1"
+        self.query_hash = gen_query_hash(self.query)
+        self.runtime = 123
+        self.utcnow = datetime.datetime.utcnow()
+        self.data = "data"
+
+    def test_stores_the_result(self):
+        query_result_id = data_manager.store_query_result(self.data_source.id, self.query,
+                                                          self.data, self.runtime, self.utcnow)
+
+        query_result = models.QueryResult.get_by_id(query_result_id)
+
+        self.assertEqual(query_result.data, self.data)
+        self.assertEqual(query_result.runtime, self.runtime)
+        self.assertEqual(query_result.retrieved_at, self.utcnow)
+        self.assertEqual(query_result.query, self.query)
+        self.assertEqual(query_result.query_hash, self.query_hash)
+        self.assertEqual(query_result.data_source, self.data_source)
+
+    def test_updates_existing_queries(self):
+        query1 = query_factory.create(query=self.query, data_source=self.data_source)
+        query2 = query_factory.create(query=self.query, data_source=self.data_source)
+        query3 = query_factory.create(query=self.query, data_source=self.data_source)
+
+        query_result_id = data_manager.store_query_result(self.data_source.id, self.query,
+                                                          self.data, self.runtime, self.utcnow)
+
+        self.assertEqual(models.Query.get_by_id(query1.id)._data['latest_query_data'], query_result_id)
+        self.assertEqual(models.Query.get_by_id(query2.id)._data['latest_query_data'], query_result_id)
+        self.assertEqual(models.Query.get_by_id(query3.id)._data['latest_query_data'], query_result_id)
+
+    def test_doesnt_update_queries_with_different_hash(self):
+        query1 = query_factory.create(query=self.query, data_source=self.data_source)
+        query2 = query_factory.create(query=self.query, data_source=self.data_source)
+        query3 = query_factory.create(query=self.query + "123", data_source=self.data_source)
+
+        query_result_id = data_manager.store_query_result(self.data_source.id, self.query,
+                                                          self.data, self.runtime, self.utcnow)
+
+        self.assertEqual(models.Query.get_by_id(query1.id)._data['latest_query_data'], query_result_id)
+        self.assertEqual(models.Query.get_by_id(query2.id)._data['latest_query_data'], query_result_id)
+        self.assertNotEqual(models.Query.get_by_id(query3.id)._data['latest_query_data'], query_result_id)
+
+
+    def test_doesnt_update_queries_with_different_data_source(self):
+        query1 = query_factory.create(query=self.query, data_source=self.data_source)
+        query2 = query_factory.create(query=self.query, data_source=self.data_source)
+        query3 = query_factory.create(query=self.query, data_source=data_source_factory.create())
+
+        query_result_id = data_manager.store_query_result(self.data_source.id, self.query,
+                                                          self.data, self.runtime, self.utcnow)
+
+        self.assertEqual(models.Query.get_by_id(query1.id)._data['latest_query_data'], query_result_id)
+        self.assertEqual(models.Query.get_by_id(query2.id)._data['latest_query_data'], query_result_id)
+        self.assertNotEqual(models.Query.get_by_id(query3.id)._data['latest_query_data'], query_result_id)

tests/test_models.py

@@ -1,6 +1,7 @@
+import datetime
 from tests import BaseTestCase
 from redash import models
-from factories import dashboard_factory, query_factory
+from factories import dashboard_factory, query_factory, data_source_factory, query_result_factory
 
 
 class DashboardTest(BaseTestCase):
@@ -26,3 +27,57 @@ class QueryTest(BaseTestCase):
         q = models.Query.get_by_id(q.id)
 
         self.assertNotEquals(old_hash, q.query_hash)
+
+
+class QueryResultTest(BaseTestCase):
+    def setUp(self):
+        super(QueryResultTest, self).setUp()
+
+    def test_get_latest_returns_none_if_not_found(self):
+        ds = data_source_factory.create()
+        found_query_result = models.QueryResult.get_latest(ds, "SELECT 1", 60)
+        self.assertIsNone(found_query_result)
+
+    def test_get_latest_returns_when_found(self):
+        qr = query_result_factory.create()
+        found_query_result = models.QueryResult.get_latest(qr.data_source, qr.query, 60)
+
+        self.assertEqual(qr, found_query_result)
+
+    def test_get_latest_works_with_data_source_id(self):
+        qr = query_result_factory.create()
+        found_query_result = models.QueryResult.get_latest(qr.data_source.id, qr.query, 60)
+
+        self.assertEqual(qr, found_query_result)
+
+    def test_get_latest_doesnt_return_query_from_different_data_source(self):
+        qr = query_result_factory.create()
+        data_source = data_source_factory.create()
+        found_query_result = models.QueryResult.get_latest(data_source, qr.query, 60)
+
+        self.assertIsNone(found_query_result)
+
+    def test_get_latest_doesnt_return_if_ttl_expired(self):
+        yesterday = datetime.datetime.now() - datetime.timedelta(days=1)
+        qr = query_result_factory.create(retrieved_at=yesterday)
+
+        found_query_result = models.QueryResult.get_latest(qr.data_source, qr.query, ttl=60)
+
+        self.assertIsNone(found_query_result)
+
+    def test_get_latest_returns_if_ttl_not_expired(self):
+        yesterday = datetime.datetime.now() - datetime.timedelta(seconds=30)
+        qr = query_result_factory.create(retrieved_at=yesterday)
+
+        found_query_result = models.QueryResult.get_latest(qr.data_source, qr.query, ttl=120)
+
+        self.assertEqual(found_query_result, qr)
+
+    def test_get_latest_returns_the_most_recent_result(self):
+        yesterday = datetime.datetime.now() - datetime.timedelta(seconds=30)
+        old_qr = query_result_factory.create(retrieved_at=yesterday)
+        qr = query_result_factory.create()
+
+        found_query_result = models.QueryResult.get_latest(qr.data_source, qr.query, 60)
+
+        self.assertEqual(found_query_result.id, qr.id)