Merge pull request #1 from getredash/master

Pull from origin

README.md

@@ -26,7 +26,7 @@ Presto, Google Spreadsheets, Cloudera Impala, Hive and custom scripts.
 
 ## Demo
 
-![Screenshots](https://raw.github.com/getredash/redash/screenshots/screenshots.gif)
+<img src="https://cloud.githubusercontent.com/assets/71468/12611424/1faf4d6a-c4f5-11e5-89b5-31efc1155d2c.gif" width="60%"/>
 
 You can try out the demo instance: http://demo.redash.io/ (login with any Google account).
 

docs/datasources.rst

@@ -48,7 +48,7 @@ Google BigQuery
 -  **Options**:
 
    -  Project ID (mandatory)
-   -  JSON key file, generated when creating a service account (see `instructions <https://developers.google.com/console/help/new/#serviceaccounts>`__).
+   -  JSON key file, generated when creating a service account (see `instructions <https://developers.google.com/identity/protocols/OAuth2ServiceAccount#creatinganaccount>`__).
 
 
 -  **Additional requirements**:
@@ -145,7 +145,7 @@ Google Spreadsheets
 
 -  **Options**:
 
-   -  JSON key file, generated when creating a service account (see `instructions <https://developers.google.com/console/help/new/#serviceaccounts>`__).
+   -  JSON key file, generated when creating a service account (see `instructions <https://developers.google.com/identity/protocols/OAuth2ServiceAccount#creatinganaccount>`__).
 
 -  **Additional requirements**:
 

docs/misc/backup_restore.rst

@@ -48,7 +48,7 @@ How To: Backup your re:dash database and restore it on a different server
 
 .. code::
 
-    dbdrop redash
+    dropdb redash
     createdb -T template0 redash
     gunzip -c redash_backup.gz | psql redash
 

docs/setup.rst

@@ -126,8 +126,7 @@ If you're passing multiple domains, separate them with commas.
 
 6. Once you have Google OAuth enabled, you can login using your Google
    Apps account. If you want to grant admin permissions to some users,
-   you can do this by editing the user profile and enabling admin
-   permission for it.
+   you can do this by adding them to the admin group (from ``/groups`` page).
 
 7. If you don't use Google OAuth or just need username/password logins,
    you can create additional users by opening the ``/users/new`` page.

docs/usage/maintenance.rst

@@ -60,7 +60,9 @@ DB
 Backup re:dash's DB:
 --------------------
 
-``sudo -u redash pg_dump > backup_filename.sql``
+Uncompressed backup: ``sudo -u redash pg_dump > backup_filename.sql``
+
+Compressed backup: ``sudo -u redash pg_dump redash | gzip > backup_filename.gz``
 
 Version
 =======

migrations/0003_update_data_source_config.py

@@ -65,9 +65,9 @@ def update(data_source):
         print "[%s] No need to convert type of: %s" % (data_source.name, data_source.type)
 
     print "[%s] New options: %s" % (data_source.name, data_source.options)
-    data_source.save()
+    data_source.save(only=data_source.dirty_fields)
 
 
 if __name__ == '__main__':
-    for data_source in DataSource.select():
+    for data_source in DataSource.select(DataSource.id, DataSource.name, DataSource.type, DataSource.options):
         update(data_source)

migrations/0009_add_api_key_to_user.py

@@ -14,8 +14,8 @@ if __name__ == '__main__':
             migrator.add_column('users', 'api_key', models.User.api_key),
         )
 
-        for user in models.User.select():
-            user.save()
+        for user in models.User.select(models.User.id, models.User.api_key):
+            user.save(only=user.dirty_fields)
 
         migrate(
             migrator.add_not_null('users', 'api_key')

migrations/0011_migrate_bigquery_to_json.py

@@ -12,7 +12,7 @@ def convert_p12_to_pem(p12file):
 
 if __name__ == '__main__':
 
-    for ds in DataSource.select():
+    for ds in DataSource.select(DataSource.id, DataSource.type, DataSource.options):
 
         if ds.type == 'bigquery':
             options = json.loads(ds.options)
@@ -29,7 +29,7 @@ if __name__ == '__main__':
             }
 
             ds.options = json.dumps(new_options)
-            ds.save()
+            ds.save(only=ds.dirty_fields)
         elif ds.type == 'google_spreadsheets':
             options = json.loads(ds.options)
             if 'jsonKeyFile' in options:
@@ -41,4 +41,4 @@ if __name__ == '__main__':
                 }
 
             ds.options = json.dumps(new_options)
-            ds.save()
+            ds.save(only=ds.dirty_fields)

migrations/0012_add_list_users_permission.py

@@ -1,6 +1,7 @@
 from redash import models
 
 if __name__ == '__main__':
-    default_group = models.Group.get(models.Group.name=='default')
+
+    default_group = models.Group.select(models.Group.id, models.Group.permissions).where(models.Group.name=='default').first()
     default_group.permissions.append('list_users')
-    default_group.save()
+    default_group.save(only=[models.Group.permissions])

migrations/0014_add_alert_rearm_seconds.py

@@ -1,3 +1,4 @@
+import peewee
 from playhouse.migrate import PostgresqlMigrator, migrate
 
 from redash.models import db
@@ -7,8 +8,14 @@ if __name__ == '__main__':
     db.connect_db()
     migrator = PostgresqlMigrator(db.database)
 
+    cursor = db.database.execute_sql("SELECT column_name FROM information_schema.columns WHERE table_name='alerts' and column_name='rearm';")
+    if cursor.rowcount > 0:
+        print "Column exists. Skipping."
+        exit()
+
     with db.database.transaction():
         migrate(
             migrator.add_column('alerts', 'rearm', models.Alert.rearm),
         )
+
     db.close_db(None)

migrations/0014_migrate_existing_es_to_kibana.py

@@ -4,7 +4,7 @@ from redash.models import DataSource
 
 if __name__ == '__main__':
 
-    for ds in DataSource.select():
+    for ds in DataSource.select(DataSource.id, DataSource.type):
         if ds.type == 'elasticsearch':
             ds.type = 'kibana'
-            ds.save()
+            ds.save(only=ds.dirty_fields)

migrations/0015_add_schedule_query_permission.py

@@ -1,6 +1,6 @@
 from redash import models
 
 if __name__ == '__main__':
-    default_group = models.Group.get(models.Group.name=='default')
+    default_group = models.Group.select(models.Group.id, models.Group.permissions).where(models.Group.name=='default').first()
     default_group.permissions.append('schedule_query')
-    default_group.save()
+    default_group.save(only=[models.Group.permissions])

rd_ui/app/scripts/controllers/users.js

@@ -306,12 +306,6 @@
         email: $scope.user.email
       };
 
-      if ($scope.user.admin === true && $scope.user.groups.indexOf("admin") === -1) {
-        data.groups = $scope.user.groups.concat("admin");
-      } else if ($scope.user.admin === false && $scope.user.groups.indexOf("admin") !== -1) {
-        data.groups = _.without($scope.user.groups, "admin");
-      }
-
       User.save(data, function(user) {
         growl.addSuccessMessage("Saved.")
         $scope.user = user;

rd_ui/app/views/users/show.html

@@ -21,25 +21,20 @@
             API Key:
             <input type="text" value="{{user.api_key}}" size="44" readonly/>
         </tab>
-        <tab heading="Settings" ng-if="showSettings || currentUser.hasPermission('admin')" active="tabs['settings']" select="setTab('settings')">
+        <tab heading="Settings" ng-if="showSettings" active="tabs['settings']" select="setTab('settings')">
             <div class="col-md-6">
                 <form class="form" name="userSettingsForm" ng-submit="updateUser(userSettingsForm)" novalidate>
-                    <div class="form-group required" ng-if="showSettings" show-errors>
+                    <div class="form-group required" show-errors>
                         <label class="control-label">Name</label>
                         <input name="name" type="text" class="form-control" ng-model="user.name" required/>
                         <input-errors errors="userSettingsForm.name.$error"/>
                     </div>
-                    <div class="form-group required" ng-if="showSettings" show-errors>
+                    <div class="form-group required" show-errors>
                         <label class="control-label">Email</label>
                         <input name="email" type="email" class="form-control" ng-model="user.email" required/>
                         <input-errors errors="userSettingsForm.email.$error"/>
                     </div>
-                    <div class="checkbox" ng-if="currentUser.hasPermission('admin')">
-                        <label>
-                            <input type="checkbox" ng-model="user.admin"> Admin
-                        </label>
-                    </div>
-                    <div class="form-gruup">
+                    <div class="form-group">
                         <button class="btn btn-primary">Save</button>
                     </div>
                 </form>

redash/cli/users.py

@@ -7,12 +7,17 @@ manager = Manager(help="Users management commands. This commands assume single o
 @manager.option('email', help="email address of the user to grant admin to")
 def grant_admin(email):
     try:
-        user = models.User.get_by_email_and_org(email, models.Organization.get_by_slug('default'))
+        org = models.Organization.get_by_slug('default')
+        admin_group = org.admin_group
+        user = models.User.get_by_email_and_org(email, org)
 
-        user.groups.append('admin')
-        user.save()
+        if admin_group.id in user.groups:
+            print "User is already an admin."
+        else:
+            user.groups.append(org.admin_group.id)
+            user.save()
 
-        print "User updated."
+            print "User updated."
     except models.User.DoesNotExist:
         print "User [%s] not found." % email
 

redash/query_runner/google_spreadsheets.py

@@ -22,6 +22,8 @@ def _load_key(filename):
 
 
 def _guess_type(value):
+    if value == '':
+        return TYPE_STRING
     try:
         val = int(value)
         return TYPE_INTEGER
@@ -45,6 +47,10 @@ def _guess_type(value):
 def _value_eval_list(value):
     value_list = []
     for member in value:
+        if member == '' or member == None:
+            val = None
+            value_list.append(val)
+            continue
         try:
             val = int(member)
             value_list.append(val)

redash/tasks.py

@@ -252,23 +252,30 @@ class QueryExecutionError(Exception):
     pass
 
 
+# TODO: convert this into a class, to simplify and avoid code duplication for logging
+# class ExecuteQueryTask(BaseTask):
+#     def run(self, ...):
+#         # logic
 @celery.task(bind=True, base=BaseTask, track_started=True, throws=(QueryExecutionError,))
 def execute_query(self, query, data_source_id, metadata):
     signal.signal(signal.SIGINT, signal_handler)
     start_time = time.time()
 
-    logger.info("Loading data source (%d)...", data_source_id)
+    logger.info("task=execute_query state=load_ds ds_id=%d", data_source_id)
 
-    # TODO: we should probably cache data sources in Redis
     data_source = models.DataSource.get_by_id(data_source_id)
 
     self.update_state(state='STARTED', meta={'start_time': start_time, 'custom_message': ''})
 
-    logger.info("Executing query:\n%s", query)
+    logger.debug("Executing query:\n%s", query)
 
     query_hash = gen_query_hash(query)
     query_runner = get_query_runner(data_source.type, data_source.options)
 
+    logger.info("task=execute_query state=before query_hash=%s type=%s ds_id=%d task_id=%s queue=%s query_id=%s username=%s",
+                query_hash, data_source.type, data_source.id, self.request.id, self.request.delivery_info['routing_key'],
+                metadata.get('Query ID', 'unknown'), metadata.get('Username', 'unknown'))
+
     if query_runner.annotate_query():
         metadata['Task ID'] = self.request.id
         metadata['Query Hash'] = query_hash
@@ -285,6 +292,10 @@ def execute_query(self, query, data_source_id, metadata):
     with statsd_client.timer('query_runner.{}.{}.run_time'.format(data_source.type, data_source.name)):
         data, error = query_runner.run_query(annotated_query)
 
+    logger.info("task=execute_query state=after query_hash=%s type=%s ds_id=%d task_id=%s queue=%s query_id=%s username=%s",
+                query_hash, data_source.type, data_source.id, self.request.id, self.request.delivery_info['routing_key'],
+                metadata.get('Query ID', 'unknown'), metadata.get('Username', 'unknown'))
+
     run_time = time.time() - start_time
     logger.info("Query finished... data length=%s, error=%s", data and len(data), error)
 
@@ -295,8 +306,14 @@ def execute_query(self, query, data_source_id, metadata):
 
     if not error:
         query_result, updated_query_ids = models.QueryResult.store_result(data_source.org_id, data_source.id, query_hash, query, data, run_time, utils.utcnow())
+        logger.info("task=execute_query state=after_store query_hash=%s type=%s ds_id=%d task_id=%s queue=%s query_id=%s username=%s",
+                    query_hash, data_source.type, data_source.id, self.request.id, self.request.delivery_info['routing_key'],
+                    metadata.get('Query ID', 'unknown'), metadata.get('Username', 'unknown'))
         for query_id in updated_query_ids:
             check_alerts_for_query.delay(query_id)
+        logger.info("task=execute_query state=after_alerts query_hash=%s type=%s ds_id=%d task_id=%s queue=%s query_id=%s username=%s",
+                    query_hash, data_source.type, data_source.id, self.request.id, self.request.delivery_info['routing_key'],
+                    metadata.get('Query ID', 'unknown'), metadata.get('Username', 'unknown'))
     else:
         raise QueryExecutionError(error)
 

redash/utils.py

@@ -53,9 +53,12 @@ class JSONEncoder(json.JSONEncoder):
         if isinstance(o, decimal.Decimal):
             return float(o)
 
-        if isinstance(o, (datetime.date, datetime.time, datetime.timedelta)):
+        if isinstance(o, (datetime.date, datetime.time)):
             return o.isoformat()
 
+        if isinstance(o, datetime.timedelta):
+            return str(o)
+
         super(JSONEncoder, self).default(o)
 
 

setup/amazon_linux/bootstrap.sh

@@ -164,7 +164,7 @@ sudo -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='redas
 if [ $pg_user_exists -ne 0 ]; then
     echo "Creating redash reader postgres user."
 
-    sudo yum install expect
+    sudo yum install -y expect
 
     REDASH_READER_PASSWORD=$(mkpasswd)
     sudo -u postgres psql -c "CREATE ROLE redash_reader WITH PASSWORD '$REDASH_READER_PASSWORD' NOCREATEROLE NOCREATEDB NOSUPERUSER LOGIN"
@@ -190,9 +190,23 @@ wget -O /etc/init.d/redash_supervisord $FILES_BASE_URL"redash_supervisord_init"
 add_service "redash_supervisord"
 
 # Nginx setup
-if [-x $(mkdir /etc/nginx/sites-available)]; then
+if [ -d "/etc/nginx/conf.d" ]; then
+  echo "/etc/nginx/conf.d exists."
+  wget -O /etc/nginx/conf.d/redash.conf $FILES_BASE_URL"nginx_redash_site"
+elif [ -d "/etc/nginx/sites-available" ]; then
   echo "/etc/nginx/sites-available exists."
+  wget -O /etc/nginx/sites-available/redash $FILES_BASE_URL"nginx_redash_site"
+  ln -nfs /etc/nginx/sites-available/redash /etc/nginx/conf.d/redash.conf
+else
+  mkdir /etc/nginx/sites-available
+  if [ $? -ne 0 ] ; then
+    echo "create /etc/nginx/sites-available ok"
+    wget -O /etc/nginx/sites-available/redash $FILES_BASE_URL"nginx_redash_site"
+    ln -nfs /etc/nginx/sites-available/redash /etc/nginx/conf.d/redash.conf
+  else
+    echo "ERROR: create /etc/nginx/sites-available failed"
+    exit
+  fi  
 fi
-wget -O /etc/nginx/sites-available/redash $FILES_BASE_URL"nginx_redash_site"
-ln -nfs /etc/nginx/sites-available/redash /etc/nginx/conf.d/redash.conf
+
 service nginx restart