Merge pull request #934 from alexanderlz/master

Feature: DynamoDB DQL query runner

redash/query_runner/dynamodb_sql.py

@@ -0,0 +1,134 @@
+import json
+import logging
+import sys
+
+from redash.query_runner import *
+from redash.utils import JSONEncoder
+
+logger = logging.getLogger(__name__)
+
+try:
+    from dql import Engine, FragmentEngine
+    enabled = True
+except ImportError, e:
+    enabled = False
+
+types_map = {
+    'UNICODE': TYPE_INTEGER,
+    'TINYINT': TYPE_INTEGER,
+    'SMALLINT': TYPE_INTEGER,
+    'INT': TYPE_INTEGER,
+    'DOUBLE': TYPE_FLOAT,
+    'DECIMAL': TYPE_FLOAT,
+    'FLOAT': TYPE_FLOAT,
+    'REAL': TYPE_FLOAT,
+    'BOOLEAN': TYPE_BOOLEAN,
+    'TIMESTAMP': TYPE_DATETIME,
+    'DATE': TYPE_DATETIME,
+    'CHAR': TYPE_STRING,
+    'STRING': TYPE_STRING,
+    'VARCHAR': TYPE_STRING
+}
+
+
+class DynamoDBSQL(BaseSQLQueryRunner):
+    @classmethod
+    def configuration_schema(cls):
+        return {
+            "type": "object",
+            "properties": {
+                "region": {
+                    "type": "string",
+                    "default": "us-west-1"
+                },
+                "host": {
+                    "type": "string",
+                    "default": "127.0.0.1"
+                },
+                "port": {
+                    "type": "number",
+                    "default": 8000
+                },
+                "access_key": {
+                    "type": "string",
+                    "default": "anything"
+
+                },
+                "secret_key": {
+                    "type": "string",
+                    "default": "anything"
+
+                },
+                "is_secure": {
+                    "type": "boolean",
+                    "default": False,
+                }
+            },
+            "required": ["host"],
+            "secret": ["secret_key"]
+        }
+
+    @classmethod
+    def annotate_query(cls):
+        return False
+
+    @classmethod
+    def type(cls):
+        return "dynamodb_sql"
+
+    @classmethod
+    def name(cls):
+        return "DynamoDB (with DQL)"
+
+    def __init__(self, configuration):
+        super(DynamoDBSQL, self).__init__(configuration)
+
+    def _get_tables(self, schema):
+
+        try:
+            engine = FragmentEngine()
+            engine.connect(**self.configuration.to_dict())
+
+            for table in engine.describe_all():
+                schema[table.name] = {'name': table.name, 'columns': table.attrs.keys()}
+
+        except Exception as e:
+            logging.exception(e)
+            raise sys.exc_info()[1], None, sys.exc_info()[2]
+
+    def run_query(self, query):
+
+        connection = None
+        try:
+            engine = FragmentEngine()
+            connection = engine.connect(**self.configuration.to_dict())
+
+            res_dict = engine.execute(query if str(query).endswith(';') else str(query)+';')
+
+            columns = []
+            rows = []
+            for item in res_dict:
+
+                if not columns:
+                    for k, v in item.iteritems():
+                        columns.append({
+                            'name': k,
+                            'friendly_name': k,
+                            'type': types_map.get(str(type(v)).upper(), None)
+                        })
+                rows.append(item)
+
+            data = {'columns': columns, 'rows': rows}
+            json_data = json.dumps(data, cls=JSONEncoder)
+            error = None
+        except KeyboardInterrupt:
+            connection.cancel()
+            error = "Query cancelled by user."
+            json_data = None
+        except Exception as e:
+            logging.exception(e)
+            raise sys.exc_info()[1], None, sys.exc_info()[2]
+
+        return json_data, error
+
+register(DynamoDBSQL)

redash/query_runner/hive_ds.py

@@ -67,7 +67,7 @@ class Hive(BaseSQLQueryRunner):
     def __init__(self, configuration):
         super(Hive, self).__init__(configuration)
 
-    def _get_tables(self, schema_dict):
+    def _get_tables(self, schema):
         try:
             schemas_query = "show schemas"
 

redash/settings.py

@@ -170,6 +170,7 @@ default_query_runners = [
     'redash.query_runner.vertica',
     'redash.query_runner.treasuredata',
     'redash.query_runner.sqlite',
+    'redash.query_runner.dynamodb_sql',
     'redash.query_runner.mssql',
 ]
 

requirements.txt

@@ -19,7 +19,7 @@ python-dateutil==2.4.2
 pytz==2015.7
 redis==2.10.5
 requests==2.3.0
-six==1.5.2
+six==1.10.0
 sqlparse==0.1.8
 wsgiref==0.1.2
 Flask-Script==0.6.6

requirements_all_ds.txt

@@ -10,3 +10,6 @@ pyOpenSSL==0.14
 vertica-python==0.5.1
 td-client==0.4.1
 pymssql==2.1.1
+dql==0.5.16
+dynamo3==0.4.7
+botocore==1.4.4