From d88d1a1e61539e1eee17611ce6f46ddd684cdc3b Mon Sep 17 00:00:00 2001 From: Alexander Leibzon Date: Thu, 17 Mar 2016 03:13:44 +0200 Subject: [PATCH 1/6] dynamodb SQL connector --- redash/settings.py | 1 + redash/utils/__init__.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/redash/settings.py b/redash/settings.py index 60717819..08bca289 100644 --- a/redash/settings.py +++ b/redash/settings.py @@ -144,6 +144,7 @@ default_query_runners = [ 'redash.query_runner.treasuredata', 'redash.query_runner.oracle', 'redash.query_runner.sqlite', + 'redash.query_runner.dynamodb_sql', 'redash.query_runner.mssql', ] diff --git a/redash/utils/__init__.py b/redash/utils/__init__.py index f0cf9613..ddee0f38 100644 --- a/redash/utils/__init__.py +++ b/redash/utils/__init__.py @@ -63,6 +63,9 @@ class JSONEncoder(json.JSONEncoder): if isinstance(o, datetime.timedelta): return str(o) + if isinstance(o, unicode): + return str(o) + super(JSONEncoder, self).default(o) From 158fc6222db715c777331f5723b95fbbc5877b3b Mon Sep 17 00:00:00 2001 From: Alexander Leibzon Date: Thu, 17 Mar 2016 12:50:39 +0200 Subject: [PATCH 2/6] add requirements --- redash/query_runner/hive_ds.py | 2 +- requirements.txt | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/redash/query_runner/hive_ds.py b/redash/query_runner/hive_ds.py index b78e3d85..e167f6f5 100644 --- a/redash/query_runner/hive_ds.py +++ b/redash/query_runner/hive_ds.py @@ -67,7 +67,7 @@ class Hive(BaseSQLQueryRunner): def __init__(self, configuration): super(Hive, self).__init__(configuration) - def _get_tables(self, schema_dict): + def _get_tables(self, schema): try: schemas_query = "show schemas" diff --git a/requirements.txt b/requirements.txt index b30ea1c3..5793313c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,10 @@ python-dateutil==2.4.2 pytz==2015.7 redis==2.7.5 requests==2.3.0 -six==1.5.2 +botocore==1.4.4 +six==1.10.0 +dql==0.5.16 +dynamo3==0.4.7 sqlparse==0.1.8 wsgiref==0.1.2 Flask-Script==0.6.6 From 629c16acbe65cb8c737da3393dec4701a79bb033 Mon Sep 17 00:00:00 2001 From: Alexander Leibzon Date: Thu, 17 Mar 2016 12:51:06 +0200 Subject: [PATCH 3/6] the dynamoDB SQL connector --- redash/query_runner/dynamodb_sql.py | 129 ++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 redash/query_runner/dynamodb_sql.py diff --git a/redash/query_runner/dynamodb_sql.py b/redash/query_runner/dynamodb_sql.py new file mode 100644 index 00000000..edbd2aa8 --- /dev/null +++ b/redash/query_runner/dynamodb_sql.py @@ -0,0 +1,129 @@ +import json +import logging +import sys + +from redash.query_runner import * +from redash.utils import JSONEncoder + +logger = logging.getLogger(__name__) + +try: + from dql import Engine, FragmentEngine + enabled = True +except ImportError, e: + enabled = False + +types_map = { + 'UNICODE': TYPE_INTEGER, + 'TINYINT': TYPE_INTEGER, + 'SMALLINT': TYPE_INTEGER, + 'INT': TYPE_INTEGER, + 'DOUBLE': TYPE_FLOAT, + 'DECIMAL': TYPE_FLOAT, + 'FLOAT': TYPE_FLOAT, + 'REAL': TYPE_FLOAT, + 'BOOLEAN': TYPE_BOOLEAN, + 'TIMESTAMP': TYPE_DATETIME, + 'DATE': TYPE_DATETIME, + 'CHAR': TYPE_STRING, + 'STRING': TYPE_STRING, + 'VARCHAR': TYPE_STRING +} + + +class DynamoDBSQL(BaseSQLQueryRunner): + @classmethod + def configuration_schema(cls): + return { + "type": "object", + "properties": { + "region": { + "type": "string", + "default": "us-west-1" + }, + "host": { + "type": "string", + "default": "127.0.0.1" + }, + "port": { + "type": "number", + "default": 8000 + }, + "access_key": { + "type": "string", + "default": "anything" + + }, + "secret_key": { + "type": "string", + "default": "anything" + + }, + "is_secure": { + "type": "boolean", + "default": False, + } + }, + "required": ["host"] + } + + @classmethod + def annotate_query(cls): + return False + + @classmethod + def type(cls): + return "ddb_sql" + + def __init__(self, configuration): + super(DynamoDBSQL, self).__init__(configuration) + + def _get_tables(self, schema): + + try: + engine = FragmentEngine() + engine.connect(**self.configuration.to_dict()) + + for table in engine.describe_all(): + schema[table.name] = {'name': table.name, 'columns': table.attrs.keys()} + + except Exception as e: + logging.exception(e) + raise sys.exc_info()[1], None, sys.exc_info()[2] + + def run_query(self, query): + + connection = None + try: + engine = FragmentEngine() + connection = engine.connect(**self.configuration.to_dict()) + + resDict = engine.execute(query if str(query).endswith(';') else str(query)+';') + + columns = [] + rows = [] + for item in resDict: + + if not columns: + for k, v in item.iteritems(): + columns.append({ + 'name': k, + 'friendly_name': k, + 'type': types_map.get(str(type(v)).upper(), None) + }) + rows.append(item) + + data = {'columns': columns, 'rows': rows} + json_data = json.dumps(data, cls=JSONEncoder) + error = None + except KeyboardInterrupt: + connection.cancel() + error = "Query cancelled by user." + json_data = None + except Exception as e: + logging.exception(e) + raise sys.exc_info()[1], None, sys.exc_info()[2] + + return json_data, error + +register(DynamoDBSQL) From 2e93af3a215ce53bad071de875fda4251aa49283 Mon Sep 17 00:00:00 2001 From: Alexander Leibzon Date: Thu, 17 Mar 2016 13:09:02 +0200 Subject: [PATCH 4/6] fixes --- redash/query_runner/dynamodb_sql.py | 3 ++- redash/utils/__init__.py | 3 --- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/redash/query_runner/dynamodb_sql.py b/redash/query_runner/dynamodb_sql.py index edbd2aa8..95e688d0 100644 --- a/redash/query_runner/dynamodb_sql.py +++ b/redash/query_runner/dynamodb_sql.py @@ -64,7 +64,8 @@ class DynamoDBSQL(BaseSQLQueryRunner): "default": False, } }, - "required": ["host"] + "required": ["host"], + "secret": ["secret_key"] } @classmethod diff --git a/redash/utils/__init__.py b/redash/utils/__init__.py index aaea67de..32dfe715 100644 --- a/redash/utils/__init__.py +++ b/redash/utils/__init__.py @@ -66,9 +66,6 @@ class JSONEncoder(json.JSONEncoder): if isinstance(o, datetime.timedelta): return str(o) - if isinstance(o, unicode): - return str(o) - super(JSONEncoder, self).default(o) From ad95a5eb5e8351b9aa6dc1667ad510e7d912b2b7 Mon Sep 17 00:00:00 2001 From: Alexander Leibzon Date: Thu, 17 Mar 2016 13:10:15 +0200 Subject: [PATCH 5/6] requirements rearrangement --- requirements.txt | 2 -- requirements_all_ds.txt | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 68fa91e5..c2d440e7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,8 +21,6 @@ redis==2.7.5 requests==2.3.0 botocore==1.4.4 six==1.10.0 -dql==0.5.16 -dynamo3==0.4.7 sqlparse==0.1.8 wsgiref==0.1.2 Flask-Script==0.6.6 diff --git a/requirements_all_ds.txt b/requirements_all_ds.txt index 977d6d5d..8ed14a58 100644 --- a/requirements_all_ds.txt +++ b/requirements_all_ds.txt @@ -10,3 +10,5 @@ pyOpenSSL==0.14 vertica-python==0.5.1 td-client==0.4.1 pymssql==2.1.1 +dql==0.5.16 +dynamo3==0.4.7 From 7bb80e375a540655bb51732425bcfca38a56abae Mon Sep 17 00:00:00 2001 From: Alexander Leibzon Date: Wed, 23 Mar 2016 11:44:57 +0200 Subject: [PATCH 6/6] code style, name method --- redash/query_runner/dynamodb_sql.py | 10 +++++++--- requirements.txt | 1 - requirements_all_ds.txt | 1 + 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/redash/query_runner/dynamodb_sql.py b/redash/query_runner/dynamodb_sql.py index 95e688d0..037eda0b 100644 --- a/redash/query_runner/dynamodb_sql.py +++ b/redash/query_runner/dynamodb_sql.py @@ -74,7 +74,11 @@ class DynamoDBSQL(BaseSQLQueryRunner): @classmethod def type(cls): - return "ddb_sql" + return "dynamodb_sql" + + @classmethod + def name(cls): + return "DynamoDB (with DQL)" def __init__(self, configuration): super(DynamoDBSQL, self).__init__(configuration) @@ -99,11 +103,11 @@ class DynamoDBSQL(BaseSQLQueryRunner): engine = FragmentEngine() connection = engine.connect(**self.configuration.to_dict()) - resDict = engine.execute(query if str(query).endswith(';') else str(query)+';') + res_dict = engine.execute(query if str(query).endswith(';') else str(query)+';') columns = [] rows = [] - for item in resDict: + for item in res_dict: if not columns: for k, v in item.iteritems(): diff --git a/requirements.txt b/requirements.txt index c2d440e7..0826601e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,6 @@ python-dateutil==2.4.2 pytz==2015.7 redis==2.7.5 requests==2.3.0 -botocore==1.4.4 six==1.10.0 sqlparse==0.1.8 wsgiref==0.1.2 diff --git a/requirements_all_ds.txt b/requirements_all_ds.txt index 8ed14a58..62ea2975 100644 --- a/requirements_all_ds.txt +++ b/requirements_all_ds.txt @@ -12,3 +12,4 @@ td-client==0.4.1 pymssql==2.1.1 dql==0.5.16 dynamo3==0.4.7 +botocore==1.4.4