valitydev/redash
14
0
Fork 0
mirror of https://github.com/valitydev/redash.git synced 2024-11-07 09:28:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #839 from getredash/feature/api_params

Browse Source 
Feature: add API to trigger query refresh and support for parameters.
This commit is contained in:
Arik Fraimovich 2016-02-22 10:43:42 +02:00
commit 3a5d59cf69
9 changed files with 279 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
redash/handlers/queries.py
View File

@ -6,10 +6,12 @@ import sqlparse
from funcy import distinct, take
from itertools import chain
from redash.handlers.query_results import run_query
from redash import models
from redash.wsgi import app, api
from redash.permissions import require_permission, require_access, require_admin_or_owner, not_view_only, view_only
from redash.handlers.base import BaseResource, get_object_or_404
from redash.utils import collect_parameters_from_request
@app.route('/api/queries/format', methods=['POST'])
@ -105,7 +107,18 @@ class QueryAPI(BaseResource):
query.archive()
class QueryRefreshResource(BaseResource):
def post(self, query_id):
query = get_object_or_404(models.Query.get_by_id_and_org, query_id, self.current_org)
require_access(query.groups, self.current_user, not_view_only)
parameter_values = collect_parameters_from_request(request.args)
return run_query(query.data_source, parameter_values, query.query, query.id)
api.add_org_resource(QuerySearchAPI, '/api/queries/search', endpoint='queries_search')
api.add_org_resource(QueryRecentAPI, '/api/queries/recent', endpoint='recent_queries')
api.add_org_resource(QueryListAPI, '/api/queries', endpoint='queries')
api.add_org_resource(QueryRefreshResource, '/api/queries/<query_id>/refresh', endpoint='query_refresh')
api.add_org_resource(QueryAPI, '/api/queries/<query_id>', endpoint='query')

53
redash/handlers/query_results.py
View File

@ -3,7 +3,9 @@ import json
import cStringIO
import time
import pystache
from flask import make_response, request
from flask.ext.login import current_user
from flask.ext.restful import abort
import xlsxwriter
from redash import models, settings, utils
@ -11,12 +13,42 @@ from redash.wsgi import api
from redash.tasks import QueryTask, record_event
from redash.permissions import require_permission, not_view_only, has_access
from redash.handlers.base import BaseResource, get_object_or_404
from redash.utils import collect_query_parameters, collect_parameters_from_request
def run_query(data_source, parameter_values, query_text, query_id, max_age=0):
query_parameters = set(collect_query_parameters(query_text))
missing_params = set(query_parameters) - set(parameter_values.keys())
if missing_params:
return {'job': {'status': 4,
'error': 'Missing parameter value for: {}'.format(", ".join(missing_params))}}, 400
if query_parameters:
query_text = pystache.render(query_text, parameter_values)
if max_age == 0:
query_result = None
else:
query_result = models.QueryResult.get_latest(data_source, query_text, max_age)
if query_result:
return {'query_result': query_result.to_dict()}
else:
job = QueryTask.add_task(query_text, data_source,
metadata={"Username": current_user.name, "Query ID": query_id})
return {'job': job.to_dict()}
class QueryResultListAPI(BaseResource):
@require_permission('execute_query')
def post(self):
params = request.get_json(force=True)
parameter_values = collect_parameters_from_request(request.args)
query = params['query']
max_age = int(params.get('max_age', -1))
query_id = params.get('query_id', 'adhoc')
data_source = models.DataSource.get_by_id_and_org(params.get('data_source_id'), self.current_org)
if not has_access(data_source.groups, self.current_user, not_view_only):
@ -27,23 +59,10 @@ class QueryResultListAPI(BaseResource):
'timestamp': int(time.time()),
'object_id': data_source.id,
'object_type': 'data_source',
'query': params['query']
'query': query
})
max_age = int(params.get('max_age', -1))
if max_age == 0:
query_result = None
else:
query_result = models.QueryResult.get_latest(data_source, params['query'], max_age)
if query_result:
return {'query_result': query_result.to_dict()}
else:
query_id = params.get('query_id', 'adhoc')
job = QueryTask.add_task(params['query'], data_source,
metadata={"Username": self.current_user.name, "Query ID": query_id})
return {'job': job.to_dict()}
return run_query(data_source, parameter_values, query, query_id, max_age)
ONE_YEAR = 60 * 60 * 24 * 365.25
@ -74,6 +93,10 @@ class QueryResultAPI(BaseResource):
@require_permission('view_query')
def get(self, query_id=None, query_result_id=None, filetype='json'):
# TODO:
# This method handles two cases: retrieving result by id & retrieving result by query id.
# They need to be split, as they have different logic (for example, retrieving by query id
# should check for query parameters and shouldn't cache the result).
should_cache = query_result_id is not None
if query_result_id is None and query_id is not None:
query = get_object_or_404(models.Query.get_by_id_and_org, query_id, self.current_org)

31
redash/utils.py
View File

@ -8,6 +8,9 @@ import random
import re
import hashlib
import pytz
import pystache
from funcy import distinct
COMMENTS_REGEX = re.compile("/\*.*?\*/")
@ -109,3 +112,31 @@ class UnicodeWriter:
def writerows(self, rows):
for row in rows:
self.writerow(row)
def _collect_key_names(nodes):
keys = []
for node in nodes._parse_tree:
if isinstance(node, pystache.parser._EscapeNode):
keys.append(node.key)
elif isinstance(node, pystache.parser._SectionNode):
keys.append(node.key)
keys.extend(_collect_key_names(node.parsed))
return distinct(keys)
def collect_query_parameters(query):
nodes = pystache.parse(query)
keys = _collect_key_names(nodes)
return keys
def collect_parameters_from_request(args):
parameters = {}
for k, v in args.iteritems():
if k.startswith('p_'):
parameters[k[2:]] = v
return parameters

3
requirements.txt
View File

@ -37,4 +37,5 @@ funcy==1.5
raven==5.9.2
semver==2.2.1
python-simple-hipchat==0.4.0
xlsxwriter==0.8.4
xlsxwriter==0.8.4
pystache==0.5.4

1
tests/__init__.py
View File

@ -51,7 +51,6 @@ class BaseTestCase(TestCase):
return make_request(method, path, user, data, is_json)
def assertResponseEqual(self, expected, actual):
for k, v in expected.iteritems():
if isinstance(v, datetime.datetime) or isinstance(actual[k], datetime.datetime):

110
tests/handlers/test_queries.py Normal file
View File

@ -0,0 +1,110 @@
from redash import models
from tests import BaseTestCase
from tests.test_handlers import AuthenticationTestMixin
class QueryAPITest(BaseTestCase, AuthenticationTestMixin):
def setUp(self):
self.paths = ['/api/queries']
super(QueryAPITest, self).setUp()
def test_update_query(self):
admin = self.factory.create_admin()
query = self.factory.create_query()
rv = self.make_request('post', '/api/queries/{0}'.format(query.id), data={'name': 'Testing'}, user=admin)
self.assertEqual(rv.status_code, 200)
self.assertEqual(rv.json['name'], 'Testing')
self.assertEqual(rv.json['last_modified_by']['id'], admin.id)
def test_create_query(self):
query_data = {
'name': 'Testing',
'query': 'SELECT 1',
'schedule': "3600",
'data_source_id': self.factory.data_source.id
}
rv = self.make_request('post', '/api/queries', data=query_data)
self.assertEquals(rv.status_code, 200)
self.assertDictContainsSubset(query_data, rv.json)
self.assertEquals(rv.json['user']['id'], self.factory.user.id)
self.assertIsNotNone(rv.json['api_key'])
self.assertIsNotNone(rv.json['query_hash'])
query = models.Query.get_by_id(rv.json['id'])
self.assertEquals(len(list(query.visualizations)), 1)
def test_get_query(self):
query = self.factory.create_query()
rv = self.make_request('get', '/api/queries/{0}'.format(query.id))
self.assertEquals(rv.status_code, 200)
self.assertResponseEqual(rv.json, query.to_dict(with_visualizations=True))
def test_get_all_queries(self):
queries = [self.factory.create_query() for _ in range(10)]
rv = self.make_request('get', '/api/queries')
self.assertEquals(rv.status_code, 200)
self.assertEquals(len(rv.json), 10)
def test_query_without_data_source_should_be_available_only_by_admin(self):
query = self.factory.create_query()
query.data_source = None
query.save()
rv = self.make_request('get', '/api/queries/{}'.format(query.id))
self.assertEquals(rv.status_code, 403)
rv = self.make_request('get', '/api/queries/{}'.format(query.id), user=self.factory.create_admin())
self.assertEquals(rv.status_code, 200)
def test_query_only_accessible_to_users_from_its_organization(self):
second_org = self.factory.create_org()
second_org_admin = self.factory.create_admin(org=second_org)
query = self.factory.create_query()
query.data_source = None
query.save()
rv = self.make_request('get', '/api/queries/{}'.format(query.id), user=second_org_admin)
self.assertEquals(rv.status_code, 404)
rv = self.make_request('get', '/api/queries/{}'.format(query.id), user=self.factory.create_admin())
self.assertEquals(rv.status_code, 200)
class QueryRefreshTest(BaseTestCase):
def setUp(self):
super(QueryRefreshTest, self).setUp()
self.query = self.factory.create_query()
self.path = '/api/queries/{}/refresh'.format(self.query.id)
def test_refresh_regular_query(self):
response = self.make_request('post', self.path)
self.assertEqual(200, response.status_code)
def test_refresh_of_query_with_parameters(self):
self.query.query = "SELECT {{param}}"
self.query.save()
response = self.make_request('post', "{}?p_param=1".format(self.path))
self.assertEqual(200, response.status_code)
def test_refresh_of_query_with_parameters_without_parameters(self):
self.query.query = "SELECT {{param}}"
self.query.save()
response = self.make_request('post', "{}".format(self.path))
self.assertEqual(400, response.status_code)
def test_refresh_query_you_dont_have_access_to(self):
group = self.factory.create_group()
user = self.factory.create_user(groups=[group.id])
response = self.make_request('post', self.path, user=user)
self.assertEqual(403, response.status_code)

56
tests/handlers/test_query_results.py
View File

@ -16,3 +16,59 @@ class TestQueryResultsCacheHeaders(BaseTestCase):
rv = self.make_request('get', '/api/queries/{}/results.json'.format(query.id))
self.assertNotIn('Cache-Control', rv.headers)
class QueryResultListAPITest(BaseTestCase):
def test_get_existing_result(self):
query_result = self.factory.create_query_result()
query = self.factory.create_query()
rv = self.make_request('post', '/api/query_results',
data={'data_source_id': self.factory.data_source.id,
'query': query.query})
self.assertEquals(rv.status_code, 200)
self.assertEquals(query_result.id, rv.json['query_result']['id'])
def test_execute_new_query(self):
query_result = self.factory.create_query_result()
query = self.factory.create_query()
rv = self.make_request('post', '/api/query_results',
data={'data_source_id': self.factory.data_source.id,
'query': query.query,
'max_age': 0})
self.assertEquals(rv.status_code, 200)
self.assertNotIn('query_result', rv.json)
self.assertIn('job', rv.json)
def test_execute_query_without_access(self):
user = self.factory.create_user(groups=[self.factory.create_group().id])
query = self.factory.create_query()
rv = self.make_request('post', '/api/query_results',
data={'data_source_id': self.factory.data_source.id,
'query': query.query,
'max_age': 0},
user=user)
self.assertEquals(rv.status_code, 403)
self.assertIn('job', rv.json)
def test_execute_query_with_params(self):
query = "SELECT {{param}}"
rv = self.make_request('post', '/api/query_results',
data={'data_source_id': self.factory.data_source.id,
'query': query,
'max_age': 0})
self.assertEquals(rv.status_code, 400)
self.assertIn('job', rv.json)
rv = self.make_request('post', '/api/query_results?p_param=1',
data={'data_source_id': self.factory.data_source.id,
'query': query,
'max_age': 0})
self.assertEquals(rv.status_code, 200)
self.assertIn('job', rv.json)

91
tests/test_handlers.py
View File

@ -204,81 +204,6 @@ class WidgetAPITest(BaseTestCase):
# TODO: test how it updates the layout
class QueryAPITest(BaseTestCase, AuthenticationTestMixin):
def setUp(self):
self.paths = ['/api/queries']
super(QueryAPITest, self).setUp()
def test_update_query(self):
admin = self.factory.create_admin()
query = self.factory.create_query()
rv = self.make_request('post', '/api/queries/{0}'.format(query.id), data={'name': 'Testing'}, user=admin)
self.assertEqual(rv.status_code, 200)
self.assertEqual(rv.json['name'], 'Testing')
self.assertEqual(rv.json['last_modified_by']['id'], admin.id)
def test_create_query(self):
query_data = {
'name': 'Testing',
'query': 'SELECT 1',
'schedule': "3600",
'data_source_id': self.factory.data_source.id
}
rv = self.make_request('post', '/api/queries', data=query_data)
self.assertEquals(rv.status_code, 200)
self.assertDictContainsSubset(query_data, rv.json)
self.assertEquals(rv.json['user']['id'], self.factory.user.id)
self.assertIsNotNone(rv.json['api_key'])
self.assertIsNotNone(rv.json['query_hash'])
query = models.Query.get_by_id(rv.json['id'])
self.assertEquals(len(list(query.visualizations)), 1)
def test_get_query(self):
query = self.factory.create_query()
rv = self.make_request('get', '/api/queries/{0}'.format(query.id))
self.assertEquals(rv.status_code, 200)
self.assertResponseEqual(rv.json, query.to_dict(with_visualizations=True))
def test_get_all_queries(self):
queries = [self.factory.create_query() for _ in range(10)]
rv = self.make_request('get', '/api/queries')
self.assertEquals(rv.status_code, 200)
self.assertEquals(len(rv.json), 10)
def test_query_without_data_source_should_be_available_only_by_admin(self):
query = self.factory.create_query()
query.data_source = None
query.save()
rv = self.make_request('get', '/api/queries/{}'.format(query.id))
self.assertEquals(rv.status_code, 403)
rv = self.make_request('get', '/api/queries/{}'.format(query.id), user=self.factory.create_admin())
self.assertEquals(rv.status_code, 200)
def test_query_only_accessible_to_users_from_its_organization(self):
second_org = self.factory.create_org()
second_org_admin = self.factory.create_admin(org=second_org)
query = self.factory.create_query()
query.data_source = None
query.save()
rv = self.make_request('get', '/api/queries/{}'.format(query.id), user=second_org_admin)
self.assertEquals(rv.status_code, 404)
rv = self.make_request('get', '/api/queries/{}'.format(query.id), user=self.factory.create_admin())
self.assertEquals(rv.status_code, 200)
class VisualizationResourceTest(BaseTestCase):
def test_create_visualization(self):
query = self.factory.create_query()
@ -378,22 +303,6 @@ class VisualizationResourceTest(BaseTestCase):
self.assertEquals(rv.status_code, 404)
class QueryResultAPITest(BaseTestCase, AuthenticationTestMixin):
def setUp(self):
self.paths = []
super(QueryResultAPITest, self).setUp()
def test_post_result_list(self):
query_result = self.factory.create_query_result()
query = self.factory.create_query()
rv = self.make_request('post', '/api/query_results',
data={'data_source_id': self.factory.data_source.id,
'query': query.query})
self.assertEquals(rv.status_code, 200)
class JobAPITest(BaseTestCase, AuthenticationTestMixin):
def setUp(self):
self.paths = []

31
tests/test_utils.py
View File

@ -1,4 +1,4 @@
from redash.utils import build_url
from redash.utils import build_url, collect_query_parameters, collect_parameters_from_request
from collections import namedtuple
from unittest import TestCase
@ -21,4 +21,31 @@ class TestBuildUrl(TestCase):
self.assertEqual("https://example.com:80/test", build_url(DummyRequest("example.com:80", "https"), "example.com", "/test"))
self.assertEqual("http://example.com:443/test", build_url(DummyRequest("example.com:443", "http"), "example.com", "/test"))
# CALL LIOR!!!
class TestCollectParametersFromQuery(TestCase):
def test_returns_empty_list_for_regular_query(self):
query = u"SELECT 1"
self.assertEqual([], collect_query_parameters(query))
def test_finds_all_params(self):
query = u"SELECT {{param}} FROM {{table}}"
params = ['param', 'table']
self.assertEqual(params, collect_query_parameters(query))
def test_deduplicates_params(self):
query = u"SELECT {{param}}, {{param}} FROM {{table}}"
params = ['param', 'table']
self.assertEqual(params, collect_query_parameters(query))
def test_handles_nested_params(self):
query = u"SELECT {{param}}, {{param}} FROM {{table}} -- {{#test}} {{nested_param}} {{/test}}"
params = ['param', 'table', 'test', 'nested_param']
self.assertEqual(params, collect_query_parameters(query))
class TestCollectParametersFromRequest(TestCase):
def test_ignores_non_prefixed_values(self):
self.assertEqual({}, collect_parameters_from_request({'test': 1}))
def test_takes_prefixed_values(self):
self.assertDictEqual({'test': 1, 'something_else': 'test'}, collect_parameters_from_request({'p_test': 1, 'p_something_else': 'test'}))