2019-01-06 08:59:50 +00:00
|
|
|
import calendar
|
2014-03-19 11:48:48 +00:00
|
|
|
import datetime
|
2015-04-01 08:23:26 +00:00
|
|
|
from unittest import TestCase
|
2017-05-18 11:47:17 +00:00
|
|
|
|
2019-01-07 08:30:42 +00:00
|
|
|
import pytz
|
2015-10-16 20:10:50 +00:00
|
|
|
from dateutil.parser import parse as date_parse
|
2019-01-08 12:03:49 +00:00
|
|
|
from tests import BaseTestCase
|
2017-05-18 11:47:17 +00:00
|
|
|
|
2019-01-07 08:30:42 +00:00
|
|
|
from redash import models, redis_connection
|
|
|
|
from redash.models import db, types
|
2015-06-03 04:58:28 +00:00
|
|
|
from redash.utils import gen_query_hash, utcnow
|
2014-02-03 07:59:50 +00:00
|
|
|
|
|
|
|
|
2014-02-05 09:01:06 +00:00
|
|
|
class DashboardTest(BaseTestCase):
|
2014-02-03 07:59:50 +00:00
|
|
|
def test_appends_suffix_to_slug_when_duplicate(self):
|
2015-12-01 13:44:08 +00:00
|
|
|
d1 = self.factory.create_dashboard()
|
2016-11-21 15:34:44 +00:00
|
|
|
db.session.flush()
|
2019-12-11 11:54:29 +00:00
|
|
|
self.assertEqual(d1.slug, "test")
|
2014-02-03 07:59:50 +00:00
|
|
|
|
2015-12-01 13:44:08 +00:00
|
|
|
d2 = self.factory.create_dashboard(user=d1.user)
|
2016-11-21 15:34:44 +00:00
|
|
|
db.session.flush()
|
Migrate the application to Python 3 (#4251)
* Make core app compatible with Python 3
No backward compatibility with Python 2.7 is kept.
This commit mostly contains changes made with 2to3 and manual
tweaking when necessary.
* Use Python 3.7 as base docker image
Since it is not possible to change redash/base:debian to Python 3
without breaking future relases, its Dockerfile is temporarly
copied here.
* Upgrade some requirements to newest versions
Some of the older versions were not compatible with Python 3.
* Migrate tests to Python 3
* Build frontend on Python 3
* Make the HMAC sign function compatible with Python 3
In Python 3, HMAC only works with bytes so the strings and the
float used in the sign function need to be encoded.
Hopefully this is still backward compatible with already generated
signatures.
* Use assertCountEqual instead of assertItemsEqual
The latter is not available in Python 3.
See https://bugs.python.org/issue17866
* Remove redundant encoding header for Python 3 modules
* Remove redundant string encoding in CLI
* Rename list() functions in CLI
These functions shadow the builtin list function which is
problematic since 2to3 adds a fair amount of calls to the builtin
list when it finds dict.keys() and dict.values().
Only the Python function is renamed, from the perspective of the
CLI nothing changes.
* Replace usage of Exception.message in CLI
`message` is not available anymore, instead use the string
representation of the exception.
* Adapt test handlers to Python 3
* Fix test that relied on dict ordering
* Make sure test results are always uploaded (#4215)
* Support encoding memoryview to JSON
psycopg2 returns `buffer` objects in Python 2.7 and `memoryview`
in Python 3. See #3156
* Fix test relying on object address ordering
* Decode bytes returned from Redis
* Stop using e.message for most exceptions
Exception.message is not available in Python 3 anymore, except
for some exceptions defined by third-party libraries.
* Fix writing XLSX files in Python 3
The buffer for the file should be made of bytes and the actual
content written to it strings.
Note: I do not know why the diff is so large as it's only a two
lines change. Probably a white space or file encoding issue.
* Fix test by comparing strings to strings
* Fix another exception message unavailable in Python 3
* Fix export to CSV in Python 3
The UnicodeWriter is not used anymore. In Python 3, the interface
provided by the CSV module only deals with strings, in and out.
The encoding of the output is left to the user, in our case
it is given to Flask via `make_response`.
* (Python 3) Use Redis' decode_responses=True option (#4232)
* Fix test_outdated_queries_works_scheduled_queries_tracker (use utcnow)
* Make sure Redis connection uses decoded_responses option
* Remove unused imports.
* Use Redis' decode_responses option
* Remove cases of explicit Redis decoding
* Rename helper function and make sure it doesn't apply twice.
* Don't add decode_responses to Celery Redis connection URL
* Fix displaying error while connecting to SQLite
The exception message is always a string in Python 3, so no
need to try to decode things.
* Fix another missing exception message
* Handle JSON encoding for datasources returning bytes
SimpleJSON assumes the bytes it receives contain text data, so it
tries to UTF-8 encode them. It is sometimes not true, for instance
the SQLite datasource returns bytes for BLOB types, which typically
do not contain text but truly binary data.
This commit disables SimpleJSON auto encoding of bytes to str and
instead uses the same method as for memoryviews: generating a
hex representation of the data.
* Fix Python 3 compatibility with RQ
* Revert some changes 2to3 tends to do (#4261)
- Revert some changes 2to3 tends to do when it errs on the side of caution regarding dict view objects.
- Also fixed some naming issues with one character variables in list comprehensions.
- Fix Flask warning.
* Upgrade dependencies
* Remove useless `iter` added by 2to3
* Fix get_next_path tests (#4280)
* Removed setting SERVER_NAME in tests setup to avoid a warning.
* Change get_next_path to not return empty string in case of a domain only value.
* Fix redirect tests:
Since version 0.15 of Werkzeug it uses full path for fixing the location header instead of the root path.
* Remove explicit dependency for Werkzeug
* Switched pytz and certifi to unbinded versions.
* Switch to new library for getting country from IP
`python-geoip-geolite2` is not compatible with Python 3, instead
use `maxminddb-geolite2` which is very similar as it includes
the geolite2 database in the package .
* Python 3 RQ modifications (#4281)
* show current worker job (alongside with minor cosmetic column tweaks)
* avoid loading entire job data for queued jobs
* track general RQ queues (default, periodic and schemas)
* get all active RQ queues
* call get_celery_queues in another place
* merge dicts the Python 3 way
* extend the result_ttl of refresh_queries to 600 seconds to allow it to continue running periodically even after longer executions
* Remove legacy Python flake8 tests
2019-10-24 09:42:13 +00:00
|
|
|
self.assertNotEqual(d1.slug, d2.slug)
|
2014-02-03 07:59:50 +00:00
|
|
|
|
2015-12-01 13:44:08 +00:00
|
|
|
d3 = self.factory.create_dashboard(user=d1.user)
|
2016-11-21 15:34:44 +00:00
|
|
|
db.session.flush()
|
Migrate the application to Python 3 (#4251)
* Make core app compatible with Python 3
No backward compatibility with Python 2.7 is kept.
This commit mostly contains changes made with 2to3 and manual
tweaking when necessary.
* Use Python 3.7 as base docker image
Since it is not possible to change redash/base:debian to Python 3
without breaking future relases, its Dockerfile is temporarly
copied here.
* Upgrade some requirements to newest versions
Some of the older versions were not compatible with Python 3.
* Migrate tests to Python 3
* Build frontend on Python 3
* Make the HMAC sign function compatible with Python 3
In Python 3, HMAC only works with bytes so the strings and the
float used in the sign function need to be encoded.
Hopefully this is still backward compatible with already generated
signatures.
* Use assertCountEqual instead of assertItemsEqual
The latter is not available in Python 3.
See https://bugs.python.org/issue17866
* Remove redundant encoding header for Python 3 modules
* Remove redundant string encoding in CLI
* Rename list() functions in CLI
These functions shadow the builtin list function which is
problematic since 2to3 adds a fair amount of calls to the builtin
list when it finds dict.keys() and dict.values().
Only the Python function is renamed, from the perspective of the
CLI nothing changes.
* Replace usage of Exception.message in CLI
`message` is not available anymore, instead use the string
representation of the exception.
* Adapt test handlers to Python 3
* Fix test that relied on dict ordering
* Make sure test results are always uploaded (#4215)
* Support encoding memoryview to JSON
psycopg2 returns `buffer` objects in Python 2.7 and `memoryview`
in Python 3. See #3156
* Fix test relying on object address ordering
* Decode bytes returned from Redis
* Stop using e.message for most exceptions
Exception.message is not available in Python 3 anymore, except
for some exceptions defined by third-party libraries.
* Fix writing XLSX files in Python 3
The buffer for the file should be made of bytes and the actual
content written to it strings.
Note: I do not know why the diff is so large as it's only a two
lines change. Probably a white space or file encoding issue.
* Fix test by comparing strings to strings
* Fix another exception message unavailable in Python 3
* Fix export to CSV in Python 3
The UnicodeWriter is not used anymore. In Python 3, the interface
provided by the CSV module only deals with strings, in and out.
The encoding of the output is left to the user, in our case
it is given to Flask via `make_response`.
* (Python 3) Use Redis' decode_responses=True option (#4232)
* Fix test_outdated_queries_works_scheduled_queries_tracker (use utcnow)
* Make sure Redis connection uses decoded_responses option
* Remove unused imports.
* Use Redis' decode_responses option
* Remove cases of explicit Redis decoding
* Rename helper function and make sure it doesn't apply twice.
* Don't add decode_responses to Celery Redis connection URL
* Fix displaying error while connecting to SQLite
The exception message is always a string in Python 3, so no
need to try to decode things.
* Fix another missing exception message
* Handle JSON encoding for datasources returning bytes
SimpleJSON assumes the bytes it receives contain text data, so it
tries to UTF-8 encode them. It is sometimes not true, for instance
the SQLite datasource returns bytes for BLOB types, which typically
do not contain text but truly binary data.
This commit disables SimpleJSON auto encoding of bytes to str and
instead uses the same method as for memoryviews: generating a
hex representation of the data.
* Fix Python 3 compatibility with RQ
* Revert some changes 2to3 tends to do (#4261)
- Revert some changes 2to3 tends to do when it errs on the side of caution regarding dict view objects.
- Also fixed some naming issues with one character variables in list comprehensions.
- Fix Flask warning.
* Upgrade dependencies
* Remove useless `iter` added by 2to3
* Fix get_next_path tests (#4280)
* Removed setting SERVER_NAME in tests setup to avoid a warning.
* Change get_next_path to not return empty string in case of a domain only value.
* Fix redirect tests:
Since version 0.15 of Werkzeug it uses full path for fixing the location header instead of the root path.
* Remove explicit dependency for Werkzeug
* Switched pytz and certifi to unbinded versions.
* Switch to new library for getting country from IP
`python-geoip-geolite2` is not compatible with Python 3, instead
use `maxminddb-geolite2` which is very similar as it includes
the geolite2 database in the package .
* Python 3 RQ modifications (#4281)
* show current worker job (alongside with minor cosmetic column tweaks)
* avoid loading entire job data for queued jobs
* track general RQ queues (default, periodic and schemas)
* get all active RQ queues
* call get_celery_queues in another place
* merge dicts the Python 3 way
* extend the result_ttl of refresh_queries to 600 seconds to allow it to continue running periodically even after longer executions
* Remove legacy Python flake8 tests
2019-10-24 09:42:13 +00:00
|
|
|
self.assertNotEqual(d1.slug, d3.slug)
|
|
|
|
self.assertNotEqual(d2.slug, d3.slug)
|
2014-02-13 11:08:48 +00:00
|
|
|
|
|
|
|
|
2015-04-01 08:23:26 +00:00
|
|
|
class ShouldScheduleNextTest(TestCase):
|
|
|
|
def test_interval_schedule_that_needs_reschedule(self):
|
2016-11-23 18:35:18 +00:00
|
|
|
now = utcnow()
|
2015-04-01 08:23:26 +00:00
|
|
|
two_hours_ago = now - datetime.timedelta(hours=2)
|
2019-01-06 08:59:50 +00:00
|
|
|
self.assertTrue(models.should_schedule_next(two_hours_ago, now, "3600"))
|
2015-04-01 08:23:26 +00:00
|
|
|
|
|
|
|
def test_interval_schedule_that_doesnt_need_reschedule(self):
|
2016-11-23 18:35:18 +00:00
|
|
|
now = utcnow()
|
2015-04-01 08:23:26 +00:00
|
|
|
half_an_hour_ago = now - datetime.timedelta(minutes=30)
|
2019-01-06 08:59:50 +00:00
|
|
|
self.assertFalse(models.should_schedule_next(half_an_hour_ago, now, "3600"))
|
2015-04-01 08:23:26 +00:00
|
|
|
|
|
|
|
def test_exact_time_that_needs_reschedule(self):
|
2016-11-23 18:35:18 +00:00
|
|
|
now = utcnow()
|
2015-04-01 08:23:26 +00:00
|
|
|
yesterday = now - datetime.timedelta(days=1)
|
2016-02-16 04:16:31 +00:00
|
|
|
scheduled_datetime = now - datetime.timedelta(hours=3)
|
|
|
|
scheduled_time = "{:02d}:00".format(scheduled_datetime.hour)
|
2019-12-11 11:54:29 +00:00
|
|
|
self.assertTrue(
|
|
|
|
models.should_schedule_next(yesterday, now, "86400", scheduled_time)
|
|
|
|
)
|
2015-04-01 08:23:26 +00:00
|
|
|
|
|
|
|
def test_exact_time_that_doesnt_need_reschedule(self):
|
2015-10-16 20:10:50 +00:00
|
|
|
now = date_parse("2015-10-16 20:10")
|
|
|
|
yesterday = date_parse("2015-10-15 23:07")
|
|
|
|
schedule = "23:00"
|
2019-01-06 08:59:50 +00:00
|
|
|
self.assertFalse(models.should_schedule_next(yesterday, now, "86400", schedule))
|
2015-04-01 08:23:26 +00:00
|
|
|
|
|
|
|
def test_exact_time_with_day_change(self):
|
2016-11-23 18:35:18 +00:00
|
|
|
now = utcnow().replace(hour=0, minute=1)
|
2019-12-11 11:54:29 +00:00
|
|
|
previous = (now - datetime.timedelta(days=2)).replace(hour=23, minute=59)
|
2015-04-01 08:23:26 +00:00
|
|
|
schedule = "23:59".format(now.hour + 3)
|
2019-01-06 08:59:50 +00:00
|
|
|
self.assertTrue(models.should_schedule_next(previous, now, "86400", schedule))
|
|
|
|
|
|
|
|
def test_exact_time_every_x_days_that_needs_reschedule(self):
|
|
|
|
now = utcnow()
|
|
|
|
four_days_ago = now - datetime.timedelta(days=4)
|
|
|
|
three_day_interval = "259200"
|
|
|
|
scheduled_datetime = now - datetime.timedelta(hours=3)
|
|
|
|
scheduled_time = "{:02d}:00".format(scheduled_datetime.hour)
|
2019-12-11 11:54:29 +00:00
|
|
|
self.assertTrue(
|
|
|
|
models.should_schedule_next(
|
|
|
|
four_days_ago, now, three_day_interval, scheduled_time
|
|
|
|
)
|
|
|
|
)
|
2019-01-06 08:59:50 +00:00
|
|
|
|
|
|
|
def test_exact_time_every_x_days_that_doesnt_need_reschedule(self):
|
|
|
|
now = utcnow()
|
|
|
|
four_days_ago = now - datetime.timedelta(days=2)
|
|
|
|
three_day_interval = "259200"
|
|
|
|
scheduled_datetime = now - datetime.timedelta(hours=3)
|
|
|
|
scheduled_time = "{:02d}:00".format(scheduled_datetime.hour)
|
2019-12-11 11:54:29 +00:00
|
|
|
self.assertFalse(
|
|
|
|
models.should_schedule_next(
|
|
|
|
four_days_ago, now, three_day_interval, scheduled_time
|
|
|
|
)
|
|
|
|
)
|
2019-01-06 08:59:50 +00:00
|
|
|
|
|
|
|
def test_exact_time_every_x_days_with_day_change(self):
|
|
|
|
now = utcnow().replace(hour=23, minute=59)
|
|
|
|
previous = (now - datetime.timedelta(days=2)).replace(hour=0, minute=1)
|
|
|
|
schedule = "23:58"
|
|
|
|
three_day_interval = "259200"
|
2019-12-11 11:54:29 +00:00
|
|
|
self.assertTrue(
|
|
|
|
models.should_schedule_next(previous, now, three_day_interval, schedule)
|
|
|
|
)
|
2019-01-06 08:59:50 +00:00
|
|
|
|
|
|
|
def test_exact_time_every_x_weeks_that_needs_reschedule(self):
|
|
|
|
# Setup:
|
|
|
|
#
|
|
|
|
# 1) The query should run every 3 weeks on Tuesday
|
|
|
|
# 2) The last time it ran was 3 weeks ago from this week's Thursday
|
|
|
|
# 3) It is now Wednesday of this week
|
|
|
|
#
|
|
|
|
# Expectation: Even though less than 3 weeks have passed since the
|
|
|
|
# last run 3 weeks ago on Thursday, it's overdue since
|
|
|
|
# it should be running on Tuesdays.
|
2019-12-11 11:54:29 +00:00
|
|
|
this_thursday = utcnow() + datetime.timedelta(
|
|
|
|
days=list(calendar.day_name).index("Thursday") - utcnow().weekday()
|
|
|
|
)
|
2019-01-06 08:59:50 +00:00
|
|
|
three_weeks_ago = this_thursday - datetime.timedelta(weeks=3)
|
|
|
|
now = this_thursday - datetime.timedelta(days=1)
|
|
|
|
three_week_interval = "1814400"
|
|
|
|
scheduled_datetime = now - datetime.timedelta(hours=3)
|
|
|
|
scheduled_time = "{:02d}:00".format(scheduled_datetime.hour)
|
2019-12-11 11:54:29 +00:00
|
|
|
self.assertTrue(
|
|
|
|
models.should_schedule_next(
|
|
|
|
three_weeks_ago, now, three_week_interval, scheduled_time, "Tuesday"
|
|
|
|
)
|
|
|
|
)
|
2019-01-06 08:59:50 +00:00
|
|
|
|
|
|
|
def test_exact_time_every_x_weeks_that_doesnt_need_reschedule(self):
|
|
|
|
# Setup:
|
|
|
|
#
|
|
|
|
# 1) The query should run every 3 weeks on Thurday
|
|
|
|
# 2) The last time it ran was 3 weeks ago from this week's Tuesday
|
|
|
|
# 3) It is now Wednesday of this week
|
|
|
|
#
|
|
|
|
# Expectation: Even though more than 3 weeks have passed since the
|
|
|
|
# last run 3 weeks ago on Tuesday, it's not overdue since
|
|
|
|
# it should be running on Thursdays.
|
2019-12-11 11:54:29 +00:00
|
|
|
this_tuesday = utcnow() + datetime.timedelta(
|
|
|
|
days=list(calendar.day_name).index("Tuesday") - utcnow().weekday()
|
|
|
|
)
|
2019-01-06 08:59:50 +00:00
|
|
|
three_weeks_ago = this_tuesday - datetime.timedelta(weeks=3)
|
|
|
|
now = this_tuesday + datetime.timedelta(days=1)
|
|
|
|
three_week_interval = "1814400"
|
|
|
|
scheduled_datetime = now - datetime.timedelta(hours=3)
|
|
|
|
scheduled_time = "{:02d}:00".format(scheduled_datetime.hour)
|
2019-12-11 11:54:29 +00:00
|
|
|
self.assertFalse(
|
|
|
|
models.should_schedule_next(
|
|
|
|
three_weeks_ago, now, three_week_interval, scheduled_time, "Thursday"
|
|
|
|
)
|
|
|
|
)
|
2015-04-01 08:23:26 +00:00
|
|
|
|
2017-02-03 07:46:02 +00:00
|
|
|
def test_backoff(self):
|
|
|
|
now = utcnow()
|
|
|
|
two_hours_ago = now - datetime.timedelta(hours=2)
|
2019-12-11 11:54:29 +00:00
|
|
|
self.assertTrue(
|
|
|
|
models.should_schedule_next(two_hours_ago, now, "3600", failures=5)
|
|
|
|
)
|
|
|
|
self.assertFalse(
|
|
|
|
models.should_schedule_next(two_hours_ago, now, "3600", failures=10)
|
|
|
|
)
|
2017-02-03 07:46:02 +00:00
|
|
|
|
2019-06-16 08:34:29 +00:00
|
|
|
def test_next_iteration_overflow(self):
|
|
|
|
now = utcnow()
|
|
|
|
two_hours_ago = now - datetime.timedelta(hours=2)
|
2019-12-11 11:54:29 +00:00
|
|
|
self.assertFalse(
|
|
|
|
models.should_schedule_next(two_hours_ago, now, "3600", failures=32)
|
|
|
|
)
|
2019-06-16 08:34:29 +00:00
|
|
|
|
2015-04-01 08:23:26 +00:00
|
|
|
|
|
|
|
class QueryOutdatedQueriesTest(BaseTestCase):
|
|
|
|
# TODO: this test can be refactored to use mock version of should_schedule_next to simplify it.
|
|
|
|
def test_outdated_queries_skips_unscheduled_queries(self):
|
2019-12-11 11:54:29 +00:00
|
|
|
query = self.factory.create_query(
|
|
|
|
schedule={
|
|
|
|
"interval": None,
|
|
|
|
"time": None,
|
|
|
|
"until": None,
|
|
|
|
"day_of_week": None,
|
|
|
|
}
|
|
|
|
)
|
2019-01-18 09:30:45 +00:00
|
|
|
query_with_none = self.factory.create_query(schedule=None)
|
|
|
|
|
2015-04-01 08:23:26 +00:00
|
|
|
queries = models.Query.outdated_queries()
|
|
|
|
|
|
|
|
self.assertNotIn(query, queries)
|
2019-01-18 09:30:45 +00:00
|
|
|
self.assertNotIn(query_with_none, queries)
|
2015-04-01 08:23:26 +00:00
|
|
|
|
|
|
|
def test_outdated_queries_works_with_ttl_based_schedule(self):
|
2016-11-23 18:35:18 +00:00
|
|
|
two_hours_ago = utcnow() - datetime.timedelta(hours=2)
|
2019-12-11 11:54:29 +00:00
|
|
|
query = self.factory.create_query(
|
|
|
|
schedule={
|
|
|
|
"interval": "3600",
|
|
|
|
"time": None,
|
|
|
|
"until": None,
|
|
|
|
"day_of_week": None,
|
|
|
|
}
|
|
|
|
)
|
|
|
|
query_result = self.factory.create_query_result(
|
|
|
|
query=query.query_text, retrieved_at=two_hours_ago
|
|
|
|
)
|
2015-04-01 08:23:26 +00:00
|
|
|
query.latest_query_data = query_result
|
|
|
|
|
|
|
|
queries = models.Query.outdated_queries()
|
|
|
|
self.assertIn(query, queries)
|
|
|
|
|
2017-05-18 11:47:17 +00:00
|
|
|
def test_outdated_queries_works_scheduled_queries_tracker(self):
|
Migrate the application to Python 3 (#4251)
* Make core app compatible with Python 3
No backward compatibility with Python 2.7 is kept.
This commit mostly contains changes made with 2to3 and manual
tweaking when necessary.
* Use Python 3.7 as base docker image
Since it is not possible to change redash/base:debian to Python 3
without breaking future relases, its Dockerfile is temporarly
copied here.
* Upgrade some requirements to newest versions
Some of the older versions were not compatible with Python 3.
* Migrate tests to Python 3
* Build frontend on Python 3
* Make the HMAC sign function compatible with Python 3
In Python 3, HMAC only works with bytes so the strings and the
float used in the sign function need to be encoded.
Hopefully this is still backward compatible with already generated
signatures.
* Use assertCountEqual instead of assertItemsEqual
The latter is not available in Python 3.
See https://bugs.python.org/issue17866
* Remove redundant encoding header for Python 3 modules
* Remove redundant string encoding in CLI
* Rename list() functions in CLI
These functions shadow the builtin list function which is
problematic since 2to3 adds a fair amount of calls to the builtin
list when it finds dict.keys() and dict.values().
Only the Python function is renamed, from the perspective of the
CLI nothing changes.
* Replace usage of Exception.message in CLI
`message` is not available anymore, instead use the string
representation of the exception.
* Adapt test handlers to Python 3
* Fix test that relied on dict ordering
* Make sure test results are always uploaded (#4215)
* Support encoding memoryview to JSON
psycopg2 returns `buffer` objects in Python 2.7 and `memoryview`
in Python 3. See #3156
* Fix test relying on object address ordering
* Decode bytes returned from Redis
* Stop using e.message for most exceptions
Exception.message is not available in Python 3 anymore, except
for some exceptions defined by third-party libraries.
* Fix writing XLSX files in Python 3
The buffer for the file should be made of bytes and the actual
content written to it strings.
Note: I do not know why the diff is so large as it's only a two
lines change. Probably a white space or file encoding issue.
* Fix test by comparing strings to strings
* Fix another exception message unavailable in Python 3
* Fix export to CSV in Python 3
The UnicodeWriter is not used anymore. In Python 3, the interface
provided by the CSV module only deals with strings, in and out.
The encoding of the output is left to the user, in our case
it is given to Flask via `make_response`.
* (Python 3) Use Redis' decode_responses=True option (#4232)
* Fix test_outdated_queries_works_scheduled_queries_tracker (use utcnow)
* Make sure Redis connection uses decoded_responses option
* Remove unused imports.
* Use Redis' decode_responses option
* Remove cases of explicit Redis decoding
* Rename helper function and make sure it doesn't apply twice.
* Don't add decode_responses to Celery Redis connection URL
* Fix displaying error while connecting to SQLite
The exception message is always a string in Python 3, so no
need to try to decode things.
* Fix another missing exception message
* Handle JSON encoding for datasources returning bytes
SimpleJSON assumes the bytes it receives contain text data, so it
tries to UTF-8 encode them. It is sometimes not true, for instance
the SQLite datasource returns bytes for BLOB types, which typically
do not contain text but truly binary data.
This commit disables SimpleJSON auto encoding of bytes to str and
instead uses the same method as for memoryviews: generating a
hex representation of the data.
* Fix Python 3 compatibility with RQ
* Revert some changes 2to3 tends to do (#4261)
- Revert some changes 2to3 tends to do when it errs on the side of caution regarding dict view objects.
- Also fixed some naming issues with one character variables in list comprehensions.
- Fix Flask warning.
* Upgrade dependencies
* Remove useless `iter` added by 2to3
* Fix get_next_path tests (#4280)
* Removed setting SERVER_NAME in tests setup to avoid a warning.
* Change get_next_path to not return empty string in case of a domain only value.
* Fix redirect tests:
Since version 0.15 of Werkzeug it uses full path for fixing the location header instead of the root path.
* Remove explicit dependency for Werkzeug
* Switched pytz and certifi to unbinded versions.
* Switch to new library for getting country from IP
`python-geoip-geolite2` is not compatible with Python 3, instead
use `maxminddb-geolite2` which is very similar as it includes
the geolite2 database in the package .
* Python 3 RQ modifications (#4281)
* show current worker job (alongside with minor cosmetic column tweaks)
* avoid loading entire job data for queued jobs
* track general RQ queues (default, periodic and schemas)
* get all active RQ queues
* call get_celery_queues in another place
* merge dicts the Python 3 way
* extend the result_ttl of refresh_queries to 600 seconds to allow it to continue running periodically even after longer executions
* Remove legacy Python flake8 tests
2019-10-24 09:42:13 +00:00
|
|
|
two_hours_ago = utcnow() - datetime.timedelta(hours=2)
|
2019-12-11 11:54:29 +00:00
|
|
|
query = self.factory.create_query(
|
|
|
|
schedule={
|
|
|
|
"interval": "3600",
|
|
|
|
"time": None,
|
|
|
|
"until": None,
|
|
|
|
"day_of_week": None,
|
|
|
|
}
|
|
|
|
)
|
|
|
|
query_result = self.factory.create_query_result(
|
|
|
|
query=query, retrieved_at=two_hours_ago
|
|
|
|
)
|
2017-05-18 11:47:17 +00:00
|
|
|
query.latest_query_data = query_result
|
|
|
|
|
|
|
|
models.scheduled_queries_executions.update(query.id)
|
|
|
|
|
|
|
|
queries = models.Query.outdated_queries()
|
|
|
|
self.assertNotIn(query, queries)
|
|
|
|
|
2015-04-01 08:23:26 +00:00
|
|
|
def test_skips_fresh_queries(self):
|
2016-11-23 18:35:18 +00:00
|
|
|
half_an_hour_ago = utcnow() - datetime.timedelta(minutes=30)
|
2019-12-11 11:54:29 +00:00
|
|
|
query = self.factory.create_query(
|
|
|
|
schedule={
|
|
|
|
"interval": "3600",
|
|
|
|
"time": None,
|
|
|
|
"until": None,
|
|
|
|
"day_of_week": None,
|
|
|
|
}
|
|
|
|
)
|
|
|
|
query_result = self.factory.create_query_result(
|
|
|
|
query=query.query_text, retrieved_at=half_an_hour_ago
|
|
|
|
)
|
2015-04-01 08:23:26 +00:00
|
|
|
query.latest_query_data = query_result
|
|
|
|
|
|
|
|
queries = models.Query.outdated_queries()
|
|
|
|
self.assertNotIn(query, queries)
|
|
|
|
|
|
|
|
def test_outdated_queries_works_with_specific_time_schedule(self):
|
2015-06-03 04:58:28 +00:00
|
|
|
half_an_hour_ago = utcnow() - datetime.timedelta(minutes=30)
|
2019-12-11 11:54:29 +00:00
|
|
|
query = self.factory.create_query(
|
|
|
|
schedule={
|
|
|
|
"interval": "86400",
|
|
|
|
"time": half_an_hour_ago.strftime("%H:%M"),
|
|
|
|
"until": None,
|
|
|
|
"day_of_week": None,
|
|
|
|
}
|
|
|
|
)
|
|
|
|
query_result = self.factory.create_query_result(
|
|
|
|
query=query.query_text,
|
|
|
|
retrieved_at=half_an_hour_ago - datetime.timedelta(days=1),
|
|
|
|
)
|
2015-04-01 08:23:26 +00:00
|
|
|
query.latest_query_data = query_result
|
|
|
|
|
|
|
|
queries = models.Query.outdated_queries()
|
|
|
|
self.assertIn(query, queries)
|
|
|
|
|
2017-02-02 04:39:27 +00:00
|
|
|
def test_enqueues_query_only_once(self):
|
|
|
|
"""
|
|
|
|
Only one query per data source with the same text will be reported by
|
|
|
|
Query.outdated_queries().
|
|
|
|
"""
|
2019-12-11 11:54:29 +00:00
|
|
|
query = self.factory.create_query(
|
|
|
|
schedule={
|
|
|
|
"interval": "60",
|
|
|
|
"until": None,
|
|
|
|
"time": None,
|
|
|
|
"day_of_week": None,
|
|
|
|
}
|
|
|
|
)
|
2017-02-02 04:39:27 +00:00
|
|
|
query2 = self.factory.create_query(
|
2019-12-11 11:54:29 +00:00
|
|
|
schedule={
|
|
|
|
"interval": "60",
|
|
|
|
"until": None,
|
|
|
|
"time": None,
|
|
|
|
"day_of_week": None,
|
|
|
|
},
|
|
|
|
query_text=query.query_text,
|
|
|
|
query_hash=query.query_hash,
|
|
|
|
)
|
2017-02-02 04:39:27 +00:00
|
|
|
retrieved_at = utcnow() - datetime.timedelta(minutes=10)
|
|
|
|
query_result = self.factory.create_query_result(
|
2019-12-11 11:54:29 +00:00
|
|
|
retrieved_at=retrieved_at,
|
|
|
|
query_text=query.query_text,
|
|
|
|
query_hash=query.query_hash,
|
|
|
|
)
|
2017-02-02 04:39:27 +00:00
|
|
|
query.latest_query_data = query_result
|
|
|
|
query2.latest_query_data = query_result
|
|
|
|
|
|
|
|
self.assertEqual(list(models.Query.outdated_queries()), [query2])
|
|
|
|
|
|
|
|
def test_enqueues_query_with_correct_data_source(self):
|
|
|
|
"""
|
|
|
|
Queries from different data sources will be reported by
|
|
|
|
Query.outdated_queries() even if they have the same query text.
|
|
|
|
"""
|
|
|
|
query = self.factory.create_query(
|
2019-12-11 11:54:29 +00:00
|
|
|
schedule={
|
|
|
|
"interval": "60",
|
|
|
|
"until": None,
|
|
|
|
"time": None,
|
|
|
|
"day_of_week": None,
|
|
|
|
},
|
|
|
|
data_source=self.factory.create_data_source(),
|
|
|
|
)
|
2017-02-02 04:39:27 +00:00
|
|
|
query2 = self.factory.create_query(
|
2019-12-11 11:54:29 +00:00
|
|
|
schedule={
|
|
|
|
"interval": "60",
|
|
|
|
"until": None,
|
|
|
|
"time": None,
|
|
|
|
"day_of_week": None,
|
|
|
|
},
|
|
|
|
query_text=query.query_text,
|
|
|
|
query_hash=query.query_hash,
|
|
|
|
)
|
2017-02-02 04:39:27 +00:00
|
|
|
retrieved_at = utcnow() - datetime.timedelta(minutes=10)
|
|
|
|
query_result = self.factory.create_query_result(
|
2019-12-11 11:54:29 +00:00
|
|
|
retrieved_at=retrieved_at,
|
|
|
|
query_text=query.query_text,
|
|
|
|
query_hash=query.query_hash,
|
|
|
|
)
|
2017-02-02 04:39:27 +00:00
|
|
|
query.latest_query_data = query_result
|
|
|
|
query2.latest_query_data = query_result
|
|
|
|
|
Migrate the application to Python 3 (#4251)
* Make core app compatible with Python 3
No backward compatibility with Python 2.7 is kept.
This commit mostly contains changes made with 2to3 and manual
tweaking when necessary.
* Use Python 3.7 as base docker image
Since it is not possible to change redash/base:debian to Python 3
without breaking future relases, its Dockerfile is temporarly
copied here.
* Upgrade some requirements to newest versions
Some of the older versions were not compatible with Python 3.
* Migrate tests to Python 3
* Build frontend on Python 3
* Make the HMAC sign function compatible with Python 3
In Python 3, HMAC only works with bytes so the strings and the
float used in the sign function need to be encoded.
Hopefully this is still backward compatible with already generated
signatures.
* Use assertCountEqual instead of assertItemsEqual
The latter is not available in Python 3.
See https://bugs.python.org/issue17866
* Remove redundant encoding header for Python 3 modules
* Remove redundant string encoding in CLI
* Rename list() functions in CLI
These functions shadow the builtin list function which is
problematic since 2to3 adds a fair amount of calls to the builtin
list when it finds dict.keys() and dict.values().
Only the Python function is renamed, from the perspective of the
CLI nothing changes.
* Replace usage of Exception.message in CLI
`message` is not available anymore, instead use the string
representation of the exception.
* Adapt test handlers to Python 3
* Fix test that relied on dict ordering
* Make sure test results are always uploaded (#4215)
* Support encoding memoryview to JSON
psycopg2 returns `buffer` objects in Python 2.7 and `memoryview`
in Python 3. See #3156
* Fix test relying on object address ordering
* Decode bytes returned from Redis
* Stop using e.message for most exceptions
Exception.message is not available in Python 3 anymore, except
for some exceptions defined by third-party libraries.
* Fix writing XLSX files in Python 3
The buffer for the file should be made of bytes and the actual
content written to it strings.
Note: I do not know why the diff is so large as it's only a two
lines change. Probably a white space or file encoding issue.
* Fix test by comparing strings to strings
* Fix another exception message unavailable in Python 3
* Fix export to CSV in Python 3
The UnicodeWriter is not used anymore. In Python 3, the interface
provided by the CSV module only deals with strings, in and out.
The encoding of the output is left to the user, in our case
it is given to Flask via `make_response`.
* (Python 3) Use Redis' decode_responses=True option (#4232)
* Fix test_outdated_queries_works_scheduled_queries_tracker (use utcnow)
* Make sure Redis connection uses decoded_responses option
* Remove unused imports.
* Use Redis' decode_responses option
* Remove cases of explicit Redis decoding
* Rename helper function and make sure it doesn't apply twice.
* Don't add decode_responses to Celery Redis connection URL
* Fix displaying error while connecting to SQLite
The exception message is always a string in Python 3, so no
need to try to decode things.
* Fix another missing exception message
* Handle JSON encoding for datasources returning bytes
SimpleJSON assumes the bytes it receives contain text data, so it
tries to UTF-8 encode them. It is sometimes not true, for instance
the SQLite datasource returns bytes for BLOB types, which typically
do not contain text but truly binary data.
This commit disables SimpleJSON auto encoding of bytes to str and
instead uses the same method as for memoryviews: generating a
hex representation of the data.
* Fix Python 3 compatibility with RQ
* Revert some changes 2to3 tends to do (#4261)
- Revert some changes 2to3 tends to do when it errs on the side of caution regarding dict view objects.
- Also fixed some naming issues with one character variables in list comprehensions.
- Fix Flask warning.
* Upgrade dependencies
* Remove useless `iter` added by 2to3
* Fix get_next_path tests (#4280)
* Removed setting SERVER_NAME in tests setup to avoid a warning.
* Change get_next_path to not return empty string in case of a domain only value.
* Fix redirect tests:
Since version 0.15 of Werkzeug it uses full path for fixing the location header instead of the root path.
* Remove explicit dependency for Werkzeug
* Switched pytz and certifi to unbinded versions.
* Switch to new library for getting country from IP
`python-geoip-geolite2` is not compatible with Python 3, instead
use `maxminddb-geolite2` which is very similar as it includes
the geolite2 database in the package .
* Python 3 RQ modifications (#4281)
* show current worker job (alongside with minor cosmetic column tweaks)
* avoid loading entire job data for queued jobs
* track general RQ queues (default, periodic and schemas)
* get all active RQ queues
* call get_celery_queues in another place
* merge dicts the Python 3 way
* extend the result_ttl of refresh_queries to 600 seconds to allow it to continue running periodically even after longer executions
* Remove legacy Python flake8 tests
2019-10-24 09:42:13 +00:00
|
|
|
outdated_queries = models.Query.outdated_queries()
|
|
|
|
self.assertEqual(len(outdated_queries), 2)
|
|
|
|
self.assertIn(query, outdated_queries)
|
|
|
|
self.assertIn(query2, outdated_queries)
|
2017-02-02 04:39:27 +00:00
|
|
|
|
|
|
|
def test_enqueues_only_for_relevant_data_source(self):
|
|
|
|
"""
|
|
|
|
If multiple queries with the same text exist, only ones that are
|
|
|
|
scheduled to be refreshed are reported by Query.outdated_queries().
|
|
|
|
"""
|
2019-12-11 11:54:29 +00:00
|
|
|
query = self.factory.create_query(
|
|
|
|
schedule={
|
|
|
|
"interval": "60",
|
|
|
|
"until": None,
|
|
|
|
"time": None,
|
|
|
|
"day_of_week": None,
|
|
|
|
}
|
|
|
|
)
|
2017-02-02 04:39:27 +00:00
|
|
|
query2 = self.factory.create_query(
|
2019-12-11 11:54:29 +00:00
|
|
|
schedule={
|
|
|
|
"interval": "3600",
|
|
|
|
"until": None,
|
|
|
|
"time": None,
|
|
|
|
"day_of_week": None,
|
|
|
|
},
|
|
|
|
query_text=query.query_text,
|
|
|
|
query_hash=query.query_hash,
|
|
|
|
)
|
2017-02-02 04:39:27 +00:00
|
|
|
retrieved_at = utcnow() - datetime.timedelta(minutes=10)
|
|
|
|
query_result = self.factory.create_query_result(
|
2019-12-11 11:54:29 +00:00
|
|
|
retrieved_at=retrieved_at,
|
|
|
|
query_text=query.query_text,
|
|
|
|
query_hash=query.query_hash,
|
|
|
|
)
|
2017-02-02 04:39:27 +00:00
|
|
|
query.latest_query_data = query_result
|
|
|
|
query2.latest_query_data = query_result
|
|
|
|
|
|
|
|
self.assertEqual(list(models.Query.outdated_queries()), [query])
|
|
|
|
|
|
|
|
def test_failure_extends_schedule(self):
|
|
|
|
"""
|
|
|
|
Execution failures recorded for a query result in exponential backoff
|
|
|
|
for scheduling future execution.
|
|
|
|
"""
|
2019-12-11 11:54:29 +00:00
|
|
|
query = self.factory.create_query(
|
|
|
|
schedule={
|
|
|
|
"interval": "60",
|
|
|
|
"until": None,
|
|
|
|
"time": None,
|
|
|
|
"day_of_week": None,
|
|
|
|
},
|
|
|
|
schedule_failures=4,
|
|
|
|
)
|
2017-02-03 07:46:02 +00:00
|
|
|
retrieved_at = utcnow() - datetime.timedelta(minutes=16)
|
2017-02-02 04:39:27 +00:00
|
|
|
query_result = self.factory.create_query_result(
|
2019-12-11 11:54:29 +00:00
|
|
|
retrieved_at=retrieved_at,
|
|
|
|
query_text=query.query_text,
|
|
|
|
query_hash=query.query_hash,
|
|
|
|
)
|
2017-02-02 04:39:27 +00:00
|
|
|
query.latest_query_data = query_result
|
|
|
|
|
|
|
|
self.assertEqual(list(models.Query.outdated_queries()), [])
|
|
|
|
|
2017-02-03 07:46:02 +00:00
|
|
|
query_result.retrieved_at = utcnow() - datetime.timedelta(minutes=17)
|
2017-02-02 04:39:27 +00:00
|
|
|
self.assertEqual(list(models.Query.outdated_queries()), [query])
|
|
|
|
|
2019-01-06 08:59:50 +00:00
|
|
|
def test_schedule_until_after(self):
|
|
|
|
"""
|
|
|
|
Queries with non-null ``schedule['until']`` are not reported by
|
|
|
|
Query.outdated_queries() after the given time is past.
|
|
|
|
"""
|
|
|
|
one_day_ago = (utcnow() - datetime.timedelta(days=1)).strftime("%Y-%m-%d")
|
|
|
|
two_hours_ago = utcnow() - datetime.timedelta(hours=2)
|
2019-12-11 11:54:29 +00:00
|
|
|
query = self.factory.create_query(
|
|
|
|
schedule={
|
|
|
|
"interval": "3600",
|
|
|
|
"until": one_day_ago,
|
|
|
|
"time": None,
|
|
|
|
"day_of_week": None,
|
|
|
|
}
|
|
|
|
)
|
|
|
|
query_result = self.factory.create_query_result(
|
|
|
|
query=query.query_text, retrieved_at=two_hours_ago
|
|
|
|
)
|
2019-01-06 08:59:50 +00:00
|
|
|
query.latest_query_data = query_result
|
|
|
|
|
|
|
|
queries = models.Query.outdated_queries()
|
|
|
|
self.assertNotIn(query, queries)
|
|
|
|
|
|
|
|
def test_schedule_until_before(self):
|
|
|
|
"""
|
|
|
|
Queries with non-null ``schedule['until']`` are reported by
|
|
|
|
Query.outdated_queries() before the given time is past.
|
|
|
|
"""
|
|
|
|
one_day_from_now = (utcnow() + datetime.timedelta(days=1)).strftime("%Y-%m-%d")
|
|
|
|
two_hours_ago = utcnow() - datetime.timedelta(hours=2)
|
2019-12-11 11:54:29 +00:00
|
|
|
query = self.factory.create_query(
|
|
|
|
schedule={
|
|
|
|
"interval": "3600",
|
|
|
|
"until": one_day_from_now,
|
|
|
|
"time": None,
|
|
|
|
"day_of_week": None,
|
|
|
|
}
|
|
|
|
)
|
|
|
|
query_result = self.factory.create_query_result(
|
|
|
|
query=query.query_text, retrieved_at=two_hours_ago
|
|
|
|
)
|
2019-01-06 08:59:50 +00:00
|
|
|
query.latest_query_data = query_result
|
|
|
|
|
|
|
|
queries = models.Query.outdated_queries()
|
|
|
|
self.assertIn(query, queries)
|
|
|
|
|
2015-04-01 08:23:26 +00:00
|
|
|
|
2014-09-25 14:42:33 +00:00
|
|
|
class QueryArchiveTest(BaseTestCase):
|
|
|
|
def test_archive_query_sets_flag(self):
|
2015-12-01 13:44:08 +00:00
|
|
|
query = self.factory.create_query()
|
2016-11-23 18:35:18 +00:00
|
|
|
db.session.flush()
|
2014-09-25 14:42:33 +00:00
|
|
|
query.archive()
|
|
|
|
|
Migrate the application to Python 3 (#4251)
* Make core app compatible with Python 3
No backward compatibility with Python 2.7 is kept.
This commit mostly contains changes made with 2to3 and manual
tweaking when necessary.
* Use Python 3.7 as base docker image
Since it is not possible to change redash/base:debian to Python 3
without breaking future relases, its Dockerfile is temporarly
copied here.
* Upgrade some requirements to newest versions
Some of the older versions were not compatible with Python 3.
* Migrate tests to Python 3
* Build frontend on Python 3
* Make the HMAC sign function compatible with Python 3
In Python 3, HMAC only works with bytes so the strings and the
float used in the sign function need to be encoded.
Hopefully this is still backward compatible with already generated
signatures.
* Use assertCountEqual instead of assertItemsEqual
The latter is not available in Python 3.
See https://bugs.python.org/issue17866
* Remove redundant encoding header for Python 3 modules
* Remove redundant string encoding in CLI
* Rename list() functions in CLI
These functions shadow the builtin list function which is
problematic since 2to3 adds a fair amount of calls to the builtin
list when it finds dict.keys() and dict.values().
Only the Python function is renamed, from the perspective of the
CLI nothing changes.
* Replace usage of Exception.message in CLI
`message` is not available anymore, instead use the string
representation of the exception.
* Adapt test handlers to Python 3
* Fix test that relied on dict ordering
* Make sure test results are always uploaded (#4215)
* Support encoding memoryview to JSON
psycopg2 returns `buffer` objects in Python 2.7 and `memoryview`
in Python 3. See #3156
* Fix test relying on object address ordering
* Decode bytes returned from Redis
* Stop using e.message for most exceptions
Exception.message is not available in Python 3 anymore, except
for some exceptions defined by third-party libraries.
* Fix writing XLSX files in Python 3
The buffer for the file should be made of bytes and the actual
content written to it strings.
Note: I do not know why the diff is so large as it's only a two
lines change. Probably a white space or file encoding issue.
* Fix test by comparing strings to strings
* Fix another exception message unavailable in Python 3
* Fix export to CSV in Python 3
The UnicodeWriter is not used anymore. In Python 3, the interface
provided by the CSV module only deals with strings, in and out.
The encoding of the output is left to the user, in our case
it is given to Flask via `make_response`.
* (Python 3) Use Redis' decode_responses=True option (#4232)
* Fix test_outdated_queries_works_scheduled_queries_tracker (use utcnow)
* Make sure Redis connection uses decoded_responses option
* Remove unused imports.
* Use Redis' decode_responses option
* Remove cases of explicit Redis decoding
* Rename helper function and make sure it doesn't apply twice.
* Don't add decode_responses to Celery Redis connection URL
* Fix displaying error while connecting to SQLite
The exception message is always a string in Python 3, so no
need to try to decode things.
* Fix another missing exception message
* Handle JSON encoding for datasources returning bytes
SimpleJSON assumes the bytes it receives contain text data, so it
tries to UTF-8 encode them. It is sometimes not true, for instance
the SQLite datasource returns bytes for BLOB types, which typically
do not contain text but truly binary data.
This commit disables SimpleJSON auto encoding of bytes to str and
instead uses the same method as for memoryviews: generating a
hex representation of the data.
* Fix Python 3 compatibility with RQ
* Revert some changes 2to3 tends to do (#4261)
- Revert some changes 2to3 tends to do when it errs on the side of caution regarding dict view objects.
- Also fixed some naming issues with one character variables in list comprehensions.
- Fix Flask warning.
* Upgrade dependencies
* Remove useless `iter` added by 2to3
* Fix get_next_path tests (#4280)
* Removed setting SERVER_NAME in tests setup to avoid a warning.
* Change get_next_path to not return empty string in case of a domain only value.
* Fix redirect tests:
Since version 0.15 of Werkzeug it uses full path for fixing the location header instead of the root path.
* Remove explicit dependency for Werkzeug
* Switched pytz and certifi to unbinded versions.
* Switch to new library for getting country from IP
`python-geoip-geolite2` is not compatible with Python 3, instead
use `maxminddb-geolite2` which is very similar as it includes
the geolite2 database in the package .
* Python 3 RQ modifications (#4281)
* show current worker job (alongside with minor cosmetic column tweaks)
* avoid loading entire job data for queued jobs
* track general RQ queues (default, periodic and schemas)
* get all active RQ queues
* call get_celery_queues in another place
* merge dicts the Python 3 way
* extend the result_ttl of refresh_queries to 600 seconds to allow it to continue running periodically even after longer executions
* Remove legacy Python flake8 tests
2019-10-24 09:42:13 +00:00
|
|
|
self.assertEqual(query.is_archived, True)
|
2014-09-25 14:42:33 +00:00
|
|
|
|
|
|
|
def test_archived_query_doesnt_return_in_all(self):
|
2019-12-11 11:54:29 +00:00
|
|
|
query = self.factory.create_query(
|
|
|
|
schedule={"interval": "1", "until": None, "time": None, "day_of_week": None}
|
|
|
|
)
|
2016-11-23 18:35:18 +00:00
|
|
|
yesterday = utcnow() - datetime.timedelta(days=1)
|
2019-10-10 07:39:55 +00:00
|
|
|
query_result = models.QueryResult.store_result(
|
2019-12-11 11:54:29 +00:00
|
|
|
query.org_id,
|
|
|
|
query.data_source,
|
|
|
|
query.query_hash,
|
|
|
|
query.query_text,
|
|
|
|
"1",
|
|
|
|
123,
|
|
|
|
yesterday,
|
|
|
|
)
|
2014-09-25 14:42:33 +00:00
|
|
|
|
|
|
|
query.latest_query_data = query_result
|
2016-11-23 18:35:18 +00:00
|
|
|
groups = list(models.Group.query.filter(models.Group.id.in_(query.groups)))
|
2016-12-08 14:07:25 +00:00
|
|
|
self.assertIn(query, list(models.Query.all_queries([g.id for g in groups])))
|
2014-09-25 14:42:33 +00:00
|
|
|
self.assertIn(query, models.Query.outdated_queries())
|
2016-11-23 18:35:18 +00:00
|
|
|
db.session.flush()
|
2014-09-25 14:42:33 +00:00
|
|
|
query.archive()
|
|
|
|
|
2016-12-08 14:07:25 +00:00
|
|
|
self.assertNotIn(query, list(models.Query.all_queries([g.id for g in groups])))
|
2014-09-25 14:42:33 +00:00
|
|
|
self.assertNotIn(query, models.Query.outdated_queries())
|
|
|
|
|
|
|
|
def test_removes_associated_widgets_from_dashboards(self):
|
2015-12-01 13:44:08 +00:00
|
|
|
widget = self.factory.create_widget()
|
2016-11-29 15:48:39 +00:00
|
|
|
query = widget.visualization.query_rel
|
2016-11-23 18:35:18 +00:00
|
|
|
db.session.commit()
|
2014-09-25 14:42:33 +00:00
|
|
|
query.archive()
|
2016-11-23 18:35:18 +00:00
|
|
|
db.session.flush()
|
2019-01-07 08:30:42 +00:00
|
|
|
self.assertEqual(models.Widget.query.get(widget.id), None)
|
2014-09-25 14:42:33 +00:00
|
|
|
|
|
|
|
def test_removes_scheduling(self):
|
2019-12-11 11:54:29 +00:00
|
|
|
query = self.factory.create_query(
|
|
|
|
schedule={"interval": "1", "until": None, "time": None, "day_of_week": None}
|
|
|
|
)
|
2014-09-25 14:42:33 +00:00
|
|
|
|
|
|
|
query.archive()
|
|
|
|
|
2019-01-18 09:30:45 +00:00
|
|
|
self.assertIsNone(query.schedule)
|
2014-09-25 14:42:33 +00:00
|
|
|
|
2016-06-14 08:09:35 +00:00
|
|
|
def test_deletes_alerts(self):
|
|
|
|
subscription = self.factory.create_alert_subscription()
|
2016-11-29 15:48:39 +00:00
|
|
|
query = subscription.alert.query_rel
|
2016-11-23 18:35:18 +00:00
|
|
|
db.session.commit()
|
2016-06-14 08:09:35 +00:00
|
|
|
query.archive()
|
2016-11-23 18:35:18 +00:00
|
|
|
db.session.flush()
|
2019-01-07 08:30:42 +00:00
|
|
|
self.assertEqual(models.Alert.query.get(subscription.alert.id), None)
|
|
|
|
self.assertEqual(models.AlertSubscription.query.get(subscription.id), None)
|
2016-06-14 08:09:35 +00:00
|
|
|
|
|
|
|
|
2014-12-25 13:39:49 +00:00
|
|
|
class TestUnusedQueryResults(BaseTestCase):
|
|
|
|
def test_returns_only_unused_query_results(self):
|
2016-11-23 18:35:18 +00:00
|
|
|
two_weeks_ago = utcnow() - datetime.timedelta(days=14)
|
2015-12-01 13:44:08 +00:00
|
|
|
qr = self.factory.create_query_result()
|
2019-01-07 08:30:42 +00:00
|
|
|
self.factory.create_query(latest_query_data=qr)
|
2016-12-08 01:59:48 +00:00
|
|
|
db.session.flush()
|
2015-12-01 13:44:08 +00:00
|
|
|
unused_qr = self.factory.create_query_result(retrieved_at=two_weeks_ago)
|
2019-01-07 08:30:42 +00:00
|
|
|
self.assertIn(unused_qr, list(models.QueryResult.unused()))
|
|
|
|
self.assertNotIn(qr, list(models.QueryResult.unused()))
|
2014-12-25 13:39:49 +00:00
|
|
|
|
|
|
|
def test_returns_only_over_a_week_old_results(self):
|
2016-11-23 18:35:18 +00:00
|
|
|
two_weeks_ago = utcnow() - datetime.timedelta(days=14)
|
2015-12-01 13:44:08 +00:00
|
|
|
unused_qr = self.factory.create_query_result(retrieved_at=two_weeks_ago)
|
2016-12-08 01:59:48 +00:00
|
|
|
db.session.flush()
|
2015-12-01 13:44:08 +00:00
|
|
|
new_unused_qr = self.factory.create_query_result()
|
2014-12-25 13:39:49 +00:00
|
|
|
|
2019-01-07 08:30:42 +00:00
|
|
|
self.assertIn(unused_qr, list(models.QueryResult.unused()))
|
|
|
|
self.assertNotIn(new_unused_qr, list(models.QueryResult.unused()))
|
2014-12-25 13:39:49 +00:00
|
|
|
|
|
|
|
|
2015-12-01 13:44:08 +00:00
|
|
|
class TestQueryAll(BaseTestCase):
|
|
|
|
def test_returns_only_queries_in_given_groups(self):
|
|
|
|
ds1 = self.factory.create_data_source()
|
|
|
|
ds2 = self.factory.create_data_source()
|
|
|
|
|
2019-12-11 11:54:29 +00:00
|
|
|
group1 = models.Group(name="g1", org=ds1.org, permissions=["create", "view"])
|
|
|
|
group2 = models.Group(name="g2", org=ds1.org, permissions=["create", "view"])
|
2015-12-01 13:44:08 +00:00
|
|
|
|
|
|
|
q1 = self.factory.create_query(data_source=ds1)
|
|
|
|
q2 = self.factory.create_query(data_source=ds2)
|
|
|
|
|
2019-12-11 11:54:29 +00:00
|
|
|
db.session.add_all(
|
|
|
|
[
|
|
|
|
ds1,
|
|
|
|
ds2,
|
|
|
|
group1,
|
|
|
|
group2,
|
|
|
|
q1,
|
|
|
|
q2,
|
|
|
|
models.DataSourceGroup(group=group1, data_source=ds1),
|
|
|
|
models.DataSourceGroup(group=group2, data_source=ds2),
|
|
|
|
]
|
|
|
|
)
|
2016-11-23 18:35:18 +00:00
|
|
|
db.session.flush()
|
2016-12-08 14:07:25 +00:00
|
|
|
self.assertIn(q1, list(models.Query.all_queries([group1.id])))
|
|
|
|
self.assertNotIn(q2, list(models.Query.all_queries([group1.id])))
|
|
|
|
self.assertIn(q1, list(models.Query.all_queries([group1.id, group2.id])))
|
|
|
|
self.assertIn(q2, list(models.Query.all_queries([group1.id, group2.id])))
|
2015-12-01 13:44:08 +00:00
|
|
|
|
2017-01-24 19:46:02 +00:00
|
|
|
def test_skips_drafts(self):
|
|
|
|
q = self.factory.create_query(is_draft=True)
|
|
|
|
self.assertNotIn(q, models.Query.all_queries([self.factory.default_group.id]))
|
|
|
|
|
|
|
|
def test_includes_drafts_of_given_user(self):
|
|
|
|
q = self.factory.create_query(is_draft=True)
|
2019-12-11 11:54:29 +00:00
|
|
|
self.assertIn(
|
|
|
|
q,
|
|
|
|
models.Query.all_queries(
|
|
|
|
[self.factory.default_group.id], user_id=q.user_id
|
|
|
|
),
|
|
|
|
)
|
2017-01-24 19:46:02 +00:00
|
|
|
|
2018-07-18 19:24:31 +00:00
|
|
|
def test_order_by_relationship(self):
|
2019-12-11 11:54:29 +00:00
|
|
|
u1 = self.factory.create_user(name="alice")
|
|
|
|
u2 = self.factory.create_user(name="bob")
|
2018-07-18 19:24:31 +00:00
|
|
|
self.factory.create_query(user=u1)
|
|
|
|
self.factory.create_query(user=u2)
|
|
|
|
db.session.commit()
|
|
|
|
# have to reset the order here with None since all_queries orders by
|
|
|
|
# created_at by default
|
|
|
|
base = models.Query.all_queries([self.factory.default_group.id]).order_by(None)
|
|
|
|
qs1 = base.order_by(models.User.name)
|
2019-12-11 11:54:29 +00:00
|
|
|
self.assertEqual(["alice", "bob"], [q.user.name for q in qs1])
|
2018-07-18 19:24:31 +00:00
|
|
|
qs2 = base.order_by(models.User.name.desc())
|
2019-12-11 11:54:29 +00:00
|
|
|
self.assertEqual(["bob", "alice"], [q.user.name for q in qs2])
|
2018-07-18 19:24:31 +00:00
|
|
|
|
2015-12-01 13:44:08 +00:00
|
|
|
|
2016-05-17 19:01:18 +00:00
|
|
|
class TestGroup(BaseTestCase):
|
|
|
|
def test_returns_groups_with_specified_names(self):
|
|
|
|
org1 = self.factory.create_org()
|
|
|
|
org2 = self.factory.create_org()
|
|
|
|
|
2016-11-23 18:35:18 +00:00
|
|
|
matching_group1 = models.Group(id=999, name="g1", org=org1)
|
|
|
|
matching_group2 = models.Group(id=888, name="g2", org=org1)
|
|
|
|
non_matching_group = models.Group(id=777, name="g1", org=org2)
|
2016-05-17 19:01:18 +00:00
|
|
|
|
|
|
|
groups = models.Group.find_by_name(org1, ["g1", "g2"])
|
|
|
|
self.assertIn(matching_group1, groups)
|
|
|
|
self.assertIn(matching_group2, groups)
|
|
|
|
self.assertNotIn(non_matching_group, groups)
|
|
|
|
|
|
|
|
def test_returns_no_groups(self):
|
|
|
|
org1 = self.factory.create_org()
|
|
|
|
|
2016-11-23 18:35:18 +00:00
|
|
|
models.Group(id=999, name="g1", org=org1)
|
2016-05-17 19:01:18 +00:00
|
|
|
self.assertEqual([], models.Group.find_by_name(org1, ["non-existing"]))
|
|
|
|
|
|
|
|
|
2014-05-16 14:56:04 +00:00
|
|
|
class TestQueryResultStoreResult(BaseTestCase):
|
|
|
|
def setUp(self):
|
|
|
|
super(TestQueryResultStoreResult, self).setUp()
|
2015-12-01 13:44:08 +00:00
|
|
|
self.data_source = self.factory.data_source
|
2014-05-16 14:56:04 +00:00
|
|
|
self.query = "SELECT 1"
|
|
|
|
self.query_hash = gen_query_hash(self.query)
|
|
|
|
self.runtime = 123
|
2015-06-03 04:58:28 +00:00
|
|
|
self.utcnow = utcnow()
|
2019-10-10 07:39:55 +00:00
|
|
|
self.data = '{"a": 1}'
|
2014-05-16 14:56:04 +00:00
|
|
|
|
|
|
|
def test_stores_the_result(self):
|
2019-10-10 07:39:55 +00:00
|
|
|
query_result = models.QueryResult.store_result(
|
2019-12-11 11:54:29 +00:00
|
|
|
self.data_source.org_id,
|
|
|
|
self.data_source,
|
|
|
|
self.query_hash,
|
|
|
|
self.query,
|
|
|
|
self.data,
|
|
|
|
self.runtime,
|
|
|
|
self.utcnow,
|
|
|
|
)
|
2014-05-16 14:56:04 +00:00
|
|
|
|
2019-10-10 07:39:55 +00:00
|
|
|
self.assertEqual(query_result._data, self.data)
|
2014-05-16 14:56:04 +00:00
|
|
|
self.assertEqual(query_result.runtime, self.runtime)
|
|
|
|
self.assertEqual(query_result.retrieved_at, self.utcnow)
|
2016-11-30 14:12:53 +00:00
|
|
|
self.assertEqual(query_result.query_text, self.query)
|
2014-05-16 14:56:04 +00:00
|
|
|
self.assertEqual(query_result.query_hash, self.query_hash)
|
|
|
|
self.assertEqual(query_result.data_source, self.data_source)
|
|
|
|
|
2014-09-27 14:41:50 +00:00
|
|
|
|
|
|
|
class TestEvents(BaseTestCase):
|
|
|
|
def raw_event(self):
|
|
|
|
timestamp = 1411778709.791
|
2015-12-01 13:44:08 +00:00
|
|
|
user = self.factory.user
|
2014-09-27 14:41:50 +00:00
|
|
|
created_at = datetime.datetime.utcfromtimestamp(timestamp)
|
2016-11-23 18:35:18 +00:00
|
|
|
db.session.flush()
|
2019-12-11 11:54:29 +00:00
|
|
|
raw_event = {
|
|
|
|
"action": "view",
|
|
|
|
"timestamp": timestamp,
|
|
|
|
"object_type": "dashboard",
|
|
|
|
"user_id": user.id,
|
|
|
|
"object_id": 1,
|
|
|
|
"org_id": 1,
|
|
|
|
}
|
2014-09-27 14:41:50 +00:00
|
|
|
|
|
|
|
return raw_event, user, created_at
|
|
|
|
|
|
|
|
def test_records_event(self):
|
|
|
|
raw_event, user, created_at = self.raw_event()
|
|
|
|
|
|
|
|
event = models.Event.record(raw_event)
|
2016-11-23 18:35:18 +00:00
|
|
|
db.session.flush()
|
2014-09-27 14:41:50 +00:00
|
|
|
self.assertEqual(event.user, user)
|
|
|
|
self.assertEqual(event.action, "view")
|
|
|
|
self.assertEqual(event.object_type, "dashboard")
|
|
|
|
self.assertEqual(event.object_id, 1)
|
|
|
|
self.assertEqual(event.created_at, created_at)
|
|
|
|
|
|
|
|
def test_records_additional_properties(self):
|
|
|
|
raw_event, _, _ = self.raw_event()
|
2019-12-11 11:54:29 +00:00
|
|
|
additional_properties = {"test": 1, "test2": 2, "whatever": "abc"}
|
2014-09-27 14:41:50 +00:00
|
|
|
raw_event.update(additional_properties)
|
|
|
|
|
|
|
|
event = models.Event.record(raw_event)
|
|
|
|
|
2017-02-02 08:39:21 +00:00
|
|
|
self.assertDictEqual(event.additional_properties, additional_properties)
|
2014-09-25 14:42:33 +00:00
|
|
|
|
|
|
|
|
2016-03-29 16:27:35 +00:00
|
|
|
def _set_up_dashboard_test(d):
|
2019-12-11 11:54:29 +00:00
|
|
|
d.g1 = d.factory.create_group(name="First", permissions=["create", "view"])
|
|
|
|
d.g2 = d.factory.create_group(name="Second", permissions=["create", "view"])
|
2016-03-29 16:27:35 +00:00
|
|
|
d.ds1 = d.factory.create_data_source()
|
|
|
|
d.ds2 = d.factory.create_data_source()
|
2016-11-23 18:35:18 +00:00
|
|
|
db.session.flush()
|
2016-11-21 15:34:44 +00:00
|
|
|
d.u1 = d.factory.create_user(group_ids=[d.g1.id])
|
|
|
|
d.u2 = d.factory.create_user(group_ids=[d.g2.id])
|
2019-12-11 11:54:29 +00:00
|
|
|
db.session.add_all(
|
|
|
|
[
|
|
|
|
models.DataSourceGroup(group=d.g1, data_source=d.ds1),
|
|
|
|
models.DataSourceGroup(group=d.g2, data_source=d.ds2),
|
|
|
|
]
|
|
|
|
)
|
2016-03-29 16:27:35 +00:00
|
|
|
d.q1 = d.factory.create_query(data_source=d.ds1)
|
|
|
|
d.q2 = d.factory.create_query(data_source=d.ds2)
|
2016-11-29 15:48:39 +00:00
|
|
|
d.v1 = d.factory.create_visualization(query_rel=d.q1)
|
|
|
|
d.v2 = d.factory.create_visualization(query_rel=d.q2)
|
2016-03-29 16:27:35 +00:00
|
|
|
d.w1 = d.factory.create_widget(visualization=d.v1)
|
|
|
|
d.w2 = d.factory.create_widget(visualization=d.v2)
|
2016-04-20 15:30:31 +00:00
|
|
|
d.w3 = d.factory.create_widget(visualization=d.v2, dashboard=d.w2.dashboard)
|
|
|
|
d.w4 = d.factory.create_widget(visualization=d.v2)
|
|
|
|
d.w5 = d.factory.create_widget(visualization=d.v1, dashboard=d.w4.dashboard)
|
2016-11-29 15:48:39 +00:00
|
|
|
d.w1.dashboard.is_draft = False
|
|
|
|
d.w2.dashboard.is_draft = False
|
|
|
|
d.w4.dashboard.is_draft = False
|
2016-03-29 16:27:35 +00:00
|
|
|
|
2019-01-07 08:30:42 +00:00
|
|
|
|
2016-03-29 16:27:35 +00:00
|
|
|
class TestDashboardAll(BaseTestCase):
|
|
|
|
def setUp(self):
|
|
|
|
super(TestDashboardAll, self).setUp()
|
|
|
|
_set_up_dashboard_test(self)
|
|
|
|
|
|
|
|
def test_requires_group_or_user_id(self):
|
|
|
|
d1 = self.factory.create_dashboard()
|
2019-12-11 11:54:29 +00:00
|
|
|
self.assertNotIn(
|
|
|
|
d1, list(models.Dashboard.all(d1.user.org, d1.user.group_ids, None))
|
|
|
|
)
|
|
|
|
l2 = list(models.Dashboard.all(d1.user.org, [0], d1.user.id))
|
2016-11-23 18:35:18 +00:00
|
|
|
self.assertIn(d1, l2)
|
2016-03-29 16:27:35 +00:00
|
|
|
|
|
|
|
def test_returns_dashboards_based_on_groups(self):
|
2019-12-11 11:54:29 +00:00
|
|
|
self.assertIn(
|
|
|
|
self.w1.dashboard,
|
|
|
|
list(models.Dashboard.all(self.u1.org, self.u1.group_ids, None)),
|
|
|
|
)
|
|
|
|
self.assertIn(
|
|
|
|
self.w2.dashboard,
|
|
|
|
list(models.Dashboard.all(self.u2.org, self.u2.group_ids, None)),
|
|
|
|
)
|
|
|
|
self.assertNotIn(
|
|
|
|
self.w1.dashboard,
|
|
|
|
list(models.Dashboard.all(self.u2.org, self.u2.group_ids, None)),
|
|
|
|
)
|
|
|
|
self.assertNotIn(
|
|
|
|
self.w2.dashboard,
|
|
|
|
list(models.Dashboard.all(self.u1.org, self.u1.group_ids, None)),
|
|
|
|
)
|
2016-03-29 16:27:35 +00:00
|
|
|
|
2016-04-20 15:30:31 +00:00
|
|
|
def test_returns_each_dashboard_once(self):
|
2016-11-23 18:35:18 +00:00
|
|
|
dashboards = list(models.Dashboard.all(self.u2.org, self.u2.group_ids, None))
|
2016-04-20 15:30:31 +00:00
|
|
|
self.assertEqual(len(dashboards), 2)
|
|
|
|
|
|
|
|
def test_returns_dashboard_you_have_partial_access_to(self):
|
2019-12-11 11:54:29 +00:00
|
|
|
self.assertIn(
|
|
|
|
self.w5.dashboard,
|
|
|
|
models.Dashboard.all(self.u1.org, self.u1.group_ids, None),
|
|
|
|
)
|
2016-04-20 15:30:31 +00:00
|
|
|
|
2016-03-29 16:27:35 +00:00
|
|
|
def test_returns_dashboards_created_by_user(self):
|
|
|
|
d1 = self.factory.create_dashboard(user=self.u1)
|
2016-11-23 18:35:18 +00:00
|
|
|
db.session.flush()
|
2019-12-11 11:54:29 +00:00
|
|
|
self.assertIn(
|
|
|
|
d1, list(models.Dashboard.all(self.u1.org, self.u1.group_ids, self.u1.id))
|
|
|
|
)
|
2016-11-23 18:35:18 +00:00
|
|
|
self.assertIn(d1, list(models.Dashboard.all(self.u1.org, [0], self.u1.id)))
|
2019-12-11 11:54:29 +00:00
|
|
|
self.assertNotIn(
|
|
|
|
d1, list(models.Dashboard.all(self.u2.org, self.u2.group_ids, self.u2.id))
|
|
|
|
)
|
2016-03-29 16:27:35 +00:00
|
|
|
|
|
|
|
def test_returns_dashboards_with_text_widgets(self):
|
|
|
|
w1 = self.factory.create_widget(visualization=None)
|
|
|
|
|
2019-12-11 11:54:29 +00:00
|
|
|
self.assertIn(
|
|
|
|
w1.dashboard, models.Dashboard.all(self.u1.org, self.u1.group_ids, None)
|
|
|
|
)
|
|
|
|
self.assertIn(
|
|
|
|
w1.dashboard, models.Dashboard.all(self.u2.org, self.u2.group_ids, None)
|
|
|
|
)
|
2016-05-04 13:32:49 +00:00
|
|
|
|
|
|
|
def test_returns_dashboards_from_current_org_only(self):
|
|
|
|
w1 = self.factory.create_widget(visualization=None)
|
|
|
|
|
|
|
|
user = self.factory.create_user(org=self.factory.create_org())
|
|
|
|
|
2019-12-11 11:54:29 +00:00
|
|
|
self.assertIn(
|
|
|
|
w1.dashboard, models.Dashboard.all(self.u1.org, self.u1.group_ids, None)
|
|
|
|
)
|
|
|
|
self.assertNotIn(
|
|
|
|
w1.dashboard, models.Dashboard.all(user.org, user.group_ids, None)
|
|
|
|
)
|