Move event recording to Celery/database instead of log file

manage.py

@@ -42,6 +42,7 @@ def check_settings():
 
 @manager.command
 def import_events(events_file):
+    # TODO: remove this code past 1/11/2014.
     import json
     from collections import Counter
 

redash/__init__.py

@@ -3,7 +3,7 @@ import urlparse
 import redis
 from statsd import StatsClient
 
-from redash import settings, events
+from redash import settings
 
 __version__ = '0.4.0'
 
@@ -15,8 +15,6 @@ def setup_logging():
     logging.getLogger().addHandler(handler)
     logging.getLogger().setLevel(settings.LOG_LEVEL)
 
-    events.setup_logging(settings.EVENTS_LOG_PATH, settings.EVENTS_CONSOLE_OUTPUT)
-
 
 def create_redis_connection():
     redis_url = urlparse.urlparse(settings.REDIS_URL)

redash/controllers.py

@@ -10,23 +10,20 @@ import json
 import numbers
 import cStringIO
 import datetime
+import logging
 
 from flask import render_template, send_from_directory, make_response, request, jsonify, redirect, \
     session, url_for
 from flask.ext.restful import Resource, abort
 from flask_login import current_user, login_user, logout_user
-
 import sqlparse
-import events
-from permissions import require_permission
 
 from redash import redis_connection, statsd_client, models, settings, utils, __version__
 from redash.wsgi import app, auth, api
+from redash.tasks import QueryTask, record_event
+from redash.cache import headers as cache_headers
+from redash.permissions import require_permission
 
-import logging
-from tasks import QueryTask
-
-from cache import headers as cache_headers
 
 @app.route('/ping', methods=['GET'])
 def ping():
@@ -158,7 +155,7 @@ class EventAPI(BaseResource):
     def post(self):
         events_list = request.get_json(force=True)
         for event in events_list:
-            events.record_event(event)
+            record_event.delay(event)
 
 
 api.add_resource(EventAPI, '/api/events', endpoint='events')

redash/events.py

@@ -1,23 +0,0 @@
-import logging
-import json
-
-logger = logging.getLogger("redash.events")
-logger.propagate = False
-
-
-def setup_logging(log_path, console_output=False):
-    if log_path:
-        fh = logging.FileHandler(log_path)
-        formatter = logging.Formatter('%(message)s')
-        fh.setFormatter(formatter)
-        logger.addHandler(fh)
-
-    if console_output:
-        handler = logging.StreamHandler()
-        formatter = logging.Formatter('[%(name)s] %(message)s')
-        handler.setFormatter(formatter)
-        logger.addHandler(handler)
-
-
-def record_event(event):
-    logger.info(json.dumps(event))

redash/models.py

@@ -520,6 +520,21 @@ class Event(BaseModel):
     def __unicode__(self):
         return u"%s,%s,%s,%s" % (self._data['user'], self.action, self.object_type, self.object_id)
 
+    @classmethod
+    def record(cls, event):
+        user = event.pop('user_id')
+        action = event.pop('action')
+        object_type = event.pop('object_type')
+        object_id = event.pop('object_id', None)
+
+        created_at = datetime.datetime.utcfromtimestamp(event.pop('timestamp'))
+        additional_properties = json.dumps(event)
+
+        event = cls.create(user=user, action=action, object_type=object_type, object_id=object_id,
+                           additional_properties=additional_properties, created_at=created_at)
+
+        return event
+
 
 all_models = (DataSource, User, QueryResult, Query, Dashboard, Visualization, Widget, ActivityLog, Group, Event)
 

redash/settings.py

@@ -70,8 +70,6 @@ WORKERS_COUNT = int(os.environ.get("REDASH_WORKERS_COUNT", "2"))
 JOB_EXPIRY_TIME = int(os.environ.get("REDASH_JOB_EXPIRY_TIME", 3600*6))
 COOKIE_SECRET = os.environ.get("REDASH_COOKIE_SECRET", "c292a0a3aa32397cdb050e233733900f")
 LOG_LEVEL = os.environ.get("REDASH_LOG_LEVEL", "INFO")
-EVENTS_LOG_PATH = os.environ.get("REDASH_EVENTS_LOG_PATH", "")
-EVENTS_CONSOLE_OUTPUT = parse_boolean(os.environ.get("REDASH_EVENTS_CONSOLE_OUTPUT", "false"))
 CLIENT_SIDE_METRICS = parse_boolean(os.environ.get("REDASH_CLIENT_SIDE_METRICS", "false"))
 ANALYTICS = os.environ.get("REDASH_ANALYTICS", "")
 

redash/tasks.py

@@ -246,3 +246,7 @@ def execute_query(self, query, data_source_id):
 
     return query_result.id
 
+
+@celery.task(base=BaseTask)
+def record_event(event):
+    models.Event.record(event)

tests/test_models.py

@@ -1,7 +1,8 @@
 import datetime
+import json
 from tests import BaseTestCase
 from redash import models
-from factories import dashboard_factory, query_factory, data_source_factory, query_result_factory
+from factories import dashboard_factory, query_factory, data_source_factory, query_result_factory, user_factory
 from redash.utils import gen_query_hash
 
 
@@ -148,4 +149,38 @@ class TestQueryResultStoreResult(BaseTestCase):
 
         self.assertEqual(models.Query.get_by_id(query1.id)._data['latest_query_data'], query_result.id)
         self.assertEqual(models.Query.get_by_id(query2.id)._data['latest_query_data'], query_result.id)
-        self.assertNotEqual(models.Query.get_by_id(query3.id)._data['latest_query_data'], query_result.id)
+        self.assertNotEqual(models.Query.get_by_id(query3.id)._data['latest_query_data'], query_result.id)
+
+
+class TestEvents(BaseTestCase):
+    def raw_event(self):
+        timestamp = 1411778709.791
+        user = user_factory.create()
+        created_at = datetime.datetime.utcfromtimestamp(timestamp)
+        raw_event = {"action": "view",
+                      "timestamp": timestamp,
+                      "object_type": "dashboard",
+                      "user_id": user.id,
+                      "object_id": 1}
+
+        return raw_event, user, created_at
+
+    def test_records_event(self):
+        raw_event, user, created_at = self.raw_event()
+
+        event = models.Event.record(raw_event)
+
+        self.assertEqual(event.user, user)
+        self.assertEqual(event.action, "view")
+        self.assertEqual(event.object_type, "dashboard")
+        self.assertEqual(event.object_id, 1)
+        self.assertEqual(event.created_at, created_at)
+
+    def test_records_additional_properties(self):
+        raw_event, _, _ = self.raw_event()
+        additional_properties = {'test': 1, 'test2': 2, 'whatever': "abc"}
+        raw_event.update(additional_properties)
+
+        event = models.Event.record(raw_event)
+
+        self.assertDictEqual(json.loads(event.additional_properties), additional_properties)