Databricks Schema Browser: Allow eventlet worker instead of RQ (#5045)

* Add loading button in UI

* Handle databricks schema requests without RQ

* Don't use gevent worker

* Revert "Don't use gevent worker"

This reverts commit 9704c70a941a68c249db73e0450961e608fc0507.

* Use eventlet

* Use first column instead of 'namespace' one

* Revert "Add loading button in UI"

This reverts commit c0e4dfb966714a9f9e23977ab659e64afb5ce255.

* Remove databricks tasks

* Update eventlet

* Add libevent

* Display logs on failure

* Revert "Add libevent"

This reverts commit a00d067cb77b6f4f9919cf47f1d15c34d107a18c.

* Test updating gunicorn

* Don't set eventlet as the default for Redash

Co-authored-by: Arik Fraimovich <arik@arikfr.com>

* Remove fetchDataFromJob usage

Co-authored-by: Arik Fraimovich <arik@arikfr.com>

.circleci/config.yml

@@ -108,6 +108,11 @@ jobs:
       - run:
           name: Execute Cypress tests
           command: npm run cypress run-ci
+      - run:
+          name: "Failure: output container logs to console"
+          command: |
+            docker-compose logs
+          when: on_fail
   build-docker-image: *build-docker-image-job
   build-preview-docker-image: *build-docker-image-job
 workflows:

bin/docker-entrypoint

@@ -126,4 +126,3 @@ case "$1" in
     exec "$@"
     ;;
 esac
-

client/app/services/databricks-data-source.js

@@ -1,16 +1,22 @@
-import { has } from "lodash";
 import { axios } from "@/services/axios";
 import DataSource from "@/services/data-source";
-import { fetchDataFromJob } from "@/services/query-result";
 
 export default {
   ...DataSource,
-  getDatabases: ({ id }) =>
-    axios
-      .get(`api/databricks/databases/${id}`)
-      .then(data => (has(data, "job.id") ? fetchDataFromJob(data.job.id, 300).catch(() => []) : Promise.resolve([]))),
-  getDatabaseTables: (data, databaseName) =>
-    axios
-      .get(`api/databricks/databases/${data.id}/${databaseName}/tables`)
-      .then(data => (has(data, "job.id") ? fetchDataFromJob(data.job.id, 300).catch(() => []) : Promise.resolve([]))),
+  getDatabases: ({ id }, refresh = false) => {
+    const params = {};
+
+    if (refresh) {
+      params.refresh = true;
+    }
+    return axios.get(`api/databricks/databases/${id}`, { params });
+  },
+  getDatabaseTables: (data, databaseName, refresh = false) => {
+    const params = {};
+
+    if (refresh) {
+      params.refresh = true;
+    }
+    return axios.get(`api/databricks/databases/${data.id}/${databaseName}/tables`, { params });
+  },
 };

redash/handlers/databricks.py

@@ -5,7 +5,6 @@ from redash.permissions import (
     require_access,
     view_only,
 )
-from redash.tasks.databricks import get_databricks_databases, get_databricks_schema
 from redash.serializers import serialize_job
 
 
@@ -21,8 +20,10 @@ class DatabricksDatabaseListResource(BaseResource):
                 400, message="Resource only available for the Databricks query runner."
             )
 
-        job = get_databricks_databases.delay(data_source.id)
-        return serialize_job(job)
+        try:
+            return data_source.query_runner.get_databases()
+        except Exception:
+            return {"error": {"code": 2, "message": "Error retrieving schema."}}
 
 
 class DatabricksSchemaResource(BaseResource):
@@ -37,5 +38,7 @@ class DatabricksSchemaResource(BaseResource):
                 400, message="Resource only available for the Databricks query runner."
             )
 
-        job = get_databricks_schema.delay(data_source.id, database_name)
-        return serialize_job(job)
+        try:
+            return data_source.query_runner.get_database_schema(database_name)
+        except Exception:
+            return {"error": {"code": 2, "message": "Error retrieving schema."}}

redash/query_runner/databricks.py

@@ -141,7 +141,8 @@ class Databricks(BaseSQLQueryRunner):
 
         results = json_loads(results)
 
-        return [row["namespace"] for row in results["rows"]]
+        first_column_name = results["columns"][0]["name"]
+        return [row[first_column_name] for row in results["rows"]]
 
     def get_database_schema(self, database_name):
         schema = {}

redash/tasks/databricks.py

@@ -1,22 +0,0 @@
-from rq.registry import FailedJobRegistry
-from redash import models
-from redash.worker import job
-from redash.tasks.worker import Queue
-
-
-@job("schemas", queue_class=Queue, at_front=True, timeout=300, ttl=90)
-def get_databricks_databases(data_source_id):
-    try:
-        data_source = models.DataSource.get_by_id(data_source_id)
-        return data_source.query_runner.get_databases()
-    except Exception:
-        return {"error": {"code": 2, "message": "Error retrieving schema."}}
-
-
-@job("schemas", queue_class=Queue, at_front=True, timeout=300, ttl=90)
-def get_databricks_schema(data_source_id, database_name):
-    try:
-        data_source = models.DataSource.get_by_id(data_source_id)
-        return data_source.query_runner.get_database_schema(database_name)
-    except Exception:
-        return {"error": {"code": 2, "message": "Error retrieving schema."}}

requirements.txt

@@ -33,7 +33,7 @@ pyparsing==2.3.0
 SQLAlchemy-Utils==0.34.2
 sqlparse==0.3.0
 statsd==3.3.0
-gunicorn==19.9.0
+gunicorn==20.0.4
 rq==1.1.0
 rq-scheduler==0.9.1
 jsonschema==3.1.1
@@ -55,6 +55,7 @@ maxminddb-geolite2==2018.703
 pypd==1.1.0
 disposable-email-domains>=0.0.52
 gevent==1.4.0
+eventlet==0.25.2
 sshtunnel==0.1.5
 supervisor==4.1.0
 supervisor_checks==0.8.1