Metric fully supported

scripts/KibanaStuff/TODO.md

@@ -1,26 +1,26 @@
 # Metric
 
-- show/hide label
-- only count
-- use internal search
+- [x] show/hide label
+- [x] only count (count was made sure is working fine)
+- [x] use internal search
 
 # Pie
 
-- only index name
-- use internal search
-- show labels
-- split slices (terms) - else default
+- [ ] only index name
+- [ ] use internal search
+- [ ] show labels
+- [ ] split slices (terms) - else default
 
 # Bars - vertical
 
-- X-axis - terms
-- split series - terms
-- else default
+- [ ] X-axis - terms
+- [ ] split series - terms
+- [ ] else default
 
 # Area
 
-- not focus
+- [x] not focus
 
 # Saved Search 
 
-- it's needed for dashboards (add as object inside dashboard)
+- [ ] it's needed for dashboards (add as object inside dashboard)

scripts/KibanaStuff/dashboard.py

@@ -15,7 +15,7 @@ class KibanaDashboardObject(base.BaseKibana):
         self.title = str()
         self.description = str()
         self.panelsJSON = list()  # double escaping
-        self.optionsJSON = str()  # double escaping
+        self.optionsJSON = dict()  # double escaping
         self.timeRestore = bool()
         self.kibanaSavedObjectMeta = dict()
         self.version = 1
@@ -25,13 +25,22 @@ class KibanaDashboardObject(base.BaseKibana):
         if title:
             self.title = title
 
-    #def validate(self):
-    #    # TODO: Write validate method
-    #    return True
+        self.optionsJSON = {'darkTheme': False}
+        self.kibanaSavedObjectMeta["searchSourceJSON"] = {
+            "query": {
+                "query": "",
+                "language": "lucene"
+            },
+            "filter": []
+        }
 
-    #def __call__(self):
-    #    if self.validate():
-    #        return self.__dict__
+    def validate(self):
+        # TODO: Write validate method
+        return True
+
+    def __call__(self):
+        if self.validate():
+            return self.__dict__
 
     def __repr__(self):
         return str(self.__call__())
@@ -42,20 +51,27 @@ class KibanaDashboardObject(base.BaseKibana):
         str_test = str(test)
         _tmp["_source"] = literal_eval(str_test)
         _tmp["_type"] = "dashboard"
-        # _tmp["version"] = "1"
         _tmp.pop("_id", None)
 
-        _tmp["_source"]["panelsJSON"] = json.dumps(_tmp["_source"]["panelsJSON"])
-        _tmp["_source"]["optionsJSON"] = json.dumps(_tmp["_source"]["optionsJSON"])
+        _tmp["_source"]["panelsJSON"] = json.dumps(
+            _tmp["_source"]["panelsJSON"]
+        )
+        _tmp["_source"]["optionsJSON"] = json.dumps(
+            _tmp["_source"]["optionsJSON"]
+        )
+        _tmp["_source"]["kibanaSavedObjectMeta"]["searchSourceJSON"] =\
+            json.dumps(
+                _tmp["_source"]["kibanaSavedObjectMeta"]["searchSourceJSON"]
+        )
         _tmp["_source"].pop("_id", None)
         _tmp["_id"] = str(uuid.uuid4())
-        
+
         if return_dict:
             return _tmp
         else:
             return json.dumps(_tmp)
 
-    def add_visualization(self, vis, x=0, y=0, w=1, h=1):
+    def add_visualization(self, vis, x=0, y=0, w=20, h=20):
         _vis = base.BasePanelsJson(vis_uuid=vis["uuid"])
         _vis.gridData.x = x
         _vis.gridData.y = y
@@ -63,7 +79,12 @@ class KibanaDashboardObject(base.BaseKibana):
         _vis.gridData.h = h
         _vis.gridData.i = self._id
         _vis.panelIndex = self._id
-        # print(type(self.panelsJSON))
         self.panelsJSON.append(_vis)
         self._id += 1
 
+    def set_dark_theme(self):
+        self.optionsJSON.update({'darkTheme': True})
+
+    def set_query(self, query):
+        ssjs = self.kibanaSavedObjectMeta["searchSourceJSON"]
+        ssjs["query"]["query"] = str(query)

scripts/KibanaStuff/test_dashboard_document.yml

@@ -2,10 +2,9 @@ type: dashboard
 name: test dashboard
 title: test dashboard
 darktheme: True
+query: port:123455667
 visualizations:
   - Count vis
   - Count vis
   - Count vis
   - Count vis
-  - Count vis
-  - Count vis

scripts/KibanaStuff/visualisation.py

@@ -25,6 +25,16 @@ class BaseKibanaVisualizationDoc(base.BaseKibanaDoc):
         self.visualization.visState = base.BaseKibanaVisState(
             title=title, type=type)
         self.visualization.visState.params = base.BaseKibanaParams(type=type)
+        self.visualization.kibanaSavedObjectMeta["searchSourceJSON"] = {
+            "index": "",
+            "query": {
+                "query_string": {
+                    "analyze_wildcard": True,
+                    "query": "*"
+                }
+            },
+            "filter": []
+        }
 
     def default_axis(self):
         # TODO: Make class for that as well as proper handling in the code
@@ -102,6 +112,12 @@ class BaseKibanaVisualizationDoc(base.BaseKibanaDoc):
             tmp_dictionary.pop("metric_id", None)
             tmp_dictionary.pop("updated_at", None)
             tmp_dictionary.pop("_meta_data_set", None)
+            kbsvd = tmp_dictionary["visualization"]["kibanaSavedObjectMeta"]
+            kbsvd["searchSourceJSON"] = json.dumps(
+                tmp_dictionary.get("visualization")
+                .get("kibanaSavedObjectMeta")
+                .get("searchSourceJSON")
+            )
             tmp_dictionary["_source"] = tmp_dictionary.pop("visualization")
             if return_dict:
                 return tmp_dictionary
@@ -115,10 +131,9 @@ class BaseKibanaVisualizationDoc(base.BaseKibanaDoc):
             if self.search_id_of_title_by_type(
                     search_type="index-pattern", search_title=index_name
             ):
-                self.visualization.kibanaSavedObjectMeta["searchSourceJSON"] =\
-                    ("{\"index\":\"%s\",\"query\":{\"query_string\":{" +
-                     "\"analyze_wildcard\":true,\"query\":\"*\"}}," +
-                     "\"filter\":[]}") % index_name
+                self.visualization\
+                    .kibanaSavedObjectMeta["searchSourceJSON"]["index"]\
+                    = index_name
                 self._meta_data_set = True
             else:
                 raise Exception(
@@ -126,10 +141,9 @@ class BaseKibanaVisualizationDoc(base.BaseKibanaDoc):
                     " Didn't you forget an asterisk?"
                 )
         else:
-            self.visualization.kibanaSavedObjectMeta["searchSourceJSON"] = \
-                ("{\"index\":\"%s\",\"query\":{\"query_string\":{" +
-                 "\"analyze_wildcard\":true,\"query\":\"*\"}}," +
-                 "\"filter\":[]}") % index_name
+            self.visualization\
+                .kibanaSavedObjectMeta["searchSourceJSON"]["index"]\
+                = index_name
             self._meta_data_set = True
 
     def set_saved_search(self, saved_search_name=None, saved_search_id=None):
@@ -150,11 +164,14 @@ class BaseKibanaVisualizationDoc(base.BaseKibanaDoc):
                 search_type="search", search_title=saved_search_name
             )
         self.visualization.savedSearchId = _id
-        self.visualization.kibanaSavedObjectMeta["searchSourceJSON"] = \
-            ("{\"query\":{\"query\":\"\",\"language\"" +
-             ":\"lucene\"},\"filter\":[]}")
+        self.visualization.kibanaSavedObjectMeta["searchSourceJSON"]\
+            .pop("index", None)
         self._meta_data_set = True
 
+    def set_query(self, query):
+        ssjs = self.visualization.kibanaSavedObjectMeta["searchSourceJSON"]
+        ssjs["query"]["query_string"]["query"] = str(query)
+
 # ########################################################################### #
 # ############################ Area Visualisation ########################### #
 # ########################################################################### #
@@ -172,12 +189,57 @@ class AreaVisualisation(BaseKibanaVisualizationDoc):
 # ############################ Metric Visualisation ######################### #
 # ########################################################################### #
 
+class MetricKibanaParams(base.BaseKibanaParams):
+
+    def __init__(self, type=None, grid=None, categoryAxes=None, valueAxes=None,
+                 seriesParams=None, addTooltip=None, addLegend=None,
+                 legendPosition=None, times=None, addTimeMarker=None):
+
+        super().__init__(
+            type=type, grid=grid, categoryAxes=categoryAxes,
+            valueAxes=valueAxes, seriesParams=seriesParams,
+            addTooltip=addTooltip, addLegend=addLegend,
+            legendPosition=legendPosition, times=times,
+            addTimeMarker=addTimeMarker)
+
+        self.metric = {
+            "percentageMode": False,
+            "useRanges": False,
+            "colorSchema": "Green to Red",
+            "metricColorMode": "None",
+            "colorsRange": [
+                {
+                    "from": 0,
+                    "to": 10000
+                }
+            ],
+            "labels": {
+                "show": True
+            },
+            "invertColors": False,
+            "style": {
+                "bgFill": "#000",
+                "bgColor": False,
+                "labelColor": False,
+                "subText": "",
+                "fontSize": 60
+            }
+        }
+
+    def disable_labels(self):
+        self.metric["labels"]["show"] = False
+
 
 class MetricVisualisation(BaseKibanaVisualizationDoc):
 
     def __init__(self, title):
 
         super().__init__(title=title, type="metric")
+        self.visualization.visState.params = MetricKibanaParams(type="metric")
+
+    def disable_labels(self):
+        self.visualization.visState.params.disable_labels()
+
 
 # ########################################################################### #
 # ############################ Pie Visualisation ############################ #

scripts/KibanaStuff/visualizations/test_visualisation_document.yml

@@ -2,9 +2,10 @@ type: visualization
 name: metric
 title: Count vis
 index_name: atc-analytics
-options:
-  add_metric:
-    - count
-    - average:
-        field: type
-        enabled: False
+query: dupa
+labels: False
+metrics:
+  - count
+  - average:
+      field: port
+      enabled: False

scripts/KibanaStuff/yaml_handler.py

@@ -6,9 +6,10 @@ import metrics
 import argparse
 import json
 import dashboard
-import base
+# import base
 import uuid
 
+from yaml.scanner import ScannerError
 from os import listdir
 from os.path import isfile, join
 
@@ -62,7 +63,6 @@ class YamlHandler:
 
     def iter_over_yamls(self):
         for yaml_document in self.yamls:
-            print(yaml_document)
             _type = yaml_document.get('type')
             if not _type:
                 raise Exception("Type not defined")
@@ -91,7 +91,6 @@ class YamlHandler:
   * saved_search_name
   * index_name
 """)
-        _options = yaml_document.get('options')
         if not _title:
             raise Exception("No title defined")
         if not self._name:
@@ -114,6 +113,13 @@ class YamlHandler:
         if not _vis:
             raise Exception("Unsupported or invalid visualisation")
 
+        if yaml_document.get('query'):
+            _vis.set_query(yaml_document.get('query'))
+
+        if 'labels' in yaml_document.keys():
+            if isinstance(yaml_document['labels'], bool):
+                self.vis_handle_labels(yaml_document['labels'], _vis)
+
         if _saved_search_name:
             _vis.set_saved_search(saved_search_name=_saved_search_name)
 
@@ -123,28 +129,31 @@ class YamlHandler:
         if _index_name:
             _vis.set_index_search(_index_name)
 
-        if _options:
-            for option in _options:
-                if option not in self._options:
-                    raise Exception("Not known option")
-                if option == "add_metric":
-                    for option_val in _options[option]:
-                        _metric = None
-                        if isinstance(option_val, str):
-                            _metric = self.handle_metric(
-                                _vis.metric_id, option_val
-                            )
-                        elif isinstance(option_val, dict)\
-                                and len(option_val) == 1:
-                            _option_metric_name = [x for x in option_val][0]
-                            _metric = self.handle_metric(
-                                _vis.metric_id, _option_metric_name,
-                                args=option_val[_option_metric_name]
-                            )
-                        if _metric:
-                            _vis.add_metric(_metric)
+        if yaml_document.get('metrics'):
+            for metric in yaml_document.get('metrics'):
+                _metric = None
+                if isinstance(metric, str):
+                    _metric = self.handle_metric(
+                        _vis.metric_id, metric
+                    )
+                elif isinstance(metric, dict)\
+                        and len(metric) == 1:
+                    _option_metric_name = [x for x in metric][0]
+                    _metric = self.handle_metric(
+                        _vis.metric_id, _option_metric_name,
+                        args=metric[_option_metric_name]
+                    )
+                if _metric:
+                    _vis.add_metric(_metric)
         self._results.append(_vis.json_export(return_dict=True, uuid_=uuid_))
-        # return _vis
+
+    def vis_handle_labels(self, bool_var, vis_):
+        if self._name == "metric":
+            if bool_var is False:
+                vis_.disable_labels()
+        else:
+            print("Warning! Can't disable labels for given visualization. " +
+                  "Not supported.")
 
     def dashboard(self, yaml_document):
         if not yaml_document.get('visualizations'):
@@ -161,22 +170,28 @@ class YamlHandler:
 
         _dashboard = dashboard.KibanaDashboardObject()
         _dashboard.title = _title
-        _dashboard.kibanaSavedObjectMeta = {
-          "searchSourceJSON": "{\"query\":{\"query\":\"\"," +
-            "\"language\":\"lucene\"},\"filter\":[]}"
-        }
-        _dashboard.optionsJSON = { "darkTheme": False }
 
-        visualization_objects_list = self.load_yamls("/Users/yugoslavskiy/Desktop/re_of_jk_fancy_classes/atomic-threat-coverage/scripts/KibanaStuff/visualizations")
+        if yaml_document.get('query'):
+            _dashboard.set_query(yaml_document.get('query'))
 
-        for visualization in visualization_objects_list:
-            if visualization['title'] not in yaml_document.get('visualizations'):
+        if yaml_document.get('darktheme'):
+            _dashboard.set_dark_theme()
+
+        vis_list = self.load_yamls("visualizations")
+        _vis_objects_dict = {}
+
+        for visualization in vis_list:
+            if visualization['title'] \
+                    not in yaml_document.get('visualizations'):
                 continue
-            _result = self.visualization_f(visualization, uuid_=visualization.get('uuid'))
-            _dashboard.add_visualization(visualization)
+            self.visualization_f(
+                visualization, uuid_=visualization.get('uuid')
+            )
+            _vis_objects_dict[visualization.get('title')] = visualization
+        for title in yaml_document.get('visualizations'):
+            _dashboard.add_visualization(_vis_objects_dict[title])
         self._results.append(_dashboard.json_export(return_dict=True))
 
-
     def handle_metric(self, id, metric_name, args=None):
         if metric_name not in self._general_metrics:
             raise Exception("Metric not supported")
@@ -382,8 +397,7 @@ class YamlHandler:
         yamls = [
             join(path, f) for f in listdir(path)
             if isfile(join(path, f))
-            if f.endswith('.yaml')
-            or f.endswith('.yml')
+            if f.endswith('.yaml') or f.endswith('.yml')
         ]
 
         result = []