Integration of visualizations module into ATC

Makefile

@@ -1,10 +1,11 @@
-.PHONY: all analytics navigator elastic setup clean
+.PHONY: all analytics navigator elastic setup clean visualizations
 
 all: setup_repo markdown confluence analytics navigator elastic
 analytics: create_analytics_and_pivoting_csv
 navigator: create_attack_navigator_profile create_attack_navigator_profile_per_customer
 elastic: create_es_export
 setup: setup_repo setup_confluence setup_markdown
+visualizations: make_visualizations
 
 setup_repo:
 	@echo "[*] Updating 3rd party repository"
@@ -52,6 +53,14 @@ create_es_export:
 	@echo "[*] Creating elastic index"
 	@cd scripts && python3 es_index_export.py
 
+make_visualizations:
+	@echo "[*] Creating visualizations.."
+ifeq ($(GUI), 1)
+	@cd scripts && python3 main.py -V --vis-export-type
+else
+	@cd scripts && python3 main.py -V
+endif
+
 # TODO: make clean works with non default paths from config
 clean:
 	@echo "[*] Cleaning up..."

analytics/generated/visualizations/README.md

@@ -0,0 +1,3 @@
+# Visulisations
+
+This is a directory where all generated visualisations will be saved by default. No matter if it's a dashboard, visualisation or other entity.

requirements.txt

@@ -2,4 +2,6 @@ pyyaml
 requests
 jinja2
 elasticsearch
-sanitize
+sanitize
+random
+string

scripts/atc_visualizations/README.md

@@ -1,6 +1,7 @@
 # Menu
 
 * [Structure](#structure)
+* [How to run](#how-to-run)
 * [Tips](#tips)
 * [Saved Search](#saved-search)
 * [Visualizations](#visualizations)
@@ -8,7 +9,37 @@
 
 # Structure
 
-Files consist of both required and optional fields. The structure depends on the object you want to define. All the visualizations and saved searches have to be inside `visualizations` directory. Dashboards can be anywhere but to keep it simple, just put it in `dashboards` directory.
+Files consist of both required and optional fields. The structure depends on the object you want to define. All the visualizations and saved searches have to be inside `${ATC}/visualizations/visualizations` directory. Dashboards have to be in `${ATC}/visualizations/dashboards` directory. By default, outputs will be saved to `${ATC}/analytics/generated/visualizations/`.
+
+# How to run
+
+## Curl / API
+
+Run the following command in the `${ATC}` directory:
+
+`make visualizations`
+
+Define variables:
+
+```bash
+KIBANA_URL="http://<kibana ip/domain>:<kibana port>"
+USER=""
+PASSWORD=""
+
+```
+
+Then you can use following curl:
+
+```bash
+curl -k --user ${USER}:${PASSWORD} -H "Content-Type: application/json"\
+  -H "kbn-xsrf: true"\
+  -XPOST "${KIBANA_URL}/api/kibana/dashboards/import?exclude=index-pattern&force=true"\
+  -d@analytics/generated/visualizations/${FILENAME}.json
+```
+
+## WebUI / GUI
+
+
 
 # Tips
 

scripts/atc_visualizations/aggs.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-from base import BaseKibanaAgg
+from atc_visualizations.base import BaseKibanaAgg
 
 # ########################################################################### #
 # ############################ Aggs ######################################### #

scripts/atc_visualizations/dashboard.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-import base
+import atc_visualizations.base as base
 import json
 import uuid
 

scripts/atc_visualizations/metrics.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
-import aggs
-import params
+import atc_visualizations.aggs as aggs
+import atc_visualizations.params as params
 
 # ########################################################################### #
 # ############################ Metrics ###################################### #

scripts/atc_visualizations/params.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-from base import BaseKibanaSeriesParams
+from atc_visualizations.base import BaseKibanaSeriesParams
 
 # ########################################################################### #
 # ############################ Params ####################################### #

scripts/atc_visualizations/visualisation.py

@@ -1,10 +1,10 @@
 #!/usr/bin/env python3
 
-import base
+import atc_visualizations.base as base
 import json
 import uuid
 
-from metrics import BaseMetric
+from atc_visualizations.metrics import BaseMetric
 from ast import literal_eval
 
 # ########################################################################### #

scripts/atc_visualizations/yaml_handler.py

@@ -1,12 +1,12 @@
 #!/usr/bin/env python3
 
 import yaml
-import visualisation
-import metrics
+import atc_visualizations.visualisation as visualisation
+import atc_visualizations.metrics as metrics
 import argparse
 import json
-import dashboard
-import base
+import atc_visualizations.dashboard as dashboard
+import atc_visualizations.base as base
 import uuid
 
 from yaml.scanner import ScannerError
@@ -39,7 +39,8 @@ def read_yaml_file(path):
 class YamlHandler(base.BaseKibana):
     """YamlHandler class"""
 
-    def __init__(self, yaml_path, output_file, omit_kibana, export_type):
+    def __init__(self, yaml_path, output_file, omit_kibana, export_type,
+                 vis_path="../visualizations/visualizations/"):
         self._export_type = export_type
         if omit_kibana:
             self.omit_kibana()
@@ -68,11 +69,11 @@ class YamlHandler(base.BaseKibana):
             "ip_range", "range", "significant_terms", "terms"
         ]
 
-        self.iter_over_yamls()
+        self.iter_over_yamls(vis_path=vis_path)
         with open(output_file, 'w') as f:
             json.dump(self._results, f)
 
-    def iter_over_yamls(self):
+    def iter_over_yamls(self, vis_path):
         for yaml_document in self.yamls:
             _type = yaml_document.get('type')
             if not _type:
@@ -86,7 +87,7 @@ class YamlHandler(base.BaseKibana):
             if _type == "visualization":
                 self.visualization_f(yaml_document)
             elif _type == "dashboard":
-                self.dashboard(yaml_document)
+                self.dashboard(yaml_document, vis_path=vis_path)
             elif _type == "search":
                 self.search_f(yaml_document)
             else:
@@ -212,7 +213,7 @@ class YamlHandler(base.BaseKibana):
             else:
                 vis.disable_labels()
 
-    def dashboard(self, yaml_document):
+    def dashboard(self, yaml_document, vis_path):
         if not yaml_document.get('visualizations'):
             raise Exception("No visualizations, no sense. Provide it!")
 
@@ -234,7 +235,7 @@ class YamlHandler(base.BaseKibana):
         if yaml_document.get('darktheme'):
             _dashboard.set_dark_theme()
 
-        vis_list = self.load_yamls("visualizations")
+        vis_list = self.load_yamls(vis_path)
         _vis_objects_dict = {}
 
         self._w, self._h, self._x, self._y = 15, 15, 0, 0
@@ -529,9 +530,10 @@ def main():
     parser.add_argument('-e', help="JSON export type [api/gui]",
                         required=False, default="api", const="gui",
                         action="store_const")
+    parser.add_argument('--vis-output', help="Provide where to save output " +
+                        "for visualisations module")
 
     args = parser.parse_args()
-
     YamlHandler(args.i, args.o, args.f, args.e)
 
 

scripts/main.py

@@ -6,24 +6,35 @@ from populateconfluence import PopulateConfluence
 # For confluence
 from requests.auth import HTTPBasicAuth
 
+from atcutils import ATCutils
+
+from atc_visualizations.yaml_handler import YamlHandler
+
 # Others
 import argparse
 import getpass
+import random
+import string
+import os
 
 if __name__ == '__main__':
 
-    parser = argparse.ArgumentParser(description='Main function of ATC. ' +
-                                     'This function is handling generating' +
-                                     'markdown files and/or ' +
-                                     'populating confluence')
+    parser = argparse.ArgumentParser(
+        description="""Main function of ATC.
+
+    You can not only choose to export analytics but also to use different
+    modules.
+""")
 
     # Mutually exclusive group for chosing the output of the script
     group = parser.add_mutually_exclusive_group(required=True)
 
     group.add_argument('-C', '--confluence', action='store_true',
-                       help='Set the output to be a Confluence')
+                       help='Export analytics to Confluence')
     group.add_argument('-M', '--markdown', action='store_true',
-                       help='Set the output to be markdown files')
+                       help='Export analytics to Markdown repository')
+    group.add_argument('-V', '--visualisations', action='store_true',
+                       help='Use visualisations module')
 
     # Mutually exclusive group for chosing type of data
     group2 = parser.add_mutually_exclusive_group(required=False)
@@ -51,6 +62,30 @@ if __name__ == '__main__':
     parser.add_argument('-i', '--init', action='store_true',
                         help="Build initial pages or directories " +
                         "depending on the export type")
+    # Input
+    parser.add_argument('--vis-input', help="Provide input file for " +
+                        "visualisations module")
+    # Output
+    parser.add_argument('--vis-output-dir', help="""
+    Provide directory path where to save output for visualisations module.
+    Default is created by joining exported_analytics_directory field from
+    config file with `dashboards` directory, so in the end it is:
+
+        ${exported_analytics_directory}/dashboards/
+""")
+    parser.add_argument('--vis-output-file-name', help="Provide file name " +
+                        "which will be used to save a file in output " +
+                        "directory\nDefault is: [randomstring].yml")
+    # Force
+    parser.add_argument('--vis-force', action='store_true',
+                        help="Force visualisations module to not use Kibana")
+
+    # Export type
+    parser.add_argument('--vis-export-type', help="Switch JSON export type " +
+                        "from api (uploaded using curl) to gui (imported in " +
+                        "kibana)", required=False, default="api", const="gui",
+                        action="store_const")
+
     args = parser.parse_args()
 
     if args.markdown:
@@ -73,3 +108,47 @@ if __name__ == '__main__':
                            tg=args.triggers, en=args.enrichment,
                            ra=args.responseactions, rp=args.responseplaybook,
                            cu=args.customers, init=args.init)
+
+    elif args.visualisations:
+        ATCconfig = ATCutils.load_config("config.yml")
+        ATCconfig_default = ATCutils.load_config("config.default.yml")
+        if not args.vis_output_dir:
+            analytics_generated = ATCconfig.get(
+                "exported_analytics_directory",
+                ATCconfig_default.get("exported_analytics_directory")
+            )
+            analytics_generated = analytics_generated if \
+                analytics_generated[-1] == "/" else analytics_generated + "/"
+            output_path = analytics_generated + "visualizations/"
+
+            if not args.vis_output_file_name:
+                output_name = ''.join(
+                    random.choices(
+                        string.ascii_uppercase + string.ascii_lowercase +
+                        string.digits, k=20)
+                )
+                # output_name += ".json"
+            else:
+                output_name = args.vis_output_file_name
+            output_path += output_name
+
+        else:
+            analytics_generated = args.vis_output_dir if \
+                args.vis_output_dir[-1] == "/" else args.vis_output_dir + "/"
+            output_path = analytics_generated
+
+        dashboard_path = "../visualizations/dashboards/"
+
+        if not args.vis_input:
+            for file in os.listdir(dashboard_path):
+                if not file.endswith((".yml", ".yaml")):
+                    continue
+                YamlHandler(dashboard_path + file, output_path + "_" +
+                            file[:-4] + ".json", args.vis_force,
+                            args.vis_export_type)
+                print("File path: %s" % (output_path + "_" +
+                      file[:-4] + ".json"))
+        else:
+            YamlHandler(args.vis_input, output_path + ".json", args.vis_force,
+                        args.vis_export_type)
+            print("File path: %s" % (output_path + ".json"))