Added merge_sigma tool

* Tests
* Restructured Makefile

Makefile

@@ -1,12 +1,18 @@
 .PHONY: test test-yaml test-sigmac
 TMPOUT = $(shell tempfile)
-test: test-yaml test-sigmac
+test: clearcov test-yaml test-sigmac test-merge finish
+
+clearcov:
+	rm -f .coverage
+
+finish:
+	coverage report --fail-under=90
+	rm -f $(TMPOUT)
 
 test-yaml:
 	yamllint rules
 
 test-sigmac:
-	rm -f .coverage
 	coverage run -a --include=tools/* tools/sigmac.py -l
 	coverage run -a --include=tools/* tools/sigmac.py -rvdI -t es-qs rules/ > /dev/null
 	coverage run -a --include=tools/* tools/sigmac.py -rvdI -t kibana rules/ > /dev/null
@@ -23,7 +29,6 @@ test-sigmac:
 	coverage run -a --include=tools/* tools/sigmac.py -rvdI -c tools/config/elk-linux.yml -t es-qs rules/ > /dev/null
 	coverage run -a --include=tools/* tools/sigmac.py -rvdI -c tools/config/elk-windows.yml -t kibana rules/ > /dev/null
 	coverage run -a --include=tools/* tools/sigmac.py -rvdI -c tools/config/elk-linux.yml -t kibana rules/ > /dev/null
-	coverage run -a --include=tools/* tools/sigmac.py -rvdI -c tools/config/elk-defaultindex-filebeat.yml -t kibana rules/ > /dev/null
 	coverage run -a --include=tools/* tools/sigmac.py -rvdI -c tools/config/elk-windows.yml -t xpack-watcher rules/ > /dev/null
 	coverage run -a --include=tools/* tools/sigmac.py -rvdI -c tools/config/elk-linux.yml -t xpack-watcher rules/ > /dev/null
 	coverage run -a --include=tools/* tools/sigmac.py -rvdI -c tools/config/elk-defaultindex.yml -t xpack-watcher rules/ > /dev/null
@@ -47,5 +52,7 @@ test-sigmac:
 	! coverage run -a --include=tools/* tools/sigmac.py -t es-qs -c tests/invalid_yaml.yml rules/windows/sysmon/sysmon_mimikatz_detection_lsass.yml
 	! coverage run -a --include=tools/* tools/sigmac.py -t es-qs -c tests/invalid_config.yml rules/windows/sysmon/sysmon_mimikatz_detection_lsass.yml
 	! coverage run -a --include=tools/* tools/sigmac.py -rvI -c tools/config/elk-defaultindex.yml -t kibana rules/ > /dev/null
-	coverage report --fail-under=90
-	rm -f $(TMPOUT)
+
+test-merge:
+	tests/test-merge.sh
+	! coverage run -a --include=tools/* tools/merge_sigma.py tests/not_existing.yml > /dev/null

tests/test-merge.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+for f in $(find rules/ -type f -name '*.yml')
+do
+    echo -n .
+    if ! coverage run -a --include=tools/* tools/merge_sigma.py $f > /dev/null
+    then
+        exit 1
+    fi
+done

tools/merge_sigma.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+# Merge a Sigma rule collection into full Sigma rules
+
+import sys
+import argparse
+import yaml
+
+from sigma import SigmaCollectionParser
+
+argparser = argparse.ArgumentParser(description="Convert Sigma rules into SIEM signatures.")
+argparser.add_argument("input", help="Sigma input file")
+cmdargs = argparser.parse_args()
+
+try:
+    f = open(cmdargs.input, "r")
+except IOError as e:
+    print("Error while opening input file: %s" % str(e), file=sys.stderr)
+    sys.exit(1)
+
+content = "".join(f.readlines())
+f.close()
+sc = SigmaCollectionParser(content)
+
+print(yaml.dump_all(sc, default_flow_style=False))

tools/sigma.py

@@ -22,7 +22,9 @@ class SigmaCollectionParser:
     * reset: resets global attributes from previous set_global statements
     * repeat: takes attributes from this YAML document, merges into previous rule YAML and regenerates the rule
     """
-    def __init__(self, content, config, rulefilter=None):
+    def __init__(self, content, config=None, rulefilter=None):
+        if config is None:
+            config = SigmaConfiguration()
         self.yamls = yaml.safe_load_all(content)
         globalyaml = dict()
         self.parsers = list()
@@ -59,6 +61,9 @@ class SigmaCollectionParser:
         for parser in self.parsers:
             backend.generate(parser)
 
+    def __iter__(self):
+        return iter([parser.parsedyaml for parser in self.parsers])
+
 def deep_update_dict(dest, src):
     for key, value in src.items():
         if isinstance(value, dict) and key in dest and isinstance(dest[key], dict):     # source is dict, destination key already exists and is dict: merge