Merge remote-tracking branch 'upstream/master'

Makefile

@@ -1,4 +1,4 @@
-.PHONY: test test-yaml test-sigmac
+    .PHONY: test test-yaml test-sigmac
 test: test-yaml test-sigmac
 
 test-yaml:
@@ -7,6 +7,8 @@ test-yaml:
 test-sigmac:
 	tools/sigmac.py -l
 	tools/sigmac.py -rvdI -t es-qs rules/ 
+	tools/sigmac.py -rvdI -t kibana rules/ 
+	tools/sigmac.py -rvdI -t xpack-watcher rules/ 
 	tools/sigmac.py -rvdI -t splunk rules/ 
 	tools/sigmac.py -rvdI -t logpoint rules/ 
 	tools/sigmac.py -rvdI -t fieldlist rules/ 

rules/proxy/proxy_ua_apt.yml

@@ -24,6 +24,7 @@ detection:
         - 'Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-EN; rv:1.7.12) Gecko/20100719 Firefox/1.0.7'  # Unit78020 Malware
         - 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2.13) Firefox/3.6.13 GTB7.1'  # Winnti related
         - 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)'  # Winnti related
+        - 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NETCLR 2.0.50727)'  # APT17
     condition: selection
 fields:
     - ClientIP

rules/web/web_multiple_suspicious_resp_codes_single_source.yml

@@ -14,6 +14,7 @@ detection:
     condition: selection | count() by clientip > 10
 fields:
     - client_ip
+    - vhost
     - url
     - response
 falsepositives:

rules/web/web_webshell_keyword.yml

@@ -11,6 +11,7 @@ detection:
     condition: keywords
 fields:
     - client_ip
+    - vhost
     - url
     - response
 falsepositives:

tools/backends.py

@@ -1,5 +1,6 @@
 # Output backends for sigmac
 
+import sys
 import json
 import re
 import sigma
@@ -17,25 +18,128 @@ def getBackend(name):
     except KeyError as e:
         raise LookupError("Backend not found") from e
 
-### Generic base classes
+class BackendOptions(dict):
+    """Object contains all options that should be passed to the backend from command line (or other user interfaces)"""
 
+    def __init__(self, options):
+        """
+        Receives the argparser result from the backend option paramater value list (nargs=*) and builds the dict from it. There are two option types:
+
+        * key=value: self{key} = value
+        * key: self{key} = True
+        """
+        if options == None:
+            return
+        for option in options:
+            parsed = option.split("=", 1)
+            try:
+                self[parsed[0]] = parsed[1]
+            except IndexError:
+                self[parsed[0]] = True
+
+### Output classes
+class SingleOutput:
+    """
+    Single file output
+
+    By default, this opens the given file or stdin and passes everything into this.
+    """
+    def __init__(self, filename=None):
+        if type(filename) == str:
+            self.fd = open(filename, "w")
+        else:
+            self.fd = sys.stdout
+
+    def print(self, *args, **kwargs):
+        print(*args, file=self.fd, **kwargs)
+
+    def close(self):
+        self.fd.close()
+
+class MultiOutput:
+    """
+    Multiple file output
+
+    Prepares multiple SingleOutput instances with basename + suffix as file names, on for each suffix.
+    The switch() method is used to switch between these outputs.
+
+    This class must be inherited and suffixes must be a dict as follows: file id -> suffix
+    """
+    suffixes = None
+
+    def __init__(self, basename):
+        """Initializes all outputs with basename and corresponding suffix as SingleOutput object."""
+        if suffixes == None:
+            raise NotImplementedError("OutputMulti must be derived, at least suffixes must be set")
+        if type(basename) != str:
+            raise TypeError("OutputMulti constructor basename parameter must be string")
+
+        self.outputs = dict()
+        self.output = None
+        for name, suffix in self.suffixes:
+            self.outputs[name] = SingleOutput(basename + suffix)
+
+    def select(self, name):
+        """Select an output as current output"""
+        self.output = self.outputs[name]
+
+    def print(self, *args, **kwargs):
+        self.output.print(*args, **kwargs)
+
+    def close(self):
+        for out in self.outputs:
+            out.close()
+
+class StringOutput(SingleOutput):
+    """Collect input silently and return resulting string."""
+    def __init__(self, filename=None):
+        self.out = ""
+
+    def print(self, *args, **kwargs):
+        try:
+            del kwargs['file']
+        except KeyError:
+            pass
+        print(*args, file=self, **kwargs)
+
+    def write(self, s):
+        self.out += s
+
+    def result(self):
+        return self.out
+
+    def close(self):
+        pass
+
+### Generic backend base classes and mixins
 class BaseBackend:
     """Base class for all backends"""
     identifier = "base"
     active = False
-    index_field = None      # field name that is used to address indices
+    index_field = None    # field name that is used to address indices
+    output_class = None   # one of the above output classes
+    file_list = None
 
-    def __init__(self, sigmaconfig):
+    def __init__(self, sigmaconfig, backend_options=None, filename=None):
+        """
+        Initialize backend. This gets a sigmaconfig object, which is notified about the used backend class by
+        passing the object instance to it. Further, output files are initialized by the output class defined in output_class.
+        """
+        super().__init__()
         if not isinstance(sigmaconfig, (sigma.SigmaConfiguration, None)):
             raise TypeError("SigmaConfiguration object expected")
+        self.options = backend_options
         self.sigmaconfig = sigmaconfig
         self.sigmaconfig.set_backend(self)
+        self.output = self.output_class(filename)
 
-    def generate(self, parsed):
-        result = self.generateNode(parsed.parsedSearch)
-        if parsed.parsedAgg:
-            result += self.generateAggregation(parsed.parsedAgg)
-        return result
+    def generate(self, sigmaparser):
+        """Method is called for each sigma rule and receives the parsed rule (SigmaParser)"""
+        for parsed in sigmaparser.condparsed:
+            result = self.generateNode(parsed.parsedSearch)
+            if parsed.parsedAgg:
+                result += self.generateAggregation(parsed.parsedAgg)
+            self.output.print(result)
 
     def generateNode(self, node):
         if type(node) == sigma.ConditionAND:
@@ -79,10 +183,18 @@ class BaseBackend:
     def generateAggregation(self, agg):
         raise NotImplementedError("Aggregations not implemented for this backend")
 
+    def finalize(self):
+        """
+        Is called after the last file was processed with generate(). The right place if this backend is not intended to
+        look isolated at each rule, but generates an output which incorporates multiple rules, e.g. dashboards.
+        """
+        pass
+
 class SingleTextQueryBackend(BaseBackend):
     """Base class for backends that generate one text-based expression from a Sigma rule"""
     identifier = "base-textquery"
     active = False
+    output_class = SingleOutput
 
     # the following class variables define the generation and behavior of queries from a parse tree some are prefilled with default values that are quite usual
     reEscape = None                     # match characters that must be quoted
@@ -138,6 +250,32 @@ class SingleTextQueryBackend(BaseBackend):
     def generateValueNode(self, node):
         return self.valueExpression % (self.cleanValue(str(node)))
 
+class MultiRuleOutputMixin:
+    """Mixin with common for multi-rule outputs"""
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.rulenames = set()
+
+    def getRuleName(self, sigmaparser):
+        """
+        Generate a rule name from the title of the Sigma rule with following properties:
+
+        * Spaces are replaced with -
+        * Unique name by addition of a counter if generated name already in usage
+
+        Generated names are tracked by the Mixin.
+        
+        """
+        rulename = sigmaparser.parsedyaml["title"].replace(" ", "-")
+        if rulename in self.rulenames:   # add counter if name collides
+            cnt = 2
+            while "%s-%d" % (rulename, cnt) in self.rulenames:
+                cnt += 1
+            rulename = "%s-%d" % (rulename, cnt)
+        self.rulenames.add(rulename)
+
+        return rulename
+
 ### Backends for specific SIEMs
 
 class ElasticsearchQuerystringBackend(SingleTextQueryBackend):
@@ -157,15 +295,184 @@ class ElasticsearchQuerystringBackend(SingleTextQueryBackend):
     mapExpression = "%s:%s"
     mapListsSpecialHandling = False
 
-class ElasticsearchDSLBackend(BaseBackend):
-    """Converts Sigma rule into Elasticsearch DSL query (JSON)."""
-    identifier = "es-dsl"
-    active = False
-
-class KibanaBackend(ElasticsearchDSLBackend):
-    """Converts Sigma rule into Kibana JSON Configurations."""
+class KibanaBackend(ElasticsearchQuerystringBackend, MultiRuleOutputMixin):
+    """Converts Sigma rule into Kibana JSON Configuration files (searches only)."""
     identifier = "kibana"
-    active = False
+    active = True
+    output_class = SingleOutput
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.kibanaconf = list()
+
+    def generate(self, sigmaparser):
+        rulename = self.getRuleName(sigmaparser)
+        description = sigmaparser.parsedyaml.setdefault("description", "")
+
+        columns = list()
+        try:
+            for field in sigmaparser.parsedyaml["fields"]:
+                mapped = sigmaparser.config.get_fieldmapping(field).resolve_fieldname(field)
+                if type(mapped) == str:
+                    columns.append(mapped)
+                elif type(mapped) == list:
+                    columns.extend(mapped)
+                else:
+                    raise TypeError("Field mapping must return string or list")
+        except KeyError:    # no 'fields' attribute
+            pass
+
+        indices = sigmaparser.get_logsource().index
+        if len(indices) == 0:
+            indices = ["logstash-*"]
+
+        for parsed in sigmaparser.condparsed:
+            result = self.generateNode(parsed.parsedSearch)
+
+            for index in indices:
+                final_rulename = rulename
+                if len(indices) > 1:     # add index names if rule must be replicated because of ambigiuous index patterns
+                    final_rulename += "-" + indexname
+                    title = "%s (%s)" % (sigmaparser.parsedyaml["title"], index)
+                else:
+                    title = sigmaparser.parsedyaml["title"]
+                try:
+                    title = self.options["prefix"] + title
+                except KeyError:
+                    pass
+
+                self.kibanaconf.append({
+                        "_id": final_rulename,
+                        "_type": "search",
+                        "_source": {
+                            "title": title,
+                            "description": description,
+                            "hits": 0,
+                            "columns": columns,
+                            "sort": ["@timestamp", "desc"],
+                            "version": 1,
+                            "kibanaSavedObjectMeta": {
+                                "searchSourceJSON": json.dumps({
+                                    "index": index,
+                                    "filter":  [],
+                                    "highlight": {
+                                        "pre_tags": ["@kibana-highlighted-field@"],
+                                        "post_tags": ["@/kibana-highlighted-field@"],
+                                        "fields": { "*":{} },
+                                        "require_field_match": False,
+                                        "fragment_size": 2147483647
+                                        },
+                                    "query": {
+                                        "query_string": {
+                                            "query": result,
+                                            "analyze_wildcard": True
+                                            }
+                                        }
+                                    }
+                                )
+                            }
+                        }
+                    })
+
+    def finalize(self):
+        self.output.print(json.dumps(self.kibanaconf, indent=2))
+
+class XPackWatcherBackend(ElasticsearchQuerystringBackend, MultiRuleOutputMixin):
+    """Converts Sigma Rule into X-Pack Watcher JSON for alerting"""
+    identifier = "xpack-watcher"
+    active = True
+    output_class = SingleOutput
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.watcher_alert = dict()
+        try:
+            self.output_type = self.options["output"]
+        except KeyError:
+            self.output_type = "curl"
+
+        try:
+            self.es = self.options["es"]
+        except KeyError:
+            self.es = "localhost:9200"
+
+    def generate(self, sigmaparser):
+        # get the details if this alert occurs
+        rulename = self.getRuleName(sigmaparser)
+        description = sigmaparser.parsedyaml.setdefault("description", "")
+        false_positives = sigmaparser.parsedyaml.setdefault("falsepositives", "")
+        level = sigmaparser.parsedyaml.setdefault("level", "")
+        logging_result = "Rule description: "+str(description)+", false positives: "+str(false_positives)+", level: "+level
+        # Get time frame if exists
+        interval = sigmaparser.parsedyaml["detection"].setdefault("timeframe", "30m")
+
+        # creating condition
+        indices = sigmaparser.get_logsource().index
+        if len(indices) == 0:
+            indices = ["logstash-*"]
+
+        for condition in sigmaparser.condparsed:
+            result = self.generateNode(condition.parsedSearch)
+            try:
+                if condition.parsedAgg.cond_op == ">":
+                    alert_condition = { "gt": int(condition.parsedAgg.condition) }
+                elif condition.parsedAgg.cond_op == ">=":
+                    alert_condition = { "gte": int(condition.parsedAgg.condition) }
+                elif condition.parsedAgg.cond_op == "<":
+                    alert_condition = { "lt": int(condition.parsedAgg.condition) }
+                elif condition.parsedAgg.cond_op == "<=":
+                    alert_condition = { "lte": int(condition.parsedAgg.condition) }
+                else:
+                    alert_condition = {"not_eq": 0}
+            except KeyError:
+                alert_condition = {"not_eq": 0}
+            except AttributeError:
+                alert_condition = {"not_eq": 0}
+
+            self.watcher_alert[rulename] = {
+                              "trigger": {
+                                "schedule": {
+                                  "interval": interval  # how often the watcher should check
+                                }
+                              },
+                              "input": {
+                                "search": {
+                                  "request": {
+                                    "body": {
+                                      "size": 0,
+                                      "query": {
+                                        "query_string": {
+                                            "query": result,  # this is where the elasticsearch query syntax goes
+                                            "analyze_wildcard": True
+                                        }
+                                      }
+                                    },
+                                    "indices": indices
+                                  }
+                                }
+                              },
+                              "condition": {
+                                  "compare": {    # TODO: Issue #49
+                                  "ctx.payload.hits.total": alert_condition
+                                }
+                              },
+                              "actions": {
+                                "logging-action": {
+                                  "logging": {
+                                    "text": logging_result
+                                  }
+                                }
+                              }
+                            }
+
+    def finalize(self):
+        for rulename, rule in self.watcher_alert.items():
+            if self.output_type == "plain":     # output request line + body
+                self.output.print("PUT _xpack/watcher/watch/%s\n%s\n" % (rulename, json.dumps(rule, indent=2)))
+            elif self.output_type == "curl":      # output curl command line
+                self.output.print("curl -s -XPUT --data-binary @- %s/_xpack/watcher/watch/%s <<EOF\n%s\nEOF" % (self.es, rulename, json.dumps(rule, indent=2)))
+            else:
+                raise NotImplementedError("Output type '%s' not supported" % self.output_type)
 
 class LogPointBackend(SingleTextQueryBackend):
     """Converts Sigma rule into LogPoint query"""
@@ -233,9 +540,11 @@ class FieldnameListBackend(BaseBackend):
     """List all fieldnames from given Sigma rules for creation of a field mapping configuration."""
     identifier = "fieldlist"
     active = True
+    output_class = SingleOutput
 
-    def generate(self, parsed):
-        return "\n".join(sorted(set(list(flatten(self.generateNode(parsed.parsedSearch))))))
+    def generate(self, sigmaparser):
+        for parsed in sigmaparser.condparsed:
+            self.output.print("\n".join(sorted(set(list(flatten(self.generateNode(parsed.parsedSearch)))))))
 
     def generateANDNode(self, node):
         return [self.generateNode(val) for val in node]

tools/config/elk-linux.yml

@@ -0,0 +1,14 @@
+logsources:
+    apache:
+        category: webserver
+        index: logstash-apache-*
+    webapp-error:
+        category: application
+        index: logstash-apache_error-*
+    linux-auth:
+        product: linux
+        service: auth
+        index: logstash-auth-*
+fieldmappings:
+    client_ip: clientip
+    url: request

tools/sigma.py

@@ -12,8 +12,9 @@ class SigmaParser:
     def __init__(self, sigma, config):
         self.definitions = dict()
         self.values = dict()
-        self.parsedyaml = yaml.safe_load(sigma)
         self.config = config
+        self.parsedyaml = yaml.safe_load(sigma)
+        self.parse_sigma()
 
     def parse_sigma(self):
         try:    # definition uniqueness check
@@ -680,7 +681,6 @@ class ConditionalFieldMapping(SimpleFieldMapping):
                 rulefieldvalues = sigmaparser.values[condfield]
                 for condvalue in self.conditions[condfield]:
                     if condvalue in rulefieldvalues:
-                        print("found!")
                         targets.update(self.conditions[condfield][condvalue])
         if len(targets) == 0:       # no matching condition, try with default mapping
             if self.default != None:
@@ -869,9 +869,9 @@ class SigmaLogsourceConfiguration:
         """Match log source definition against given criteria, None = ignore"""
         searched = 0
         for searchval, selfval in zip((category, product, service), (self.category, self.product, self.service)):
-            if searchval == None and selfval != None:   #
+            if searchval == None and selfval != None:
                 return False
-            if searchval != None:
+            if selfval != None:
                 searched += 1
                 if searchval != selfval:
                     return False

tools/sigmac.py

@@ -36,7 +36,8 @@ argparser.add_argument("--recurse", "-r", action="store_true", help="Recurse int
 argparser.add_argument("--target", "-t", default="es-qs", choices=backends.getBackendDict().keys(), help="Output target format")
 argparser.add_argument("--target-list", "-l", action="store_true", help="List available output target formats")
 argparser.add_argument("--config", "-c", help="Configuration with field name and index mapping for target environment (not yet implemented)")
-argparser.add_argument("--output", "-o", help="Output file or filename prefix if multiple files are generated (not yet implemented)")
+argparser.add_argument("--output", "-o", default=None, help="Output file or filename prefix if multiple files are generated (not yet implemented)")
+argparser.add_argument("--backend-option", "-O", action="append", help="Options and switches that are passed to the backend")
 argparser.add_argument("--defer-abort", "-d", action="store_true", help="Don't abort on parse or conversion errors, proceed with next rule. The exit code from the last error is returned")
 argparser.add_argument("--ignore-not-implemented", "-I", action="store_true", help="Only return error codes for parse errors and ignore errors for rules with not implemented features")
 argparser.add_argument("--verbose", "-v", action="store_true", help="Be verbose")
@@ -50,13 +51,6 @@ if cmdargs.target_list:
     sys.exit(0)
 
 out = sys.stdout
-if cmdargs.output:
-    try:
-        out = open(cmdargs.output, mode='w')
-    except IOError:
-        print("Failed to open output file '%s': %s" % (cmdargs.output, str(e)), file=sys.stderr)
-        exit(1)
-
 sigmaconfig = SigmaConfiguration()
 if cmdargs.config:
     try:
@@ -70,11 +64,16 @@ if cmdargs.config:
     except SigmaParseError as e:
         print("Sigma configuration parse error in %s: %s" % (conffile, str(e)), file=sys.stderr)
 
+backend_options = backends.BackendOptions(cmdargs.backend_option)
+
 try:
-    backend = backends.getBackend(cmdargs.target)(sigmaconfig)
+    backend = backends.getBackend(cmdargs.target)(sigmaconfig, backend_options, cmdargs.output)
 except LookupError as e:
     print("Backend not found!", file=sys.stderr)
     sys.exit(2)
+except IOError:
+    print("Failed to open output file '%s': %s" % (cmdargs.output, str(e)), file=sys.stderr)
+    exit(1)
 
 error = 0
 for sigmafile in get_inputs(cmdargs.inputs, cmdargs.recurse):
@@ -83,12 +82,11 @@ for sigmafile in get_inputs(cmdargs.inputs, cmdargs.recurse):
         f = sigmafile.open()
         parser = SigmaParser(f, sigmaconfig)
         print_debug("Parsed YAML:\n", json.dumps(parser.parsedyaml, indent=2))
-        parser.parse_sigma()
         for condtoken in parser.condtoken:
             print_debug("Condition Tokens:", condtoken)
         for condparsed in parser.condparsed:
             print_debug("Condition Parse Tree:", condparsed)
-            print(backend.generate(condparsed), file=out)
+        backend.generate(parser)
     except OSError as e:
         print("Failed to open Sigma file %s: %s" % (sigmafile, str(e)), file=sys.stderr)
         error = 5
@@ -120,5 +118,6 @@ for sigmafile in get_inputs(cmdargs.inputs, cmdargs.recurse):
         except:
             print_debug("Sigma rule didn't reached condition tokenization")
         print_debug()
+backend.finalize()
 
 sys.exit(error)