Merge branch 'elastalert_dsl_backend' of https://github.com/agix/sigma into agix-elastalert_dsl_backend

tools/config/elk-defaultindex-filebeat.yml

@@ -6,5 +6,6 @@ backends:
   - kibana
   - xpack-watcher
   - elastalert
+  - elastalert-dsl
 defaultindex:
     - filebeat-*

tools/config/elk-defaultindex-logstash.yml

@@ -6,5 +6,6 @@ backends:
   - kibana
   - xpack-watcher
   - elastalert
+  - elastalert-dsl
 defaultindex:
     - logstash-*

tools/config/elk-defaultindex.yml

@@ -6,6 +6,7 @@ backends:
   - kibana
   - xpack-watcher
   - elastalert
+  - elastalert-dsl
 defaultindex:
     - logstash-*
     - filebeat-*

tools/config/elk-linux.yml

@@ -6,6 +6,7 @@ backends:
   - kibana
   - xpack-watcher
   - elastalert
+  - elastalert-dsl
 logsources:
     apache:
         category: webserver

tools/config/elk-windows.yml

@@ -6,6 +6,7 @@ backends:
   - kibana
   - xpack-watcher
   - elastalert
+  - elastalert-dsl
 logsources:
   windows:
     product: windows
@@ -38,6 +39,6 @@ logsources:
   windows-dhcp:
     product: windows
     service: dhcp
-    conditions: 
+    conditions:
       source: 'Microsoft-Windows-DHCP-Server/Operational'
 defaultindex: logstash-*

tools/config/elk-winlogbeat.yml

@@ -6,6 +6,7 @@ backends:
   - kibana
   - xpack-watcher
   - elastalert
+  - elastalert-dsl
 logsources:
   windows:
     product: windows
@@ -38,7 +39,7 @@ logsources:
   windows-dhcp:
     product: windows
     service: dhcp
-    conditions: 
+    conditions:
       source: 'Microsoft-Windows-DHCP-Server/Operational'
 defaultindex: winlogbeat-*
 # Extract all field names qith yq:

tools/config/helk.yml

@@ -6,6 +6,7 @@ backends:
   - kibana
   - xpack-watcher
   - elastalert
+  - elastalert-dsl
 logsources:
   windows-application:
     product: windows

tools/sigma/backends/base.py

@@ -25,7 +25,7 @@ from .mixins import RulenameCommentMixin, QuoteCharMixin
 class BackendOptions(dict):
     """
     Object containing all the options that should be passed to the backend.
-    
+
     The options can come from command line and a YAML configuration file, and will be merged together.
     Options from the command line take precedence.
     """

tools/sigma/backends/elasticsearch.py

@@ -588,9 +588,8 @@ class XPackWatcherBackend(ElasticsearchQuerystringBackend, MultiRuleOutputMixin)
                 raise NotImplementedError("Output type '%s' not supported" % self.output_type)
         return result
 
-class ElastalertBackend(MultiRuleOutputMixin, ElasticsearchQuerystringBackend):
+class ElastalertBackend(MultiRuleOutputMixin):
     """Elastalert backend"""
-    identifier = 'elastalert'
     active = True
     supported_alert_methods = {'email', 'http_post'}
 
@@ -646,7 +645,9 @@ class ElastalertBackend(MultiRuleOutputMixin, ElasticsearchQuerystringBackend):
                 "realert": self.generateTimeframe(self.realert_time),
                 #"exponential_realert": self.generateTimeframe(self.expo_realert_time)
             }
+
             rule_object['filter'] = self.generateQuery(parsed)
+            self.queries = []
 
             #Handle aggregation
             if parsed.parsedAgg:
@@ -722,10 +723,6 @@ class ElastalertBackend(MultiRuleOutputMixin, ElasticsearchQuerystringBackend):
             #Clear fields
             self.fields = []
 
-    def generateQuery(self, parsed):
-        #Generate ES QS Query
-        return [{ 'query' : { 'query_string' : { 'query' : super().generateQuery(parsed) } } }]
-
     def generateNode(self, node):
         #Save fields for adding them in query_key
         #if type(node) == sigma.parser.NodeSubexpression:
@@ -761,12 +758,12 @@ class ElastalertBackend(MultiRuleOutputMixin, ElasticsearchQuerystringBackend):
                 raise NotImplementedError("%s : The '%s' aggregation operator is not yet implemented for this backend"%(self.title, funcname))
 
     def convertLevel(self, level):
-    	return {
-        	'critical': 1,
-        	'high': 2,
-        	'medium': 3,
-        	'low': 4
-    	}.get(level, 2)
+        return {
+            'critical': 1,
+            'high': 2,
+            'medium': 3,
+            'low': 4
+        }.get(level, 2)
 
     def finalize(self):
         result = ""
@@ -774,3 +771,27 @@ class ElastalertBackend(MultiRuleOutputMixin, ElasticsearchQuerystringBackend):
             result += yaml.dump(rule, default_flow_style=False)
             result += '\n'
         return result
+
+class ElastalertBackendDsl(ElastalertBackend, ElasticsearchDSLBackend):
+    """Elastalert backend"""
+    identifier = 'elastalert-dsl'
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def generateQuery(self, parsed):
+        #Generate ES DSL Query
+        super().generateBefore(parsed)
+        super().generateQuery(parsed)
+        super().generateAfter(parsed)
+        return self.queries
+
+class ElastalertBackendQs(ElastalertBackend, ElasticsearchQuerystringBackend):
+    """Elastalert backend"""
+    identifier = 'elastalert'
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def generateQuery(self, parsed):
+        #Generate ES QS Query
+        return [{ 'query' : { 'query_string' : { 'query' : super().generateQuery(parsed) } } }]
+