Merge pull request #2012 from frack113/upgrade_test

Upgrade test_rules.py

rules/cloud/aws/aws_root_account_usage.yml

@@ -14,7 +14,7 @@ detection:
     userIdentity.type: Root
   selection_eventtype:
     eventType: AwsServiceEvent
-  condition: selection_usertype AND NOT selection_eventtype
+  condition: selection_usertype and not selection_eventtype
 falsepositives:
   - AWS Tasks That Require AWS Account Root User Credentials https://docs.aws.amazon.com/general/latest/gr/aws_tasks-that-require-root.html
 level: medium

rules/network/zeek/zeek_http_executable_download_from_webdav.yml

@@ -20,7 +20,7 @@ detection:
     selection_executable:
         - resp_mime_types|contains: 'dosexec'
         - c-uri|endswith: '.exe'
-    condition: selection_webdav AND selection_executable
+    condition: selection_webdav and selection_executable
 falsepositives:
     - unknown
 level: medium

rules/windows/builtin/win_metasploit_authentication.yml

@@ -25,7 +25,7 @@ detection:
         ProcessName:
         EventID: 4776
         Workstation|re: '^[A-Za-z0-9]{16}$'
-    condition: selection1 OR selection2
+    condition: selection1 or selection2
 falsepositives:
     - Linux hostnames composed of 16 characters.
 level: high

rules/windows/builtin/win_possible_dc_shadow.yml

@@ -24,7 +24,7 @@ detection:
         EventID: 5136
         AttributeLDAPDisplayName: servicePrincipalName
         AttributeValue|startswith: 'GC/'
-    condition: selection1 OR selection2
+    condition: selection1 or selection2
 falsepositives:
     - Exclude known DCs
 level: high

rules/windows/create_remote_thread/sysmon_suspicious_remote_thread.yml

@@ -72,7 +72,7 @@ detection:
             - '\wscript.exe'
     filter:
         SourceImage|contains: 'Visual Studio'
-    condition: selection AND NOT filter
+    condition: selection and not filter
 fields:
     - ComputerName
     - User

rules/windows/process_creation/Monitor_LOLBins_Process_Creations_by_Office_applications.yml

@@ -30,7 +30,7 @@ detection:
       - winword.exe
       - excel.exe
       - powerpnt.exe
-  condition: selection1 AND selection2
+  condition: selection1 and selection2
 falsepositives:
   - Unknown
 level: high

rules/windows/process_creation/Monitor_Office_Applications_Spawning_WMI_command-line.yml

@@ -26,7 +26,7 @@ detection:
     - winword.exe
     - excel.exe
     - powerpnt.exe
-  condition: selection1 AND selection2
+  condition: selection1 and selection2
 falsepositives:
 - Unknown
 level: high

rules/windows/process_creation/Monitor_executable_and_script_files_creation_by_Office_applications_using_file_extentions.yml

@@ -38,7 +38,7 @@ detection:
   selection3:
     FileMagicBytes|startswith:
     - "4D5A"
-  condition: selection1 AND (selection2 OR selection3)
+  condition: selection1 and (selection2 or selection3)
 falsepositives:
 - Unknown
 level: high

rules/windows/process_creation/process_creation_command_execution_by_office_applications.yml

@@ -27,7 +27,7 @@ detection:
       - '\winword.exe'
       - '\excel.exe'
       - '\powerpnt.exe'
-  condition: selection1 AND selection2
+  condition: selection1 and selection2
 falsepositives:
 - Unknown
 level: high

rules/windows/process_creation/process_creation_office_application_from_proxy_executing_regsvr32_with_payload.yml

@@ -38,7 +38,7 @@ detection:
       - 'process'
       - 'create'
       - 'call' 
-  condition: selection1 AND selection2 AND selection3 AND selection4
+  condition: selection1 and selection2 and selection3 and selection4
 falsepositives:
 - Unknown
 level: high

rules/windows/process_creation/win_nltest_recon.yml

@@ -31,7 +31,7 @@ detection:
             - '/parentdomain'
             - '/domain_trusts'
             - '/user'
-    condition: selection_nltest AND (selection_recon1 OR selection_recon2)
+    condition: selection_nltest and (selection_recon1 or selection_recon2)
 falsepositives:
     - Legitimate administration use but user must be check out
 level: medium

rules/windows/registry_event/sysmon_asep_reg_keys_modification.yml

@@ -190,19 +190,19 @@ detection:
             - '\Lsa\Notification Packages'
             - '\Lsa\Authentication Packages'
             - '\BootVerificationProgram\ImagePath'
-    condition: main_selection OR
-               session_manager_base AND session_manager OR
-               current_version_base AND current_version OR
-               nt_current_version_base AND nt_current_version OR
-               wow_current_version_base AND wow_current_version OR
-               wow_nt_current_version_base AND wow_nt_current_version OR
-               (wow_office OR office) AND wow_office_details OR
-               (wow_ie OR ie) AND wow_ie_details OR
-               wow_classes_base AND wow_classes OR
-               classes_base AND classes OR
-               scripts_base AND scripts OR
-               winsock_parameters_base AND winsock_parameters OR
-               system_control_base AND system_control
+    condition: main_selection or
+               session_manager_base and session_manager or
+               current_version_base and current_version or
+               nt_current_version_base and nt_current_version or
+               wow_current_version_base and wow_current_version or
+               wow_nt_current_version_base and wow_nt_current_version or
+               (wow_office or office) and wow_office_details or
+               (wow_ie or ie) and wow_ie_details or
+               wow_classes_base and wow_classes or
+               classes_base and classes or
+               scripts_base and scripts or
+               winsock_parameters_base and winsock_parameters or
+               system_control_base and system_control
 fields:
     - SecurityID
     - ObjectName

rules/windows/sysmon/Monitor_Office_Applications_from_proxy_executing_regsvr32_with_payload.yml

@@ -40,7 +40,7 @@ detection:
       - 'process'
       - 'create'
       - 'call'
-  condition: selection1 AND selection2 AND selection3 AND selection4
+  condition: selection1 and selection2 and selection3 and selection4
 falsepositives:
 - Unknown
 level: high

rules/windows/sysmon/Monitor_executable_and_script_files_creation_by_Office_applications_using_file_extentions.yml

@@ -35,7 +35,7 @@ detection:
     - ".bat"
     - ".scr"
     - ".proj"
-  condition: selection1 AND selection2
+  condition: selection1 and selection2
 falsepositives:
 - Unknown
 level: high

rules/windows/sysmon/Office_Applications_Spawning_WMI_command-line.yml

@@ -28,7 +28,7 @@ detection:
       - winword.exe
       - excel.exe
       - powerpnt.exe
-  condition: selection1 AND selection2
+  condition: selection1 and selection2
 falsepositives:
 - Unknown
 level: high

tests/test_rules.py

@@ -39,7 +39,7 @@ class TestRules(unittest.TestCase):
     def get_rule_yaml(self, file_path:str) -> dict:
         data = []
 
-        with open(file_path) as f:
+        with open(file_path,encoding='utf-8') as f:
             yaml_parts = yaml.safe_load_all(f)
             for part in yaml_parts:
                 data.append(part)
@@ -64,7 +64,7 @@ class TestRules(unittest.TestCase):
         files_with_legal_issues = []
 
         for file in self.yield_next_rule_file_path(self.path_to_rules):
-            with open(file, 'r') as fh:
+            with open(file, 'r',encoding='utf-8') as fh:
                 file_data = fh.read()
                 for tm in self.TRADE_MARKS:
                     if tm in file_data:
@@ -245,7 +245,7 @@ class TestRules(unittest.TestCase):
     def test_event_id_instead_of_process_creation(self):
         faulty_detections = []
         for file in self.yield_next_rule_file_path(self.path_to_rules):
-            with open(file) as f:
+            with open(file,encoding='utf-8') as f:
                 for line in f:
                     if re.search(r'.*EventID: (?:1|4688)\s*$', line) and file not in faulty_detections:
                         faulty_detections.append(file)
@@ -310,7 +310,7 @@ class TestRules(unittest.TestCase):
             logsource = self.get_rule_part(file_path=file, part_name="logsource")
             service = logsource.get('service', '')
             if service.lower() == 'sysmon':
-                with open(file) as f:
+                with open(file,encoding='utf-8') as f:
                     found = False
                     for line in f:
                         if re.search(r'.*EventID:.*$', line):  # might be on a single line or in multiple lines
@@ -620,6 +620,29 @@ class TestRules(unittest.TestCase):
    
         self.assertEqual(faulty_rules, [], Fore.RED + "There are rules using list with only 1 element")
 
+    def test_condition_operator_casesensitive(self):
+        faulty_rules = []
+        for file in self.yield_next_rule_file_path(self.path_to_rules):
+             detection = self.get_rule_part(file_path=file, part_name="detection")
+             if detection:
+                 valid = True
+                 if isinstance(detection["condition"],str):
+                     param = detection["condition"].split(' ')
+                     for item in param:
+                        if item.lower() == 'or' and not item == 'or':
+                            valid = False
+                        elif item.lower() == 'and' and not item == 'and':
+                            valid = False
+                        elif item.lower() == 'not' and not item == 'not':
+                            valid = False   
+                        elif item.lower() == 'of' and not item == 'of':
+                            valid = False                            
+                     if not valid:
+                         print(Fore.RED + "Rule {} has a invalid condition '{}' : 'or','and','not','of' are lowercase".format(file,detection["condition"]))
+                         faulty_rules.append(file)
+                         
+        self.assertEqual(faulty_rules, [], Fore.RED + "There are rules using condition whitout lowercase operator")
+
 def get_mitre_data():
     """
     Generate tags from live TAXI service to get up-to-date data