# -*- coding: utf-8 -*- import sys import os import yara # install 'yara-python' module not the outdated 'yara' module import logging import traceback import codecs import re YARA_RULE_DIRECTORIES = [r'./yara'] FILENAME_IOC_DIRECTORY = r'./iocs' def walk_error(err): try: if "Error 3" in str(err): logging.error(removeNonAsciiDrop(str(err))) print "Directory walk error" sys.exit(1) except UnicodeError, e: print "Unicode decode error in walk error message" sys.exit(1) def removeNonAsciiDrop(string): nonascii = "error" try: # Generate a new string without disturbing characters nonascii = "".join(i for i in string if ord(i)<127 and ord(i)>31) except Exception, e: traceback.print_exc() pass return nonascii def transformOS(regex, platform): # Replace '\' with '/' on Linux/Unix/OSX if platform != "windows": regex = regex.replace(r'\\', r'/') regex = regex.replace(r'C:', '') return regex def replaceEnvVars(path): # Setting new path to old path for default new_path = path # ENV VARS ---------------------------------------------------------------- # Now check if an environment env is included in the path string res = re.search(r"([@]?%[A-Za-z_]+%)", path) if res: env_var_full = res.group(1) env_var = env_var_full.replace("%", "").replace("@", "") # Check environment varibales if there is a matching var if env_var in os.environ: if os.environ[env_var]: new_path = path.replace(env_var_full, re.escape(os.environ[env_var])) # TYPICAL REPLACEMENTS ---------------------------------------------------- if path[:11].lower() == "\\systemroot": new_path = path.replace("\\SystemRoot", os.environ["SystemRoot"]) if path[:8].lower() == "system32": new_path = path.replace("system32", "%s\\System32" % os.environ["SystemRoot"]) #if path != new_path: # print "OLD: %s NEW: %s" % (path, new_path) return new_path def initialize_filename_iocs(): try: for ioc_filename in os.listdir(FILENAME_IOC_DIRECTORY): if 'filename' in ioc_filename: logging.info("Compiling Filename IOCs from %s" % ioc_filename) with codecs.open(os.path.join(FILENAME_IOC_DIRECTORY, ioc_filename), 'r', encoding='utf-8') as file: lines = file.readlines() # Last Comment Line last_comment = "" for line in lines: try: # Empty if re.search(r'^[\s]*$', line): continue # Comments if re.search(r'^#', line): last_comment = line.lstrip("#").lstrip(" ").rstrip("\n") continue # Elements with description if ";" in line: line = line.rstrip(" ").rstrip("\n\r") row = line.split(';') regex = row[0] score = row[1] if len(row) > 2: regex_fp = row[2] desc = last_comment # Catch legacy lines if not score.isdigit(): desc = score # score is description (old format) score = 60 # default value # Elements without description else: regex = line # Replace environment variables regex = replaceEnvVars(regex) # OS specific transforms regex = transformOS(regex, "windows") # If false positive definition exists regex_fp_comp = None if 'regex_fp' in locals(): # Replacements regex_fp = replaceEnvVars(regex_fp) regex_fp = transformOS(regex_fp, "windows") # String regex as key - value is compiled regex of false positive values regex_fp_comp = re.compile(regex_fp) # Create dictionary with IOC data fioc = {'regex': re.compile(regex), 'score': score, 'description': desc, 'regex_fp': regex_fp_comp} except Exception, e: traceback.print_exc() logging.error("Error reading line: %s" % line) sys.exit(1) except Exception, e: traceback.print_exc() logging.error("Error reading File IOC file: %s" % ioc_filename) sys.exit(1) def initialize_yara_rules(): yaraRules = "" dummy = "" try: for yara_rule_directory in YARA_RULE_DIRECTORIES: if not os.path.exists(yara_rule_directory): continue logging.info("Processing YARA rules folder {0}".format(yara_rule_directory)) for root, directories, files in os.walk(yara_rule_directory, onerror=walk_error, followlinks=False): for file in files: try: # Full Path yaraRuleFile = os.path.join(root, file) # Skip hidden, backup or system related files if file.startswith(".") or file.startswith("~") or file.startswith("_"): continue # Extension extension = os.path.splitext(file)[1].lower() # Test Compile try: compiledRules = yara.compile(yaraRuleFile, externals={ 'filename': dummy, 'filepath': dummy, 'extension': dummy, 'filetype': dummy, 'md5': dummy }) logging.info("Initializing Yara rule %s" % file) except Exception, e: logging.error("Error in YARA rule: %s" % yaraRuleFile) traceback.print_exc() continue # Encrypted if extension == ".yar": with open(yaraRuleFile, 'r') as rulefile: data = rulefile.read() yaraRules += data except Exception, e: logging.error("Error reading signature file %s ERROR: %s" % yaraRuleFile) traceback.print_exc() sys.exit(1) # Compile try: logging.info("Initializing all YARA rules at once (composed string of all rule files)") compiledRules = yara.compile(source=yaraRules, externals={ 'filename': dummy, 'filepath': dummy, 'extension': dummy, 'filetype': dummy, 'md5': dummy }) logging.info("Initialized all Yara rules at once") except Exception, e: traceback.print_exc() logging.error("Error during YARA rule compilation when all YARA rules are are combined in a single file") sys.exit(1) except Exception, e: traceback.print_exc() logging.error("Unexpected error while walking the directories") sys.exit(1) # MAIN ################################################################ if __name__ == '__main__': logging.basicConfig(level=logging.DEBUG) console = logging.StreamHandler() console.setLevel(logging.INFO) logging.getLogger('sigbase').addHandler(console) # Compile YARA rules initialize_yara_rules() # Compile Filename IOCs initialize_filename_iocs()