From 67bf09920753dcf3a2d905594eecdb627cb3206c Mon Sep 17 00:00:00 2001 From: Wesley Shields Date: Wed, 22 Apr 2015 19:41:51 -0400 Subject: [PATCH] YARA tests, SQL matching, sigfile loading 1. Minor refactoring. - Generate one row per sigfile or sig_group. - While here, when a signature file fails to compile, VLOG() it. 2. Bring in a couple of YARA tests. Write a couple of tests for YARA functionality. Right now the only tests make sure rules are compiled properly and that rules match where they should and don't match where they shouldn't. 3. Allow sigfiles to be relative to /var/osquery. - Also, only create a row if scanning happened. 4. Add pattern support to yara table. - Also, optimize things so that rules are only compiled once. --- osquery/tables/CMakeLists.txt | 1 + osquery/tables/specs/yara.table | 2 + osquery/tables/utils/tests/yara_tests.cpp | 83 ++++++++++++ osquery/tables/utils/yara.cpp | 154 ++++++++++++++++------ 4 files changed, 197 insertions(+), 43 deletions(-) create mode 100644 osquery/tables/utils/tests/yara_tests.cpp diff --git a/osquery/tables/CMakeLists.txt b/osquery/tables/CMakeLists.txt index 005963c9..f98079e1 100644 --- a/osquery/tables/CMakeLists.txt +++ b/osquery/tables/CMakeLists.txt @@ -67,6 +67,7 @@ ADD_OSQUERY_LIBRARY(FALSE osquery_utils ADD_OSQUERY_TEST(FALSE etc_hosts_tests networking/tests/etc_hosts_tests.cpp) ADD_OSQUERY_TEST(FALSE etc_protocols_tests networking/tests/etc_protocols_tests.cpp) +ADD_OSQUERY_TEST(FALSE yara_tests utils/tests/yara_tests.cpp) if(APPLE) ADD_OSQUERY_TEST(FALSE apps_tests system/darwin/tests/apps_tests.cpp) ADD_OSQUERY_TEST(FALSE certificates_tests system/darwin/tests/certificates_tests.cpp) diff --git a/osquery/tables/specs/yara.table b/osquery/tables/specs/yara.table index 37559bb9..141190d3 100644 --- a/osquery/tables/specs/yara.table +++ b/osquery/tables/specs/yara.table @@ -6,5 +6,7 @@ schema([ Column("count", INTEGER, "Number of YARA matches"), Column("sig_group", TEXT, "Signature group used"), Column("sigfile", TEXT, "Signature file used"), + Column("pattern", TEXT, "A pattern which can be used to match file paths"), + ]) implementation("yara@genYara") diff --git a/osquery/tables/utils/tests/yara_tests.cpp b/osquery/tables/utils/tests/yara_tests.cpp new file mode 100644 index 00000000..a4ffdc47 --- /dev/null +++ b/osquery/tables/utils/tests/yara_tests.cpp @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2015, Wesley Shields + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ + +#include + +#include + +#include "osquery/tables/utils/yara_utils.h" + +using namespace osquery::tables; + +namespace osquery { + +const std::string ruleFile = "/tmp/osquery-yara.sig"; +const std::string ls = "/bin/ls"; +const std::string alwaysTrue = "rule always_true { condition: true }"; +const std::string alwaysFalse = "rule always_false { condition: false }"; + +class YARATest : public testing::Test { + protected: + void SetUp() { + remove(ruleFile); + if (pathExists(ruleFile).ok()) { + throw std::domain_error("Rule file exists."); + } + } + + void TearDown() { + remove(ruleFile); + } + + Row scanFile(const std::string ruleContent) { + YR_RULES* rules = nullptr; + int result = yr_initialize(); + EXPECT_TRUE(result == ERROR_SUCCESS); + + writeTextFile(ruleFile, ruleContent); + + Status status = compileSingleFile(ruleFile, &rules); + EXPECT_TRUE(status.ok()); + + Row r; + r["count"] = "0"; + r["matches"] = ""; + + result = yr_rules_scan_file(rules, + ls.c_str(), + SCAN_FLAGS_FAST_MODE, + YARACallback, + (void*)&r, + 0); + EXPECT_TRUE(result == ERROR_SUCCESS); + + yr_rules_destroy(rules); + + return r; + } +}; + +TEST_F(YARATest, test_match_true) { + Row r = scanFile(alwaysTrue); + // Should have 1 count + EXPECT_TRUE(r["count"] == "1"); +} + +TEST_F(YARATest, test_match_false) { + Row r = scanFile(alwaysFalse); + // Should have 0 count + EXPECT_TRUE(r["count"] == "0"); +} +} + +int main(int argc, char* argv[]) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/osquery/tables/utils/yara.cpp b/osquery/tables/utils/yara.cpp index 9b04bc0e..93fa3e1c 100644 --- a/osquery/tables/utils/yara.cpp +++ b/osquery/tables/utils/yara.cpp @@ -10,6 +10,7 @@ #include +#include #include #include #include @@ -24,15 +25,52 @@ namespace osquery { namespace tables { +Status doYARAScan(YR_RULES* rules, + const std::string& path, + const std::string& pattern, + QueryData& results, + const std::string& group, + const std::string& sigfile) { + Row r; + + // These are default values, to be updated in YARACallback. + r["count"] = INTEGER(0); + r["matches"] = std::string(""); + + // XXX: use target_path instead to be consistent with yara_events? + r["path"] = path; + + r["pattern"] = pattern; + + r["sig_group"] = std::string(group); + r["sigfile"] = std::string(sigfile); + + int result = yr_rules_scan_file(rules, + path.c_str(), + SCAN_FLAGS_FAST_MODE, + YARACallback, + (void*)&r, + 0); + + if (result != ERROR_SUCCESS) { + return Status(1, "Scan error (" + std::to_string(result) + ")"); + } + + results.push_back(r); + return Status(0, "OK"); +} + QueryData genYara(QueryContext& context) { QueryData results; + Status status; auto paths = context.constraints["path"].getAll(EQUALS); + auto patterns = context.constraints["pattern"].getAll(EQUALS); auto groups = context.constraints["sig_group"].getAll(EQUALS); auto sigfiles = context.constraints["sigfile"].getAll(EQUALS); // Must specify a path constraint and at least one of sig_group or sigfile. - if (paths.size() == 0 || (groups.size() == 0 && sigfiles.size() == 0)) { + if (groups.size() == 0 && sigfiles.size() == 0) { return results; } @@ -45,66 +83,96 @@ QueryData genYara(QueryContext& context) { const auto& yaraParser = std::static_pointer_cast(parser); auto rules = yaraParser->rules(); - Row r; + // Store resolved paths in a vector of pairs. + // Each pair has the first element as the path to scan and the second + // element as the pattern which generated it. + std::vector > path_pairs; - // These are filled in depending upon what is used to scan. - r["sigfile"] = std::string(""); - r["sig_group"] = std::string(""); + // Expand patterns and push onto path_pairs. + for (const auto& pattern : patterns) { + std::vector expanded_patterns; + auto status = resolveFilePattern(pattern, expanded_patterns); + if (!status.ok()) { + VLOG(1) << "Could not expand pattern properly: " << status.toString(); + return results; + } + for (const auto& resolved : expanded_patterns) { + if (!isReadable(resolved)) { + continue; + } + path_pairs.push_back(make_pair(resolved, pattern)); + } + } + + // Collect all paths specified too. for (const auto& path_string : paths) { - boost::filesystem::path path = path_string; - if (!boost::filesystem::is_regular_file(path)) { + if (!isReadable(path_string)) { continue; } + path_pairs.push_back(make_pair(path_string, "")); + } - // These are default values, to be updated in YARACallback. - r["count"] = INTEGER(0); - r["matches"] = std::string(""); + // Compile all sigfiles into a map. + std::map compiled_rules; + for (const auto& file : sigfiles) { + YR_RULES *rules = nullptr; - // XXX: use target_path instead to be consistent with yara_events? - r["path"] = path_string; + std::string full_path; + if (file[0] != '/') { + full_path = std::string("/var/osquery/") + file; + } else { + full_path = file; + } + status = compileSingleFile(full_path, &rules); + if (!status.ok()) { + VLOG(1) << "YARA error: " << status.toString(); + } else { + compiled_rules[file] = rules; + } + } + + // Scan every path pair. + for (const auto& path_pair : path_pairs) { + // Scan using siggroups. for (const auto& group : groups) { - r["sig_group"] = std::string(group); - if (rules.count(group) == 0) + if (rules.count(group) == 0) { continue; + } + VLOG(1) << "Scanning with group: " << group; - int result = yr_rules_scan_file(rules[group], - path_string.c_str(), - SCAN_FLAGS_FAST_MODE, - YARACallback, - (void*)&r, - 0); - - if (result != ERROR_SUCCESS) { - return results; + status = doYARAScan(rules[group], + path_pair.first.c_str(), + path_pair.second, + results, + group, + ""); + if (!status.ok()) { + VLOG(1) << "YARA error: " << status.toString(); } - results.push_back(r); } - for (const auto& file : sigfiles) { - YR_RULES* rules = nullptr; - r["sigfile"] = std::string(file); - VLOG(1) << "Scanning with file: " << file; - Status status = compileSingleFile(file, &rules); - if (status.ok()) { - int result = yr_rules_scan_file(rules, - path_string.c_str(), - SCAN_FLAGS_FAST_MODE, - YARACallback, - (void*)&r, - 0); - - if (result != ERROR_SUCCESS) { - return results; - } - - yr_rules_destroy(rules); + // Scan using files. + for (const auto& element : compiled_rules) { + VLOG(1) << "Scanning with file: " << element.first; + status = doYARAScan(element.second, + path_pair.first.c_str(), + path_pair.second, + results, + "", + element.first); + if (!status.ok()) { + VLOG(1) << "YARA error: " << status.toString(); } - results.push_back(r); } } + // Cleanup compiled rules + for (const auto& element : compiled_rules) { + yr_rules_destroy(element.second); + } + return results; } }