YARA tests, SQL matching, sigfile loading

1. Minor refactoring.

- Generate one row per sigfile or sig_group.
- While here, when a signature file fails to compile, VLOG() it.

2. Bring in a couple of YARA tests.
Write a couple of tests for YARA functionality. Right now the only tests
make sure rules are compiled properly and that rules match where they
should and don't match where they shouldn't.

3. Allow sigfiles to be relative to /var/osquery.
- Also, only create a row if scanning happened.

4. Add pattern support to yara table.
- Also, optimize things so that rules are only compiled once.

osquery/tables/CMakeLists.txt

@@ -67,6 +67,7 @@ ADD_OSQUERY_LIBRARY(FALSE osquery_utils
 
 ADD_OSQUERY_TEST(FALSE etc_hosts_tests networking/tests/etc_hosts_tests.cpp)
 ADD_OSQUERY_TEST(FALSE etc_protocols_tests networking/tests/etc_protocols_tests.cpp)
+ADD_OSQUERY_TEST(FALSE yara_tests utils/tests/yara_tests.cpp)
 if(APPLE)
   ADD_OSQUERY_TEST(FALSE apps_tests system/darwin/tests/apps_tests.cpp)
   ADD_OSQUERY_TEST(FALSE certificates_tests system/darwin/tests/certificates_tests.cpp)

osquery/tables/specs/yara.table

@@ -6,5 +6,7 @@ schema([
     Column("count", INTEGER, "Number of YARA matches"),
     Column("sig_group", TEXT, "Signature group used"),
     Column("sigfile", TEXT, "Signature file used"),
+    Column("pattern", TEXT, "A pattern which can be used to match file paths"),
+
 ])
 implementation("yara@genYara")

osquery/tables/utils/tests/yara_tests.cpp

@@ -0,0 +1,83 @@
+/*
+ *  Copyright (c) 2015, Wesley Shields
+ *  All rights reserved.
+ *
+ *  This source code is licensed under the BSD-style license found in the
+ *  LICENSE file in the root directory of this source tree. An additional grant
+ *  of patent rights can be found in the PATENTS file in the same directory.
+ *
+ */
+
+#include <gtest/gtest.h>
+
+#include <osquery/filesystem.h>
+
+#include "osquery/tables/utils/yara_utils.h"
+
+using namespace osquery::tables;
+
+namespace osquery {
+
+const std::string ruleFile = "/tmp/osquery-yara.sig";
+const std::string ls = "/bin/ls";
+const std::string alwaysTrue = "rule always_true { condition: true }";
+const std::string alwaysFalse = "rule always_false { condition: false }";
+
+class YARATest : public testing::Test {
+ protected:
+  void SetUp() {
+    remove(ruleFile);
+    if (pathExists(ruleFile).ok()) {
+      throw std::domain_error("Rule file exists.");
+    }
+  }
+
+  void TearDown() {
+    remove(ruleFile);
+  }
+
+  Row scanFile(const std::string ruleContent) {
+    YR_RULES* rules = nullptr;
+    int result = yr_initialize();
+    EXPECT_TRUE(result == ERROR_SUCCESS);
+
+    writeTextFile(ruleFile, ruleContent);
+
+    Status status = compileSingleFile(ruleFile, &rules);
+    EXPECT_TRUE(status.ok());
+
+    Row r;
+    r["count"] = "0";
+    r["matches"] = "";
+
+    result = yr_rules_scan_file(rules,
+                                ls.c_str(),
+                                SCAN_FLAGS_FAST_MODE,
+                                YARACallback,
+                                (void*)&r,
+                                0);
+    EXPECT_TRUE(result == ERROR_SUCCESS);
+
+    yr_rules_destroy(rules);
+
+    return r;
+  }
+};
+
+TEST_F(YARATest, test_match_true) {
+  Row r = scanFile(alwaysTrue);
+  // Should have 1 count
+  EXPECT_TRUE(r["count"] == "1");
+}
+
+TEST_F(YARATest, test_match_false) {
+  Row r = scanFile(alwaysFalse);
+  // Should have 0 count
+  EXPECT_TRUE(r["count"] == "0");
+}
+}
+
+int main(int argc, char* argv[]) {
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}

osquery/tables/utils/yara.cpp

@@ -10,6 +10,7 @@
 
 #include <boost/filesystem.hpp>
 
+#include <osquery/filesystem.h>
 #include <osquery/logger.h>
 #include <osquery/tables.h>
 #include <osquery/status.h>
@@ -24,15 +25,52 @@
 namespace osquery {
 namespace tables {
 
+Status doYARAScan(YR_RULES* rules,
+                  const std::string& path,
+                  const std::string& pattern,
+                  QueryData& results,
+                  const std::string& group,
+                  const std::string& sigfile) {
+  Row r;
+
+  // These are default values, to be updated in YARACallback.
+  r["count"] = INTEGER(0);
+  r["matches"] = std::string("");
+
+  // XXX: use target_path instead to be consistent with yara_events?
+  r["path"] = path;
+
+  r["pattern"] = pattern;
+
+  r["sig_group"] = std::string(group);
+  r["sigfile"] = std::string(sigfile);
+
+  int result = yr_rules_scan_file(rules,
+                                  path.c_str(),
+                                  SCAN_FLAGS_FAST_MODE,
+                                  YARACallback,
+                                  (void*)&r,
+                                  0);
+
+  if (result != ERROR_SUCCESS) {
+    return Status(1, "Scan error (" + std::to_string(result) + ")");
+  }
+
+  results.push_back(r);
+  return Status(0, "OK");
+}
+
 QueryData genYara(QueryContext& context) {
   QueryData results;
+  Status status;
 
   auto paths = context.constraints["path"].getAll(EQUALS);
+  auto patterns = context.constraints["pattern"].getAll(EQUALS);
   auto groups = context.constraints["sig_group"].getAll(EQUALS);
   auto sigfiles = context.constraints["sigfile"].getAll(EQUALS);
 
   // Must specify a path constraint and at least one of sig_group or sigfile.
-  if (paths.size() == 0 || (groups.size() == 0 && sigfiles.size() == 0)) {
+  if (groups.size() == 0 && sigfiles.size() == 0) {
     return results;
   }
 
@@ -45,66 +83,96 @@ QueryData genYara(QueryContext& context) {
   const auto& yaraParser = std::static_pointer_cast<YARAConfigParserPlugin>(parser);
   auto rules = yaraParser->rules();
 
-  Row r;
+  // Store resolved paths in a vector of pairs.
+  // Each pair has the first element as the path to scan and the second
+  // element as the pattern which generated it.
+  std::vector<std::pair<std::string, std::string> > path_pairs;
 
-  // These are filled in depending upon what is used to scan.
-  r["sigfile"] = std::string("");
-  r["sig_group"] = std::string("");
+  // Expand patterns and push onto path_pairs.
+  for (const auto& pattern : patterns) {
+    std::vector<std::string> expanded_patterns;
+    auto status = resolveFilePattern(pattern, expanded_patterns);
+    if (!status.ok()) {
+      VLOG(1) << "Could not expand pattern properly: " << status.toString();
+      return results;
+    }
 
+    for (const auto& resolved : expanded_patterns) {
+      if (!isReadable(resolved)) {
+        continue;
+      }
+      path_pairs.push_back(make_pair(resolved, pattern));
+    }
+  }
+
+  // Collect all paths specified too.
   for (const auto& path_string : paths) {
-    boost::filesystem::path path = path_string;
-    if (!boost::filesystem::is_regular_file(path)) {
+    if (!isReadable(path_string)) {
       continue;
     }
+    path_pairs.push_back(make_pair(path_string, ""));
+  }
 
-    // These are default values, to be updated in YARACallback.
-    r["count"] = INTEGER(0);
-    r["matches"] = std::string("");
+  // Compile all sigfiles into a map.
+  std::map<std::string, YR_RULES*> compiled_rules;
+  for (const auto& file : sigfiles) {
+    YR_RULES *rules = nullptr;
 
-    // XXX: use target_path instead to be consistent with yara_events?
-    r["path"] = path_string;
+    std::string full_path;
+    if (file[0] != '/') {
+      full_path = std::string("/var/osquery/") + file;
+    } else {
+      full_path = file;
+    }
 
+    status = compileSingleFile(full_path, &rules);
+    if (!status.ok()) {
+      VLOG(1) << "YARA error: " << status.toString();
+    } else {
+      compiled_rules[file] = rules;
+    }
+  }
+
+  // Scan every path pair.
+  for (const auto& path_pair : path_pairs) {
+    // Scan using siggroups.
     for (const auto& group : groups) {
-      r["sig_group"] = std::string(group);
-      if (rules.count(group) == 0)
+      if (rules.count(group) == 0) {
         continue;
+      }
+
       VLOG(1) << "Scanning with group: " << group;
-      int result = yr_rules_scan_file(rules[group],
-                                      path_string.c_str(),
-                                      SCAN_FLAGS_FAST_MODE,
-                                      YARACallback,
-                                      (void*)&r,
-                                      0);
-
-      if (result != ERROR_SUCCESS) {
-        return results;
+      status = doYARAScan(rules[group],
+                          path_pair.first.c_str(),
+                          path_pair.second,
+                          results,
+                          group,
+                          "");
+      if (!status.ok()) {
+        VLOG(1) << "YARA error: " << status.toString();
       }
-      results.push_back(r);
     }
 
-    for (const auto& file : sigfiles) {
-      YR_RULES* rules = nullptr;
-      r["sigfile"] = std::string(file);
-      VLOG(1) << "Scanning with file: " << file;
-      Status status = compileSingleFile(file, &rules);
-      if (status.ok()) {
-        int result = yr_rules_scan_file(rules,
-                                        path_string.c_str(),
-                                        SCAN_FLAGS_FAST_MODE,
-                                        YARACallback,
-                                        (void*)&r,
-                                        0);
-
-        if (result != ERROR_SUCCESS) {
-          return results;
-        }
-
-        yr_rules_destroy(rules);
+    // Scan using files.
+    for (const auto& element : compiled_rules) {
+      VLOG(1) << "Scanning with file: " << element.first;
+      status = doYARAScan(element.second,
+                          path_pair.first.c_str(),
+                          path_pair.second,
+                          results,
+                          "",
+                          element.first);
+      if (!status.ok()) {
+        VLOG(1) << "YARA error: " << status.toString();
       }
-      results.push_back(r);
     }
   }
 
+  // Cleanup compiled rules
+  for (const auto& element : compiled_rules) {
+    yr_rules_destroy(element.second);
+  }
+
   return results;
 }
 }