Implementing exclude paths for FIM (#3530)

docs/wiki/deployment/file-integrity-monitoring.md

@@ -19,7 +19,7 @@ To get started with FIM, you must first identify which files and directories you
 
 For example, you may want to monitor `/etc` along with other files on a Linux system. After you identify your target files and directories you wish to monitor, add them to a new section in the config *file_paths*.
 
-The two areas below that are relevant to FIM are the scheduled query against `file_events` and the added `file_paths` section. The `file_events` query is scheduled to collect all of the FIM events that have occurred on any files within the paths specified within `file_paths` on a five minute interval. At a high level this means events are buffered within osquery and sent to the configured _logger_ every five minutes.
+The three areas below that are relevant to FIM are the scheduled query against `file_events`, the added `file_paths` section and the `exclude_paths` sections. The `file_events` query is scheduled to collect all of the FIM events that have occurred on any files within the paths specified within `file_paths` but excluding the paths specified within `exclude_paths` on a five minute interval. At a high level this means events are buffered within osquery and sent to the configured _logger_ every five minutes.
 
 **Note:** You cannot match recursively inside a path. For example `/Users/%%/Configuration.conf` is not a valid wildcard.
 
@@ -49,10 +49,25 @@ The two areas below that are relevant to FIM are the scheduled query against `fi
     "tmp": [
       "/tmp/%%"
     ]
+  },
+  "exclude_paths": {
+    "homes": [
+      "/home/not_to_monitor/.ssh/%%"
+    ],
+    "tmp": [
+      "/tmp/too_many_events/"
+    ]
   }
 }
 ```
 
+One must not mention arbitrary category name under the exclude_paths node, only valid categories are allowed.
+
+* `valid category` - Categories which are mentioned under `file_paths` node. In the above example config `homes`, `etc` and `tmp` are termed as valid categories.
+* `invalid category` - Any other category name apart from `homes`, `etc` and `tmp` are considered as invalid categories.
+
+**Note:** Invalid categories get dropped silently, i.e. they don't have any effect on the events generated.
+
 ## Sample Event Output
 
 As file changes happen, events will appear in the [**file_events**](https://osquery.io/docs/tables/#file_events) table.  During a file change event, the md5, sha1, and sha256 for the file will be calculated if possible. A sample event looks like this:

osquery/config/parsers/file_paths.cpp

@@ -25,7 +25,7 @@ class FilePathsConfigParserPlugin : public ConfigParserPlugin {
   virtual ~FilePathsConfigParserPlugin() {}
 
   std::vector<std::string> keys() const override {
-    return {"file_paths", "file_accesses"};
+    return {"file_paths", "file_accesses", "exclude_paths"};
   }
 
   Status setUp() override { return Status(0); };
@@ -40,6 +40,7 @@ class FilePathsConfigParserPlugin : public ConfigParserPlugin {
 FilePathsConfigParserPlugin::FilePathsConfigParserPlugin() {
   data_.put_child("file_paths", pt::ptree());
   data_.put_child("file_accesses", pt::ptree());
+  data_.put_child("exclude_paths", pt::ptree());
 }
 
 Status FilePathsConfigParserPlugin::update(const std::string& source,
@@ -67,6 +68,8 @@ Status FilePathsConfigParserPlugin::update(const std::string& source,
   }
 
   Config::get().removeFiles(source);
+
+  std::set<std::string> valid_categories;
   for (const auto& category : data_.get_child("file_paths")) {
     for (const auto& path : category.second) {
       auto pattern = path.second.get_value<std::string>("");
@@ -75,9 +78,29 @@ Status FilePathsConfigParserPlugin::update(const std::string& source,
       }
       replaceGlobWildcards(pattern);
       Config::get().addFile(source, category.first, pattern);
+      valid_categories.insert(category.first);
     }
   }
 
+  if (config.count("exclude_paths") > 0) {
+    data_.put_child("exclude_paths", config.at("exclude_paths"));
+  }
+
+  std::set<std::string> invalid_categories;
+  for (const auto& excl_category : data_.get_child("exclude_paths")) {
+    if (valid_categories.find(excl_category.first) == valid_categories.end()) {
+      // valid_categories contains all the valid categories collected from
+      // traversing "file_paths" above.
+      invalid_categories.insert(excl_category.first);
+    }
+  }
+
+  for (const auto& invalid_category : invalid_categories) {
+    // invalid_categories contains all the categories which are mentioned in
+    // exclude_paths but not found in file_paths.
+    data_.get_child("exclude_paths").erase(invalid_category);
+  }
+
   return Status(0, "OK");
 }
 

osquery/events/darwin/fsevents.cpp

@@ -12,6 +12,7 @@
 
 #include <boost/filesystem.hpp>
 
+#include <osquery/config.h>
 #include <osquery/filesystem.h>
 #include <osquery/logger.h>
 #include <osquery/tables.h>
@@ -186,10 +187,29 @@ std::set<std::string> FSEventsEventPublisher::transformSubscription(
   return paths;
 }
 
+void FSEventsEventPublisher::buildExcludePathsSet() {
+  auto parser = Config::getParser("file_paths");
+
+  WriteLock lock(subscription_lock_);
+  exclude_paths_.clear();
+  for (const auto& excl_category :
+       parser->getData().get_child("exclude_paths")) {
+    for (const auto& excl_path : excl_category.second) {
+      auto pattern = excl_path.second.get_value<std::string>("");
+      if (pattern.empty()) {
+        continue;
+      }
+      exclude_paths_.insert(pattern);
+    }
+  }
+}
+
 void FSEventsEventPublisher::configure() {
   // Rebuild the watch paths.
   stop();
 
+  buildExcludePathsSet();
+
   {
     WriteLock lock(mutex_);
     paths_.clear();
@@ -303,6 +323,15 @@ bool FSEventsEventPublisher::shouldFire(
     // Compare the event context mask to the subscription context.
     return false;
   }
+
+  auto path = ec->path.substr(0, ec->path.rfind('/'));
+  // Need to have two finds,
+  // what if somebody excluded an individual file inside a directory
+  if (!exclude_paths_.empty() &&
+      (exclude_paths_.find(path) || exclude_paths_.find(ec->path))) {
+    return false;
+  }
+
   return true;
 }
 

osquery/events/darwin/fsevents.h

@@ -21,6 +21,8 @@
 #include <osquery/events.h>
 #include <osquery/status.h>
 
+#include "osquery/events/pathset.h"
+
 namespace osquery {
 
 struct FSEventsSubscriptionContext : public SubscriptionContext {
@@ -77,6 +79,8 @@ using FSEventsEventContextRef = std::shared_ptr<FSEventsEventContext>;
 using FSEventsSubscriptionContextRef =
     std::shared_ptr<FSEventsSubscriptionContext>;
 
+using ExcludePathSet = PathSet<patternedPath>;
+
 /**
  * @brief An osquery EventPublisher for the Apple FSEvents notification API.
  *
@@ -135,6 +139,9 @@ class FSEventsEventPublisher
   std::set<std::string> transformSubscription(
       FSEventsSubscriptionContextRef& sc) const;
 
+  /// Build the set of excluded paths for which events are not to be propogated.
+  void buildExcludePathsSet();
+
  private:
   /// Check if the stream (and run loop) are running.
   bool isStreamRunning() const;
@@ -152,6 +159,9 @@ class FSEventsEventPublisher
   /// Set of paths to monitor, determined by a configure step.
   std::set<std::string> paths_;
 
+  /// Events pertaining to these paths not to be propagated.
+  ExcludePathSet exclude_paths_;
+
   /// Reference to the run loop for this thread.
   CFRunLoopRef run_loop_{nullptr};
 
@@ -174,5 +184,6 @@ class FSEventsEventPublisher
   FRIEND_TEST(FSEventsTests, test_fsevents_fire_event);
   FRIEND_TEST(FSEventsTests, test_fsevents_event_action);
   FRIEND_TEST(FSEventsTests, test_fsevents_embedded_wildcards);
+  FRIEND_TEST(FSEventsTests, test_fsevents_match_subscription);
 };
 }

osquery/events/darwin/tests/fsevents_tests.cpp

@@ -169,6 +169,40 @@ TEST_F(FSEventsTests, test_fsevents_add_subscription_success) {
   EventFactory::deregisterEventPublisher("fsevents");
 }
 
+TEST_F(FSEventsTests, test_fsevents_match_subscription) {
+  auto event_pub = std::make_shared<FSEventsEventPublisher>();
+  EventFactory::registerEventPublisher(event_pub);
+
+  auto sc = event_pub->createSubscriptionContext();
+  sc->path = "/etc/%%";
+  replaceGlobWildcards(sc->path);
+  auto subscription = Subscription::create("TestSubscriber", sc);
+  auto status = EventFactory::addSubscription("fsevents", subscription);
+  EXPECT_TRUE(status.ok());
+  event_pub->configure();
+
+  std::vector<std::string> exclude_paths = {
+      "/etc/ssh/%%", "/etc/", "/etc/ssl/openssl.cnf", "/"};
+  for (const auto& path : exclude_paths) {
+    event_pub->exclude_paths_.insert(path);
+  }
+
+  {
+    auto ec = event_pub->createEventContext();
+    ec->path = "/private/etc/ssh/ssh_config";
+    EXPECT_FALSE(event_pub->shouldFire(sc, ec));
+    ec->path = "/private/etc/passwd";
+    EXPECT_FALSE(event_pub->shouldFire(sc, ec));
+    ec->path = "/private/etc/group";
+    EXPECT_FALSE(event_pub->shouldFire(sc, ec));
+    ec->path = "/private/etc/ssl/openssl.cnf";
+    EXPECT_FALSE(event_pub->shouldFire(sc, ec));
+    ec->path = "/private/etc/ssl/certs/";
+    EXPECT_TRUE(event_pub->shouldFire(sc, ec));
+  }
+  EventFactory::deregisterEventPublisher("fsevents");
+}
+
 class TestFSEventsEventSubscriber
     : public EventSubscriber<FSEventsEventPublisher> {
  public:

osquery/events/linux/inotify.cpp

@@ -16,6 +16,7 @@
 
 #include <boost/filesystem.hpp>
 
+#include <osquery/config.h>
 #include <osquery/filesystem.h>
 #include <osquery/logger.h>
 #include <osquery/system.h>
@@ -126,6 +127,23 @@ bool INotifyEventPublisher::monitorSubscription(
   return needMonitoring(discovered, sc, sc->mask, sc->recursive, add_watch);
 }
 
+void INotifyEventPublisher::buildExcludePathsSet() {
+  auto parser = Config::getParser("file_paths");
+
+  WriteLock lock(subscription_lock_);
+  exclude_paths_.clear();
+  for (const auto& excl_category :
+       parser->getData().get_child("exclude_paths")) {
+    for (const auto& excl_path : excl_category.second) {
+      auto pattern = excl_path.second.get_value<std::string>("");
+      if (pattern.empty()) {
+        continue;
+      }
+      exclude_paths_.insert(pattern);
+    }
+  }
+}
+
 void INotifyEventPublisher::configure() {
   if (inotify_handle_ == -1) {
     // This publisher has not been setup correctly.
@@ -156,9 +174,10 @@ void INotifyEventPublisher::configure() {
     }
     ino_sc->descriptor_paths_.clear();
   }
-
   delete_subscriptions.clear();
 
+  buildExcludePathsSet();
+
   for (auto& sub : subscriptions_) {
     // Anytime a configure is called, try to monitor all subscriptions.
     // Configure is called as a response to removing/adding subscriptions.
@@ -266,6 +285,7 @@ INotifyEventContextRef INotifyEventPublisher::createEventContextFrom(
     } else {
       auto isc = descriptor_inosubctx_.at(event->wd);
       ec->path = isc->descriptor_paths_.at(event->wd);
+      ec->isub_ctx = isc;
     }
   }
 
@@ -284,26 +304,14 @@ INotifyEventContextRef INotifyEventPublisher::createEventContextFrom(
 
 bool INotifyEventPublisher::shouldFire(const INotifySubscriptionContextRef& sc,
                                        const INotifyEventContextRef& ec) const {
-  // The subscription may supply a required event mask.
-  if (sc->mask != 0 && !(ec->event->mask & sc->mask)) {
+  if (sc.get() != ec->isub_ctx.get()) {
+    /// Not my event.
     return false;
   }
 
-  if (sc->recursive && !sc->recursive_match) {
-    ssize_t found = ec->path.find(sc->path);
-    if (found != 0) {
-      return false;
-    }
-  } else if (ec->path == sc->path) {
-    return true;
-  } else {
-    auto flags = FNM_PATHNAME | FNM_CASEFOLD |
-                 ((sc->recursive_match) ? FNM_LEADING_DIR : 0);
-    if (fnmatch((sc->path + "*").c_str(), ec->path.c_str(), flags) != 0) {
-      // Only apply a leading-dir match if this is a recursive watch with a
-      // match requirement (and inline wildcard with ending recursive wildcard).
-      return false;
-    }
+  // The subscription may supply a required event mask.
+  if (sc->mask != 0 && !(ec->event->mask & sc->mask)) {
+    return false;
   }
 
   // inotify will not monitor recursively, new directories need watches.
@@ -315,6 +323,15 @@ bool INotifyEventPublisher::shouldFire(const INotifySubscriptionContextRef& sc,
         true);
   }
 
+  // exclude paths should be applied at last
+  auto path = ec->path.substr(0, ec->path.rfind('/'));
+  // Need to have two finds,
+  // what if somebody excluded an individual file inside a directory
+  if (!exclude_paths_.empty() &&
+      (exclude_paths_.find(path) || exclude_paths_.find(ec->path))) {
+    return false;
+  }
+
   return true;
 }
 

osquery/events/linux/inotify.h

@@ -18,6 +18,8 @@
 
 #include <osquery/events.h>
 
+#include "osquery/events/pathset.h"
+
 namespace osquery {
 
 extern std::map<int, std::string> kMaskActions;
@@ -95,6 +97,9 @@ inline bool operator==(const INotifySubscriptionContext& lsc,
   return ((lsc.category == rsc.category) && (lsc.opath == rsc.opath));
 }
 
+using INotifySubscriptionContextRef =
+    std::shared_ptr<INotifySubscriptionContext>;
+
 /**
  * @brief Event details for INotifyEventPublisher events.
  */
@@ -110,15 +115,18 @@ struct INotifyEventContext : public EventContext {
 
   /// A no-op event transaction id.
   uint32_t transaction_id{0};
+
+  /// This event ctx belongs to isub_ctx
+  INotifySubscriptionContextRef isub_ctx;
 };
 
 using INotifyEventContextRef = std::shared_ptr<INotifyEventContext>;
-using INotifySubscriptionContextRef =
-    std::shared_ptr<INotifySubscriptionContext>;
 
 // Publisher container
 using DescriptorINotifySubCtxMap = std::map<int, INotifySubscriptionContextRef>;
 
+using ExcludePathSet = PathSet<patternedPath>;
+
 /**
  * @brief A Linux `inotify` EventPublisher.
  *
@@ -215,6 +223,9 @@ class INotifyEventPublisher
   bool monitorSubscription(INotifySubscriptionContextRef& sc,
                            bool add_watch = true);
 
+  /// Build the set of excluded paths for which events are not to be propogated.
+  void buildExcludePathsSet();
+
   /// Remove an INotify watch (monitor) from our tracking.
   bool removeMonitor(int watch, bool force = false, bool batch_del = false);
 
@@ -242,6 +253,9 @@ class INotifyEventPublisher
   /// Map of inotify watch file descriptor to subscription context.
   DescriptorINotifySubCtxMap descriptor_inosubctx_;
 
+  /// Events pertaining to these paths not to be propagated.
+  ExcludePathSet exclude_paths_;
+
   /// The inotify file descriptor handle.
   std::atomic<int> inotify_handle_{-1};
 

osquery/events/linux/tests/inotify_tests.cpp

@@ -231,11 +231,36 @@ TEST_F(INotifyTests, test_inotify_match_subscription) {
     sc->path = dir;
     event_pub_->monitorSubscription(sc, false);
     auto ec = event_pub_->createEventContext();
+    ec->isub_ctx = sc;
     ec->path = "/etc/";
     EXPECT_TRUE(event_pub_->shouldFire(sc, ec));
     ec->path = "/etc/passwd";
     EXPECT_TRUE(event_pub_->shouldFire(sc, ec));
   }
+
+  std::vector<std::string> exclude_paths = {
+      "/etc/ssh/%%", "/etc/", "/etc/ssl/openssl.cnf", "/"};
+  for (const auto& path : exclude_paths) {
+    event_pub_->exclude_paths_.insert(path);
+  }
+
+  {
+    event_pub_->path_descriptors_.clear();
+    auto sc = event_pub_->createSubscriptionContext();
+    sc->path = "/etc/%%";
+    auto ec = event_pub_->createEventContext();
+    ec->isub_ctx = sc;
+    ec->path = "/etc/ssh/ssh_config";
+    EXPECT_FALSE(event_pub_->shouldFire(sc, ec));
+    ec->path = "/etc/passwd";
+    EXPECT_FALSE(event_pub_->shouldFire(sc, ec));
+    ec->path = "/etc/group";
+    EXPECT_FALSE(event_pub_->shouldFire(sc, ec));
+    ec->path = "/etc/ssl/openssl.cnf";
+    EXPECT_FALSE(event_pub_->shouldFire(sc, ec));
+    ec->path = "/etc/ssl/certs/";
+    EXPECT_TRUE(event_pub_->shouldFire(sc, ec));
+  }
 }
 
 class TestINotifyEventSubscriber

osquery/events/pathset.h

@@ -0,0 +1,219 @@
+/*
+ *  Copyright (c) 2014-present, Facebook, Inc.
+ *  All rights reserved.
+ *
+ *  This source code is licensed under the BSD-style license found in the
+ *  LICENSE file in the root directory of this source tree. An additional grant
+ *  of patent rights can be found in the PATENTS file in the same directory.
+ *
+ */
+
+#pragma once
+
+#include <mutex>
+#include <set>
+#include <string>
+#include <vector>
+
+#include <boost/noncopyable.hpp>
+#include <boost/tokenizer.hpp>
+
+#include <osquery/core.h>
+#include <osquery/filesystem.h>
+
+namespace osquery {
+
+/**
+ * @brief multiset based implemention for path search.
+ *
+ * 'multiset' is used because with patterns we can serach for equivalent keys.
+ * Since  '/This/Path/is' ~= '/This/Path/%' ~= '/This/Path/%%' (equivalent).
+ *
+ * multiset is protected by lock. It is threadsafe.
+ *
+ * PathSet can take any of the two policies -
+ * 1. patternedPath - Path can contain pattern '%' and '%%'.
+ *                    Path components containing only '%' and '%%' are supported
+ *                    e.g. '/This/Path/%'.
+ *                    Path components containing partial patterns are not
+ *                    supported e.g. '/This/Path/xyz%' ('xyz%' will not be
+ *                    treated as pattern).
+ *
+ * 2. resolvedPath - path is resolved before being inserted into set.
+ *                   But path can match recursively.
+ *
+ */
+template <typename PathType>
+class PathSet : private boost::noncopyable {
+ public:
+  void insert(const std::string& str) {
+    auto pattern = str;
+    replaceGlobWildcards(pattern);
+    auto vpath = PathType::createVPath(pattern);
+
+    WriteLock lock(mset_lock_);
+    for (auto& path : vpath) {
+      paths_.insert(std::move(path));
+    }
+  }
+
+  bool find(const std::string& str) const {
+    auto path = PathType::createPath(str);
+
+    ReadLock lock(mset_lock_);
+    if (paths_.find(path) != paths_.end()) {
+      return true;
+    }
+    return false;
+  }
+
+  void clear() {
+    WriteLock lock(mset_lock_);
+    paths_.clear();
+  }
+
+  bool empty() const {
+    ReadLock lock(mset_lock_);
+    return paths_.empty();
+  }
+
+ private:
+  typedef typename PathType::Path Path;
+  typedef typename PathType::Compare Compare;
+  std::multiset<Path, Compare> paths_;
+  mutable Mutex mset_lock_;
+};
+
+class patternedPath {
+ public:
+  typedef boost::tokenizer<boost::char_separator<char>> tokenizer;
+  typedef std::vector<std::string> Path;
+  typedef std::vector<Path> VPath;
+  struct Compare {
+    bool operator()(const Path& lhs, const Path& rhs) const {
+      size_t psize = (lhs.size() < rhs.size()) ? lhs.size() : rhs.size();
+      unsigned ndx;
+      for (ndx = 0; ndx < psize; ++ndx) {
+        if (lhs[ndx] == "**" || rhs[ndx] == "**") {
+          return false;
+        }
+
+        if (lhs[ndx] == "*" || rhs[ndx] == "*") {
+          continue;
+        }
+
+        int rc = lhs[ndx].compare(rhs[ndx]);
+
+        if (rc > 0) {
+          return false;
+        }
+
+        if (rc < 0) {
+          return true;
+        }
+      }
+
+      if ((ndx == rhs.size() && rhs[ndx - 1] == "*") ||
+          (ndx == lhs.size() && lhs[ndx - 1] == "*")) {
+        return false;
+      }
+
+      return (lhs.size() < rhs.size());
+    }
+  };
+
+  static Path createPath(const std::string& str) {
+    boost::char_separator<char> sep{"/"};
+    tokenizer tokens(str, sep);
+    Path path;
+
+    if (str == "/") {
+      path.push_back("");
+    }
+
+    for (std::string component : tokens) {
+      path.push_back(std::move(component));
+    }
+    return path;
+  }
+
+  static VPath createVPath(const std::string& str) {
+    boost::char_separator<char> sep{"/"};
+    tokenizer tokens(str, sep);
+    VPath vpath;
+    Path path;
+
+    if (str == "/") {
+      path.push_back("");
+    }
+
+    for (std::string component : tokens) {
+      if (component == "**") {
+        vpath.push_back(path);
+        path.push_back(std::move(component));
+        break;
+      }
+      path.push_back(std::move(component));
+    }
+    vpath.push_back(std::move(path));
+    return vpath;
+  }
+};
+
+class resolvedPath {
+ public:
+  struct Path {
+    Path(const std::string& str, bool r = false) : path(str), recursive(r) {}
+    const std::string path;
+    bool recursive{false};
+  };
+  typedef std::vector<Path> VPath;
+
+  struct Compare {
+    bool operator()(const Path& lhs, const Path& rhs) const {
+      size_t size = (lhs.path.size() < rhs.path.size()) ? lhs.path.size()
+                                                        : rhs.path.size();
+
+      int rc = lhs.path.compare(0, size, rhs.path, 0, size);
+
+      if (rc > 0) {
+        return false;
+      }
+
+      if (rc < 0) {
+        return true;
+      }
+
+      if ((size < rhs.path.size() && lhs.recursive) ||
+          (size < lhs.path.size() && rhs.recursive)) {
+        return false;
+      }
+
+      return (lhs.path.size() < rhs.path.size());
+    }
+  };
+
+  static Path createPath(const std::string& str) {
+    return Path(str);
+  }
+
+  static VPath createVPath(const std::string& str) {
+    bool recursive = false;
+    std::string pattern(str);
+    if (pattern.find("**") != std::string::npos) {
+      recursive = true;
+      pattern = pattern.substr(0, pattern.find("**"));
+    }
+
+    std::vector<std::string> paths;
+    resolveFilePattern(pattern, paths);
+
+    VPath vpath;
+    for (const auto& path : paths) {
+      vpath.push_back(Path(path, recursive));
+    }
+    return vpath;
+  }
+};
+
+} // namespace osquery