From 5a92d2c7f09d78b4ba4f15cc92bf4d41bf5524dc Mon Sep 17 00:00:00 2001 From: uptycs-nishant Date: Sun, 20 Aug 2017 08:29:23 +0530 Subject: [PATCH] Implementing exclude paths for FIM (#3530) --- .../deployment/file-integrity-monitoring.md | 17 +- osquery/config/parsers/file_paths.cpp | 25 +- osquery/events/darwin/fsevents.cpp | 29 +++ osquery/events/darwin/fsevents.h | 11 + .../events/darwin/tests/fsevents_tests.cpp | 34 +++ osquery/events/linux/inotify.cpp | 53 +++-- osquery/events/linux/inotify.h | 18 +- osquery/events/linux/tests/inotify_tests.cpp | 25 ++ osquery/events/pathset.h | 219 ++++++++++++++++++ 9 files changed, 409 insertions(+), 22 deletions(-) create mode 100644 osquery/events/pathset.h diff --git a/docs/wiki/deployment/file-integrity-monitoring.md b/docs/wiki/deployment/file-integrity-monitoring.md index 3bb8daca..b43c4b47 100644 --- a/docs/wiki/deployment/file-integrity-monitoring.md +++ b/docs/wiki/deployment/file-integrity-monitoring.md @@ -19,7 +19,7 @@ To get started with FIM, you must first identify which files and directories you For example, you may want to monitor `/etc` along with other files on a Linux system. After you identify your target files and directories you wish to monitor, add them to a new section in the config *file_paths*. -The two areas below that are relevant to FIM are the scheduled query against `file_events` and the added `file_paths` section. The `file_events` query is scheduled to collect all of the FIM events that have occurred on any files within the paths specified within `file_paths` on a five minute interval. At a high level this means events are buffered within osquery and sent to the configured _logger_ every five minutes. +The three areas below that are relevant to FIM are the scheduled query against `file_events`, the added `file_paths` section and the `exclude_paths` sections. The `file_events` query is scheduled to collect all of the FIM events that have occurred on any files within the paths specified within `file_paths` but excluding the paths specified within `exclude_paths` on a five minute interval. At a high level this means events are buffered within osquery and sent to the configured _logger_ every five minutes. **Note:** You cannot match recursively inside a path. For example `/Users/%%/Configuration.conf` is not a valid wildcard. @@ -49,10 +49,25 @@ The two areas below that are relevant to FIM are the scheduled query against `fi "tmp": [ "/tmp/%%" ] + }, + "exclude_paths": { + "homes": [ + "/home/not_to_monitor/.ssh/%%" + ], + "tmp": [ + "/tmp/too_many_events/" + ] } } ``` +One must not mention arbitrary category name under the exclude_paths node, only valid categories are allowed. + +* `valid category` - Categories which are mentioned under `file_paths` node. In the above example config `homes`, `etc` and `tmp` are termed as valid categories. +* `invalid category` - Any other category name apart from `homes`, `etc` and `tmp` are considered as invalid categories. + +**Note:** Invalid categories get dropped silently, i.e. they don't have any effect on the events generated. + ## Sample Event Output As file changes happen, events will appear in the [**file_events**](https://osquery.io/docs/tables/#file_events) table. During a file change event, the md5, sha1, and sha256 for the file will be calculated if possible. A sample event looks like this: diff --git a/osquery/config/parsers/file_paths.cpp b/osquery/config/parsers/file_paths.cpp index c9756a02..d8a24c0b 100644 --- a/osquery/config/parsers/file_paths.cpp +++ b/osquery/config/parsers/file_paths.cpp @@ -25,7 +25,7 @@ class FilePathsConfigParserPlugin : public ConfigParserPlugin { virtual ~FilePathsConfigParserPlugin() {} std::vector keys() const override { - return {"file_paths", "file_accesses"}; + return {"file_paths", "file_accesses", "exclude_paths"}; } Status setUp() override { return Status(0); }; @@ -40,6 +40,7 @@ class FilePathsConfigParserPlugin : public ConfigParserPlugin { FilePathsConfigParserPlugin::FilePathsConfigParserPlugin() { data_.put_child("file_paths", pt::ptree()); data_.put_child("file_accesses", pt::ptree()); + data_.put_child("exclude_paths", pt::ptree()); } Status FilePathsConfigParserPlugin::update(const std::string& source, @@ -67,6 +68,8 @@ Status FilePathsConfigParserPlugin::update(const std::string& source, } Config::get().removeFiles(source); + + std::set valid_categories; for (const auto& category : data_.get_child("file_paths")) { for (const auto& path : category.second) { auto pattern = path.second.get_value(""); @@ -75,9 +78,29 @@ Status FilePathsConfigParserPlugin::update(const std::string& source, } replaceGlobWildcards(pattern); Config::get().addFile(source, category.first, pattern); + valid_categories.insert(category.first); } } + if (config.count("exclude_paths") > 0) { + data_.put_child("exclude_paths", config.at("exclude_paths")); + } + + std::set invalid_categories; + for (const auto& excl_category : data_.get_child("exclude_paths")) { + if (valid_categories.find(excl_category.first) == valid_categories.end()) { + // valid_categories contains all the valid categories collected from + // traversing "file_paths" above. + invalid_categories.insert(excl_category.first); + } + } + + for (const auto& invalid_category : invalid_categories) { + // invalid_categories contains all the categories which are mentioned in + // exclude_paths but not found in file_paths. + data_.get_child("exclude_paths").erase(invalid_category); + } + return Status(0, "OK"); } diff --git a/osquery/events/darwin/fsevents.cpp b/osquery/events/darwin/fsevents.cpp index 3c0f7d47..84ce311a 100644 --- a/osquery/events/darwin/fsevents.cpp +++ b/osquery/events/darwin/fsevents.cpp @@ -12,6 +12,7 @@ #include +#include #include #include #include @@ -186,10 +187,29 @@ std::set FSEventsEventPublisher::transformSubscription( return paths; } +void FSEventsEventPublisher::buildExcludePathsSet() { + auto parser = Config::getParser("file_paths"); + + WriteLock lock(subscription_lock_); + exclude_paths_.clear(); + for (const auto& excl_category : + parser->getData().get_child("exclude_paths")) { + for (const auto& excl_path : excl_category.second) { + auto pattern = excl_path.second.get_value(""); + if (pattern.empty()) { + continue; + } + exclude_paths_.insert(pattern); + } + } +} + void FSEventsEventPublisher::configure() { // Rebuild the watch paths. stop(); + buildExcludePathsSet(); + { WriteLock lock(mutex_); paths_.clear(); @@ -303,6 +323,15 @@ bool FSEventsEventPublisher::shouldFire( // Compare the event context mask to the subscription context. return false; } + + auto path = ec->path.substr(0, ec->path.rfind('/')); + // Need to have two finds, + // what if somebody excluded an individual file inside a directory + if (!exclude_paths_.empty() && + (exclude_paths_.find(path) || exclude_paths_.find(ec->path))) { + return false; + } + return true; } diff --git a/osquery/events/darwin/fsevents.h b/osquery/events/darwin/fsevents.h index ed194faa..ba3d54ee 100644 --- a/osquery/events/darwin/fsevents.h +++ b/osquery/events/darwin/fsevents.h @@ -21,6 +21,8 @@ #include #include +#include "osquery/events/pathset.h" + namespace osquery { struct FSEventsSubscriptionContext : public SubscriptionContext { @@ -77,6 +79,8 @@ using FSEventsEventContextRef = std::shared_ptr; using FSEventsSubscriptionContextRef = std::shared_ptr; +using ExcludePathSet = PathSet; + /** * @brief An osquery EventPublisher for the Apple FSEvents notification API. * @@ -135,6 +139,9 @@ class FSEventsEventPublisher std::set transformSubscription( FSEventsSubscriptionContextRef& sc) const; + /// Build the set of excluded paths for which events are not to be propogated. + void buildExcludePathsSet(); + private: /// Check if the stream (and run loop) are running. bool isStreamRunning() const; @@ -152,6 +159,9 @@ class FSEventsEventPublisher /// Set of paths to monitor, determined by a configure step. std::set paths_; + /// Events pertaining to these paths not to be propagated. + ExcludePathSet exclude_paths_; + /// Reference to the run loop for this thread. CFRunLoopRef run_loop_{nullptr}; @@ -174,5 +184,6 @@ class FSEventsEventPublisher FRIEND_TEST(FSEventsTests, test_fsevents_fire_event); FRIEND_TEST(FSEventsTests, test_fsevents_event_action); FRIEND_TEST(FSEventsTests, test_fsevents_embedded_wildcards); + FRIEND_TEST(FSEventsTests, test_fsevents_match_subscription); }; } diff --git a/osquery/events/darwin/tests/fsevents_tests.cpp b/osquery/events/darwin/tests/fsevents_tests.cpp index f0b7ff5a..60a632ec 100644 --- a/osquery/events/darwin/tests/fsevents_tests.cpp +++ b/osquery/events/darwin/tests/fsevents_tests.cpp @@ -169,6 +169,40 @@ TEST_F(FSEventsTests, test_fsevents_add_subscription_success) { EventFactory::deregisterEventPublisher("fsevents"); } +TEST_F(FSEventsTests, test_fsevents_match_subscription) { + auto event_pub = std::make_shared(); + EventFactory::registerEventPublisher(event_pub); + + auto sc = event_pub->createSubscriptionContext(); + sc->path = "/etc/%%"; + replaceGlobWildcards(sc->path); + auto subscription = Subscription::create("TestSubscriber", sc); + auto status = EventFactory::addSubscription("fsevents", subscription); + EXPECT_TRUE(status.ok()); + event_pub->configure(); + + std::vector exclude_paths = { + "/etc/ssh/%%", "/etc/", "/etc/ssl/openssl.cnf", "/"}; + for (const auto& path : exclude_paths) { + event_pub->exclude_paths_.insert(path); + } + + { + auto ec = event_pub->createEventContext(); + ec->path = "/private/etc/ssh/ssh_config"; + EXPECT_FALSE(event_pub->shouldFire(sc, ec)); + ec->path = "/private/etc/passwd"; + EXPECT_FALSE(event_pub->shouldFire(sc, ec)); + ec->path = "/private/etc/group"; + EXPECT_FALSE(event_pub->shouldFire(sc, ec)); + ec->path = "/private/etc/ssl/openssl.cnf"; + EXPECT_FALSE(event_pub->shouldFire(sc, ec)); + ec->path = "/private/etc/ssl/certs/"; + EXPECT_TRUE(event_pub->shouldFire(sc, ec)); + } + EventFactory::deregisterEventPublisher("fsevents"); +} + class TestFSEventsEventSubscriber : public EventSubscriber { public: diff --git a/osquery/events/linux/inotify.cpp b/osquery/events/linux/inotify.cpp index 44ccfe67..a79dcf2d 100644 --- a/osquery/events/linux/inotify.cpp +++ b/osquery/events/linux/inotify.cpp @@ -16,6 +16,7 @@ #include +#include #include #include #include @@ -126,6 +127,23 @@ bool INotifyEventPublisher::monitorSubscription( return needMonitoring(discovered, sc, sc->mask, sc->recursive, add_watch); } +void INotifyEventPublisher::buildExcludePathsSet() { + auto parser = Config::getParser("file_paths"); + + WriteLock lock(subscription_lock_); + exclude_paths_.clear(); + for (const auto& excl_category : + parser->getData().get_child("exclude_paths")) { + for (const auto& excl_path : excl_category.second) { + auto pattern = excl_path.second.get_value(""); + if (pattern.empty()) { + continue; + } + exclude_paths_.insert(pattern); + } + } +} + void INotifyEventPublisher::configure() { if (inotify_handle_ == -1) { // This publisher has not been setup correctly. @@ -156,9 +174,10 @@ void INotifyEventPublisher::configure() { } ino_sc->descriptor_paths_.clear(); } - delete_subscriptions.clear(); + buildExcludePathsSet(); + for (auto& sub : subscriptions_) { // Anytime a configure is called, try to monitor all subscriptions. // Configure is called as a response to removing/adding subscriptions. @@ -266,6 +285,7 @@ INotifyEventContextRef INotifyEventPublisher::createEventContextFrom( } else { auto isc = descriptor_inosubctx_.at(event->wd); ec->path = isc->descriptor_paths_.at(event->wd); + ec->isub_ctx = isc; } } @@ -284,26 +304,14 @@ INotifyEventContextRef INotifyEventPublisher::createEventContextFrom( bool INotifyEventPublisher::shouldFire(const INotifySubscriptionContextRef& sc, const INotifyEventContextRef& ec) const { - // The subscription may supply a required event mask. - if (sc->mask != 0 && !(ec->event->mask & sc->mask)) { + if (sc.get() != ec->isub_ctx.get()) { + /// Not my event. return false; } - if (sc->recursive && !sc->recursive_match) { - ssize_t found = ec->path.find(sc->path); - if (found != 0) { - return false; - } - } else if (ec->path == sc->path) { - return true; - } else { - auto flags = FNM_PATHNAME | FNM_CASEFOLD | - ((sc->recursive_match) ? FNM_LEADING_DIR : 0); - if (fnmatch((sc->path + "*").c_str(), ec->path.c_str(), flags) != 0) { - // Only apply a leading-dir match if this is a recursive watch with a - // match requirement (and inline wildcard with ending recursive wildcard). - return false; - } + // The subscription may supply a required event mask. + if (sc->mask != 0 && !(ec->event->mask & sc->mask)) { + return false; } // inotify will not monitor recursively, new directories need watches. @@ -315,6 +323,15 @@ bool INotifyEventPublisher::shouldFire(const INotifySubscriptionContextRef& sc, true); } + // exclude paths should be applied at last + auto path = ec->path.substr(0, ec->path.rfind('/')); + // Need to have two finds, + // what if somebody excluded an individual file inside a directory + if (!exclude_paths_.empty() && + (exclude_paths_.find(path) || exclude_paths_.find(ec->path))) { + return false; + } + return true; } diff --git a/osquery/events/linux/inotify.h b/osquery/events/linux/inotify.h index 3a54f20e..04bc75e2 100644 --- a/osquery/events/linux/inotify.h +++ b/osquery/events/linux/inotify.h @@ -18,6 +18,8 @@ #include +#include "osquery/events/pathset.h" + namespace osquery { extern std::map kMaskActions; @@ -95,6 +97,9 @@ inline bool operator==(const INotifySubscriptionContext& lsc, return ((lsc.category == rsc.category) && (lsc.opath == rsc.opath)); } +using INotifySubscriptionContextRef = + std::shared_ptr; + /** * @brief Event details for INotifyEventPublisher events. */ @@ -110,15 +115,18 @@ struct INotifyEventContext : public EventContext { /// A no-op event transaction id. uint32_t transaction_id{0}; + + /// This event ctx belongs to isub_ctx + INotifySubscriptionContextRef isub_ctx; }; using INotifyEventContextRef = std::shared_ptr; -using INotifySubscriptionContextRef = - std::shared_ptr; // Publisher container using DescriptorINotifySubCtxMap = std::map; +using ExcludePathSet = PathSet; + /** * @brief A Linux `inotify` EventPublisher. * @@ -215,6 +223,9 @@ class INotifyEventPublisher bool monitorSubscription(INotifySubscriptionContextRef& sc, bool add_watch = true); + /// Build the set of excluded paths for which events are not to be propogated. + void buildExcludePathsSet(); + /// Remove an INotify watch (monitor) from our tracking. bool removeMonitor(int watch, bool force = false, bool batch_del = false); @@ -242,6 +253,9 @@ class INotifyEventPublisher /// Map of inotify watch file descriptor to subscription context. DescriptorINotifySubCtxMap descriptor_inosubctx_; + /// Events pertaining to these paths not to be propagated. + ExcludePathSet exclude_paths_; + /// The inotify file descriptor handle. std::atomic inotify_handle_{-1}; diff --git a/osquery/events/linux/tests/inotify_tests.cpp b/osquery/events/linux/tests/inotify_tests.cpp index 87aefb6a..385bfed2 100644 --- a/osquery/events/linux/tests/inotify_tests.cpp +++ b/osquery/events/linux/tests/inotify_tests.cpp @@ -231,11 +231,36 @@ TEST_F(INotifyTests, test_inotify_match_subscription) { sc->path = dir; event_pub_->monitorSubscription(sc, false); auto ec = event_pub_->createEventContext(); + ec->isub_ctx = sc; ec->path = "/etc/"; EXPECT_TRUE(event_pub_->shouldFire(sc, ec)); ec->path = "/etc/passwd"; EXPECT_TRUE(event_pub_->shouldFire(sc, ec)); } + + std::vector exclude_paths = { + "/etc/ssh/%%", "/etc/", "/etc/ssl/openssl.cnf", "/"}; + for (const auto& path : exclude_paths) { + event_pub_->exclude_paths_.insert(path); + } + + { + event_pub_->path_descriptors_.clear(); + auto sc = event_pub_->createSubscriptionContext(); + sc->path = "/etc/%%"; + auto ec = event_pub_->createEventContext(); + ec->isub_ctx = sc; + ec->path = "/etc/ssh/ssh_config"; + EXPECT_FALSE(event_pub_->shouldFire(sc, ec)); + ec->path = "/etc/passwd"; + EXPECT_FALSE(event_pub_->shouldFire(sc, ec)); + ec->path = "/etc/group"; + EXPECT_FALSE(event_pub_->shouldFire(sc, ec)); + ec->path = "/etc/ssl/openssl.cnf"; + EXPECT_FALSE(event_pub_->shouldFire(sc, ec)); + ec->path = "/etc/ssl/certs/"; + EXPECT_TRUE(event_pub_->shouldFire(sc, ec)); + } } class TestINotifyEventSubscriber diff --git a/osquery/events/pathset.h b/osquery/events/pathset.h new file mode 100644 index 00000000..c96310f4 --- /dev/null +++ b/osquery/events/pathset.h @@ -0,0 +1,219 @@ +/* + * Copyright (c) 2014-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ + +#pragma once + +#include +#include +#include +#include + +#include +#include + +#include +#include + +namespace osquery { + +/** + * @brief multiset based implemention for path search. + * + * 'multiset' is used because with patterns we can serach for equivalent keys. + * Since '/This/Path/is' ~= '/This/Path/%' ~= '/This/Path/%%' (equivalent). + * + * multiset is protected by lock. It is threadsafe. + * + * PathSet can take any of the two policies - + * 1. patternedPath - Path can contain pattern '%' and '%%'. + * Path components containing only '%' and '%%' are supported + * e.g. '/This/Path/%'. + * Path components containing partial patterns are not + * supported e.g. '/This/Path/xyz%' ('xyz%' will not be + * treated as pattern). + * + * 2. resolvedPath - path is resolved before being inserted into set. + * But path can match recursively. + * + */ +template +class PathSet : private boost::noncopyable { + public: + void insert(const std::string& str) { + auto pattern = str; + replaceGlobWildcards(pattern); + auto vpath = PathType::createVPath(pattern); + + WriteLock lock(mset_lock_); + for (auto& path : vpath) { + paths_.insert(std::move(path)); + } + } + + bool find(const std::string& str) const { + auto path = PathType::createPath(str); + + ReadLock lock(mset_lock_); + if (paths_.find(path) != paths_.end()) { + return true; + } + return false; + } + + void clear() { + WriteLock lock(mset_lock_); + paths_.clear(); + } + + bool empty() const { + ReadLock lock(mset_lock_); + return paths_.empty(); + } + + private: + typedef typename PathType::Path Path; + typedef typename PathType::Compare Compare; + std::multiset paths_; + mutable Mutex mset_lock_; +}; + +class patternedPath { + public: + typedef boost::tokenizer> tokenizer; + typedef std::vector Path; + typedef std::vector VPath; + struct Compare { + bool operator()(const Path& lhs, const Path& rhs) const { + size_t psize = (lhs.size() < rhs.size()) ? lhs.size() : rhs.size(); + unsigned ndx; + for (ndx = 0; ndx < psize; ++ndx) { + if (lhs[ndx] == "**" || rhs[ndx] == "**") { + return false; + } + + if (lhs[ndx] == "*" || rhs[ndx] == "*") { + continue; + } + + int rc = lhs[ndx].compare(rhs[ndx]); + + if (rc > 0) { + return false; + } + + if (rc < 0) { + return true; + } + } + + if ((ndx == rhs.size() && rhs[ndx - 1] == "*") || + (ndx == lhs.size() && lhs[ndx - 1] == "*")) { + return false; + } + + return (lhs.size() < rhs.size()); + } + }; + + static Path createPath(const std::string& str) { + boost::char_separator sep{"/"}; + tokenizer tokens(str, sep); + Path path; + + if (str == "/") { + path.push_back(""); + } + + for (std::string component : tokens) { + path.push_back(std::move(component)); + } + return path; + } + + static VPath createVPath(const std::string& str) { + boost::char_separator sep{"/"}; + tokenizer tokens(str, sep); + VPath vpath; + Path path; + + if (str == "/") { + path.push_back(""); + } + + for (std::string component : tokens) { + if (component == "**") { + vpath.push_back(path); + path.push_back(std::move(component)); + break; + } + path.push_back(std::move(component)); + } + vpath.push_back(std::move(path)); + return vpath; + } +}; + +class resolvedPath { + public: + struct Path { + Path(const std::string& str, bool r = false) : path(str), recursive(r) {} + const std::string path; + bool recursive{false}; + }; + typedef std::vector VPath; + + struct Compare { + bool operator()(const Path& lhs, const Path& rhs) const { + size_t size = (lhs.path.size() < rhs.path.size()) ? lhs.path.size() + : rhs.path.size(); + + int rc = lhs.path.compare(0, size, rhs.path, 0, size); + + if (rc > 0) { + return false; + } + + if (rc < 0) { + return true; + } + + if ((size < rhs.path.size() && lhs.recursive) || + (size < lhs.path.size() && rhs.recursive)) { + return false; + } + + return (lhs.path.size() < rhs.path.size()); + } + }; + + static Path createPath(const std::string& str) { + return Path(str); + } + + static VPath createVPath(const std::string& str) { + bool recursive = false; + std::string pattern(str); + if (pattern.find("**") != std::string::npos) { + recursive = true; + pattern = pattern.substr(0, pattern.find("**")); + } + + std::vector paths; + resolveFilePattern(pattern, paths); + + VPath vpath; + for (const auto& path : paths) { + vpath.push_back(Path(path, recursive)); + } + return vpath; + } +}; + +} // namespace osquery