diff --git a/docs/wiki/deployment/configuration.md b/docs/wiki/deployment/configuration.md index f5c857bd..4a8508b3 100644 --- a/docs/wiki/deployment/configuration.md +++ b/docs/wiki/deployment/configuration.md @@ -135,6 +135,17 @@ The pack value may also be a string, such as: If using a string instead of an inline JSON dictionary the configuration plugin will be asked to "generate" that resource. In the case of the default **filesystem** plugin, these strings are considered paths. +The **filesystem** plugin supports another convention for adding a directory of packs: +```json +{ + "packs": { + "*": "/path/to/*", + } +} +``` + +Here the name `*` asks the plugin to *glob* the value and construct a multi-pack. The name of each pack will correspond to the filename *leaf* without the final extension, e.g. `/path/to/external_pack.conf` will be named `external_pack`. + Queries added to the schedule from packs inherit the pack name as part of the scheduled query name identifier. For example, consider the embedded `active_directory` query above, it is in the `internal_stuff` pack so the scheduled query name becomes: `pack_internal_stuff_active_directory`. The delimiter can be changed using the `--pack_delimiter=_`, see the [CLI Options](../installation/cli-flags.md) for more details. ### Discovery queries diff --git a/include/osquery/config.h b/include/osquery/config.h index 7cecb3c2..de05ed67 100644 --- a/include/osquery/config.h +++ b/include/osquery/config.h @@ -113,10 +113,14 @@ class Config : private boost::noncopyable { void hashSource(const std::string& source, const std::string& content); /// Whether or not the last loaded config was valid. - bool isValid() const { return valid_; } + bool isValid() const { + return valid_; + } /// Get start time of config. - size_t getStartTime() const { return start_time_; } + size_t getStartTime() const { + return start_time_; + } /** * @brief Add a pack to the osquery schedule @@ -322,6 +326,7 @@ class Config : private boost::noncopyable { FRIEND_TEST(OptionsConfigParserPluginTests, test_get_option); FRIEND_TEST(EventsConfigParserPluginTests, test_get_event); FRIEND_TEST(PacksTests, test_discovery_cache); + FRIEND_TEST(PacksTests, test_multi_pack); FRIEND_TEST(SchedulerTests, test_monitor); FRIEND_TEST(SchedulerTests, test_config_results_purge); FRIEND_TEST(EventsTests, test_event_subscriber_configure); @@ -488,7 +493,9 @@ class ConfigParserPlugin : public Plugin { * * More complex parsers that require dynamic casting are not recommended. */ - const boost::property_tree::ptree& getData() const { return data_; } + const boost::property_tree::ptree& getData() const { + return data_; + } protected: /// Allow the config to request parser state resets. @@ -501,4 +508,16 @@ class ConfigParserPlugin : public Plugin { private: friend class Config; }; + +/** + * @brief Boost's 1.59 property tree based JSON parser does not accept comments. + * + * For semi-compatibility with existing configurations we will attempt to strip + * hash and C++ style comments. It is OK for the config update to be latent + * as it is a single event. But some configuration plugins may update running + * configurations. + * + * @parms json A mutable input/output string that will contain stripped JSON. + */ +void stripConfigComments(std::string& json); } diff --git a/include/osquery/filesystem.h b/include/osquery/filesystem.h index 1ce8d424..fe07173f 100644 --- a/include/osquery/filesystem.h +++ b/include/osquery/filesystem.h @@ -36,9 +36,9 @@ inline GlobLimits operator|(GlobLimits a, GlobLimits b) { } /// Globbing wildcard character. -const std::string kSQLGlobWildcard = "%"; +const std::string kSQLGlobWildcard{"%"}; /// Globbing wildcard recursive character (double wildcard). -const std::string kSQLGlobRecursive = kSQLGlobWildcard + kSQLGlobWildcard; +const std::string kSQLGlobRecursive{kSQLGlobWildcard + kSQLGlobWildcard}; /** * @brief Read a file from disk. diff --git a/include/osquery/packs.h b/include/osquery/packs.h index e304a954..2c0898a9 100644 --- a/include/osquery/packs.h +++ b/include/osquery/packs.h @@ -15,8 +15,8 @@ #include #include -#include #include +#include #include @@ -39,6 +39,7 @@ class Pack : private boost::noncopyable { public: Pack(const std::string& name, const boost::property_tree::ptree& tree) : Pack(name, "", tree) {} + Pack(const std::string& name, const std::string& source, const boost::property_tree::ptree& tree) { @@ -77,7 +78,9 @@ class Pack : private boost::noncopyable { /// Returns the minimum version that the pack is configured to run on const std::string& getVersion() const; - size_t getShard() const { return shard_; } + size_t getShard() const { + return shard_; + } /// Returns the schedule dictated by the pack const std::map& getSchedule() const; @@ -147,7 +150,7 @@ class Pack : private boost::noncopyable { * * Initialization must include pack content */ - Pack(){}; + Pack() {} private: FRIEND_TEST(PacksTests, test_check_platform); diff --git a/osquery/config/config.cpp b/osquery/config/config.cpp index c8f1ec34..72a2c6e9 100644 --- a/osquery/config/config.cpp +++ b/osquery/config/config.cpp @@ -251,14 +251,29 @@ Config::Config() void Config::addPack(const std::string& name, const std::string& source, const pt::ptree& tree) { - RecursiveLock wlock(config_schedule_mutex_); - try { - schedule_->add(std::make_shared(name, source, tree)); - if (schedule_->last()->shouldPackExecute()) { - applyParsers(source + FLAGS_pack_delimiter + name, tree, true); + auto addSinglePack = ([this, &source](const std::string pack_name, + const pt::ptree& pack_tree) { + RecursiveLock wlock(config_schedule_mutex_); + try { + schedule_->add(std::make_shared(pack_name, source, pack_tree)); + if (schedule_->last()->shouldPackExecute()) { + applyParsers( + source + FLAGS_pack_delimiter + pack_name, pack_tree, true); + } + } catch (const std::exception& e) { + LOG(WARNING) << "Error adding pack: " << pack_name << ": " << e.what(); } - } catch (const std::exception& e) { - LOG(WARNING) << "Error adding pack: " << name << ": " << e.what(); + }); + + if (name == "*") { + // This is a multi-pack, expect the config plugin to have generated a + // "name": {pack-content} response similar to embedded pack content + // within the configuration. + for (const auto& pack : tree) { + addSinglePack(pack.first, pack.second); + } + } else { + addSinglePack(name, tree); } } @@ -354,15 +369,7 @@ Status Config::load() { return status; } -/** - * @brief Boost's 1.59 property tree based JSON parser does not accept comments. - * - * For semi-compatibility with existing configurations we will attempt to strip - * hash and C++ style comments. It is OK for the config update to be latent - * as it is a single event. But some configuration plugins may update running - * configurations. - */ -inline void stripConfigComments(std::string& json) { +void stripConfigComments(std::string& json) { std::string sink; boost::replace_all(json, "\\\n", ""); diff --git a/osquery/config/plugins/filesystem.cpp b/osquery/config/plugins/filesystem.cpp index 28dd2ced..e1338e78 100644 --- a/osquery/config/plugins/filesystem.cpp +++ b/osquery/config/plugins/filesystem.cpp @@ -11,14 +11,18 @@ #include #include +#include #include #include #include #include -namespace fs = boost::filesystem; +#include "osquery/core/json.h" + namespace errc = boost::system::errc; +namespace fs = boost::filesystem; +namespace pt = boost::property_tree; namespace osquery { @@ -63,10 +67,51 @@ Status FilesystemConfigPlugin::genConfig( Status FilesystemConfigPlugin::genPack(const std::string& name, const std::string& value, std::string& pack) { + if (name == "*") { + // The config requested a multi-pack. + std::vector paths; + resolveFilePattern(value, paths); + + pt::ptree multi_pack; + for (const auto& path : paths) { + std::string content; + if (!readFile(path, content)) { + LOG(WARNING) << "Cannot read multi-pack file: " << path; + continue; + } + + // Assemble an intermediate property tree for simplified parsing. + pt::ptree single_pack; + stripConfigComments(content); + try { + std::stringstream json_stream; + json_stream << content; + pt::read_json(json_stream, single_pack); + } catch (const pt::json_parser::json_parser_error& /* e */) { + LOG(WARNING) << "Cannot read multi-pack JSON: " << path; + continue; + } + + multi_pack.put_child(fs::path(path).stem().string(), single_pack); + } + + // We should have a property tree of pack content mimicking embedded + // configuration packs, ready to parse as a string. + std::ostringstream output; + pt::write_json(output, multi_pack, false); + pack = output.str(); + if (pack.empty()) { + return Status(1, "Multi-pack content empty"); + } + + return Status(0); + } + boost::system::error_code ec; if (!fs::is_regular_file(value, ec) || ec.value() != errc::success) { return Status(1, value + " is not a valid path"); } + return readFile(value, pack); } } diff --git a/osquery/config/tests/packs_tests.cpp b/osquery/config/tests/packs_tests.cpp index fe597882..4deba8b7 100644 --- a/osquery/config/tests/packs_tests.cpp +++ b/osquery/config/tests/packs_tests.cpp @@ -150,6 +150,30 @@ TEST_F(PacksTests, test_discovery_cache) { c.reset(); } +TEST_F(PacksTests, test_multi_pack) { + std::string multi_pack_content = "{\"first\": {}, \"second\": {}}"; + pt::ptree multi_pack; + + { + // Convert the content into the expected pack form (ptree). + std::stringstream json_stream; + json_stream << multi_pack_content; + pt::read_json(json_stream, multi_pack); + } + + Config c; + c.addPack("*", "", multi_pack); + + std::vector pack_names; + c.packs(([&pack_names](std::shared_ptr& p) { + pack_names.push_back(p->getName()); + })); + + std::vector expected = {"first", "second"}; + ASSERT_EQ(expected.size(), pack_names.size()); + EXPECT_EQ(expected, pack_names); +} + TEST_F(PacksTests, test_discovery_zero_state) { Pack pack("discovery_pack", getPackWithDiscovery()); auto stats = pack.getStats();