valitydev/osquery-1
14
0
Fork 0
mirror of https://github.com/valitydev/osquery-1.git synced 2024-11-07 09:58:54 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add shared lock RAII helper around config access

Browse Source
This commit is contained in:
Teddy Reed 2015-03-21 20:51:42 -07:00
parent 5b227c8e3d
commit a97d557e5a
6 changed files with 138 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

104
include/osquery/config.h
View File

@ -14,8 +14,10 @@
#include <memory>
#include <vector>
#include <boost/noncopyable.hpp>
#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/json_parser.hpp>
#include <boost/thread/shared_mutex.hpp>
#include <osquery/database/results.h>
#include <osquery/flags.h>
@ -35,11 +37,11 @@ DECLARE_string(config_plugin);
* When you use osquery::Config::getInstance(), you are getting a singleton
* handle to interact with the data stored in an instance of this struct.
*/
struct OsqueryConfig {
struct ConfigData {
/// A vector of all of the queries that are scheduled to execute.
std::vector<OsqueryScheduledQuery> scheduledQueries;
std::vector<OsqueryScheduledQuery> schedule;
std::map<std::string, std::string> options;
std::map<std::string, std::vector<std::string> > eventFiles;
std::map<std::string, std::vector<std::string> > files;
pt::ptree all_data;
};
@ -62,7 +64,7 @@ extern const std::string kDefaultConfigRetriever;
* should be defined using the osquery::config::Config class and the pluggable
* plugin interface that is included with it.
*/
class Config {
class Config : private boost::noncopyable {
public:
/**
* @brief The primary way to access the Config singleton.
@ -96,40 +98,6 @@ class Config {
*/
static Status update(const std::map<std::string, std::string>& config);
/**
* @brief Get a vector of all scheduled queries.
*
* @code{.cpp}
* auto config = osquery::config::Config::getInstance();
* for (const auto& q : config->getScheduledQueries()) {
* LOG(INFO) << "name: " << q.name;
* LOG(INFO) << "interval: " << q.interval;
* }
* @endcode
*
* @return a vector of OsqueryScheduledQuery's which represent the queries
* that are to be executed
*/
static std::vector<OsqueryScheduledQuery> getScheduledQueries();
/**
* @brief Get a map of all the files in the intel JSON blob
*
*
*
* @return A map all the files in the JSON blob organized by category
*/
static std::map<std::string, std::vector<std::string> > getWatchedFiles();
/**
* @brief Return the configuration ptree
*
*
*
* @return Returns the unparsed, ptree representation of the given config
*/
static pt::ptree getEntireConfiguration();
/**
* @brief Calculate the has of the osquery config
*
@ -144,7 +112,8 @@ class Config {
* @return an instance of osquery::Status, indicating the success or failure
* of the operation.
*/
static osquery::Status checkConfig();
static Status checkConfig();
private:
/**
* @brief Default constructor.
@ -175,7 +144,7 @@ class Config {
* @return an instance of osquery::Status, indicating the success or failure
* of the operation.
*/
static osquery::Status genConfig(OsqueryConfig& conf);
static Status genConfig(ConfigData& conf);
/**
* @brief Uses the specified config retriever to populate a string with the
@ -191,19 +160,64 @@ class Config {
*
* @return status indicating the success or failure of the operation.
*/
static osquery::Status genConfig();
/// Prevent ConfigPlugins from implementing setUp.
osquery::Status setUp() { return Status(0, "Not used"); }
static Status genConfig();
private:
/**
* @brief the private member that stores the raw osquery config data in a
* native format
*/
OsqueryConfig cfg_;
ConfigData data_;
/// The raw JSON source map from the config plugin.
std::map<std::string, std::string> raw_;
/// The reader/writer config data mutex.
boost::shared_mutex mutex_;
private:
/// Config accessors, `ConfigDataInstance`, are the forced use of the config
/// data. This forces the caller to use a shared read lock.
friend class ConfigDataInstance;
private:
FRIEND_TEST(ConfigTests, test_locking);
};
/**
* @brief All accesses to the Config's data must request a ConfigDataInstance.
*
* This class will request a read-only lock of the config's changable internal
* data structures such as query schedule, options, monitored files, etc.
*
* Since a variable config plugin may implement `update` calls, internal uses
* of config data needs simple read and write locking.
*/
class ConfigDataInstance {
public:
ConfigDataInstance() : lock_(Config::getInstance().mutex_) {}
~ConfigDataInstance() { lock_.unlock(); }
/// Helper accessor for Config::data_.schedule.
const std::vector<OsqueryScheduledQuery> schedule() {
return Config::getInstance().data_.schedule;
}
/// Helper accessor for Config::data_.options.
const std::map<std::string, std::string>& options() {
return Config::getInstance().data_.options;
}
/// Helper accessor for Config::data_.files.
const std::map<std::string, std::vector<std::string> >& files() {
return Config::getInstance().data_.files;
}
/// Helper accessor for Config::data_.all_data.
const pt::ptree& data() { return Config::getInstance().data_.all_data; }
private:
/// A read lock on the reader/writer config data accessor/update mutex.
boost::shared_lock<boost::shared_mutex> lock_;
};
/**

52
osquery/config/config.cpp
View File

@ -11,8 +11,6 @@
#include <mutex>
#include <sstream>
#include <boost/thread/shared_mutex.hpp>
#include <osquery/config.h>
#include <osquery/flags.h>
#include <osquery/hash.h>
@ -28,11 +26,6 @@ namespace osquery {
CLI_FLAG(string, config_plugin, "filesystem", "Config plugin name");
// This lock is used to protect the entirety of the OSqueryConfig struct
// Is should be used when ever accessing the structs members, reading or
// writing.
static boost::shared_mutex rw_lock;
Status Config::load() {
auto& config_plugin = Registry::getActive("config");
if (!Registry::exists("config", config_plugin)) {
@ -43,26 +36,23 @@ Status Config::load() {
}
Status Config::update(const std::map<std::string, std::string>& config) {
boost::unique_lock<boost::shared_mutex> lock(rw_lock);
// Request a unique write lock when updating config.
boost::unique_lock<boost::shared_mutex> unique_lock(getInstance().mutex_);
for (const auto& source : config) {
getInstance().raw_[source.first] = source.second;
}
OsqueryConfig conf;
ConfigData conf;
auto status = genConfig(conf);
if (status.ok()) {
getInstance().cfg_ = conf;
getInstance().data_ = conf;
}
return status;
}
Status Config::genConfig() {
auto& config_plugin = Registry::getActive("config");
if (!Registry::exists("config", config_plugin)) {
return Status(1, "Missing config plugin " + config_plugin);
}
PluginResponse response;
auto status = Registry::call("config", {{"action", "genConfig"}}, response);
if (!status.ok()) {
@ -75,7 +65,7 @@ Status Config::genConfig() {
return Status(0, "OK");
}
inline void mergeOption(const tree_node& option, OsqueryConfig& conf) {
inline void mergeOption(const tree_node& option, ConfigData& conf) {
conf.options[option.first.data()] = option.second.data();
if (conf.all_data.count("options") > 0) {
conf.all_data.get_child("options").erase(option.first);
@ -83,7 +73,7 @@ inline void mergeOption(const tree_node& option, OsqueryConfig& conf) {
conf.all_data.add_child("options." + option.first, option.second);
}
inline void mergeAdditional(const tree_node& node, OsqueryConfig& conf) {
inline void mergeAdditional(const tree_node& node, ConfigData& conf) {
if (conf.all_data.count("additional_monitoring") > 0) {
conf.all_data.get_child("additional_monitoring").erase(node.first);
}
@ -97,24 +87,24 @@ inline void mergeAdditional(const tree_node& node, OsqueryConfig& conf) {
for (const auto& category : node.second) {
for (const auto& path : category.second) {
resolveFilePattern(path.second.data(),
conf.eventFiles[category.first],
conf.files[category.first],
REC_LIST_FOLDERS | REC_EVENT_OPT);
}
}
}
inline void mergeScheduledQuery(const tree_node& node, OsqueryConfig& conf) {
inline void mergeScheduledQuery(const tree_node& node, ConfigData& conf) {
// Read tree/JSON into a query structure.
OsqueryScheduledQuery query;
query.name = node.second.get<std::string>("name", "");
query.query = node.second.get<std::string>("query", "");
query.interval = node.second.get<int>("interval", 0);
// Also store the raw node in the property tree list.
conf.scheduledQueries.push_back(query);
conf.schedule.push_back(query);
conf.all_data.add_child("scheduledQueries", node.second);
}
Status Config::genConfig(OsqueryConfig& conf) {
Status Config::genConfig(ConfigData& conf) {
for (const auto& source : getInstance().raw_) {
std::stringstream json_data;
json_data << source.second;
@ -143,24 +133,12 @@ Status Config::genConfig(OsqueryConfig& conf) {
return Status(0, "OK");
}
std::vector<OsqueryScheduledQuery> Config::getScheduledQueries() {
boost::shared_lock<boost::shared_mutex> lock(rw_lock);
return getInstance().cfg_.scheduledQueries;
}
std::map<std::string, std::vector<std::string> > Config::getWatchedFiles() {
boost::shared_lock<boost::shared_mutex> lock(rw_lock);
return getInstance().cfg_.eventFiles;
}
pt::ptree Config::getEntireConfiguration() {
boost::shared_lock<boost::shared_mutex> lock(rw_lock);
return getInstance().cfg_.all_data;
}
Status Config::getMD5(std::string& hash_string) {
// Request an accessor to our own config, outside of an update.
ConfigDataInstance config;
std::stringstream out;
write_json(out, getEntireConfiguration());
write_json(out, config.data());
hash_string = osquery::hashFromBuffer(
HASH_TYPE_MD5, (void*)out.str().c_str(), out.str().length());

65
osquery/config/config_tests.cpp
View File

@ -32,7 +32,6 @@ class ConfigTests : public testing::Test {
}
protected:
void SetUp() {
createMockFileStructure();
Registry::setUp();
@ -67,16 +66,36 @@ TEST_F(ConfigTests, test_plugin) {
}
TEST_F(ConfigTests, test_queries_execute) {
auto queries = Config::getScheduledQueries();
EXPECT_EQ(queries.size(), 2);
ConfigDataInstance config;
EXPECT_EQ(config.schedule().size(), 2);
}
TEST_F(ConfigTests, test_watched_files) {
auto files = Config::getWatchedFiles();
ConfigDataInstance config;
EXPECT_EQ(config.files().size(), 2);
EXPECT_EQ(config.files().at("downloads").size(), 1);
EXPECT_EQ(config.files().at("system_binaries").size(), 2);
}
EXPECT_EQ(files.size(), 2);
EXPECT_EQ(files["downloads"].size(), 1);
EXPECT_EQ(files["system_binaries"].size(), 2);
TEST_F(ConfigTests, test_locking) {
{
// Assume multiple instance accessors will be active.
ConfigDataInstance config1;
ConfigDataInstance config2;
// But a unique lock cannot be aquired.
boost::unique_lock<boost::shared_mutex> lock(Config::getInstance().mutex_,
boost::defer_lock);
ASSERT_FALSE(lock.try_lock());
}
{
// However, a unique lock can be obtained when without instances accessors.
boost::unique_lock<boost::shared_mutex> lock(Config::getInstance().mutex_,
boost::defer_lock);
ASSERT_TRUE(lock.try_lock());
}
}
TEST_F(ConfigTests, test_config_update) {
@ -96,25 +115,33 @@ TEST_F(ConfigTests, test_config_update) {
EXPECT_NE(digest, new_digest);
// Access the option that was added in the update to source 'new_source1'.
auto config = Config::getEntireConfiguration();
auto option = config.get<std::string>("options.new1", "");
EXPECT_EQ(option, "value");
{
ConfigDataInstance config;
auto option = config.data().get<std::string>("options.new1", "");
EXPECT_EQ(option, "value");
}
// Add a lexically larger source that emits the same option 'new1'.
Config::update({{"2new_source", "{\"options\": {\"new1\": \"changed\"}}"}});
config = Config::getEntireConfiguration();
option = config.get<std::string>("options.new1", "");
// Expect the amalgamation to have overritten 'new_source1'.
EXPECT_EQ(option, "changed");
{
ConfigDataInstance config;
auto option = config.data().get<std::string>("options.new1", "");
// Expect the amalgamation to have overritten 'new_source1'.
EXPECT_EQ(option, "changed");
}
// Again add a source but emit a different option, both 'new1' and 'new2'
// should be in the amalgamated/merged config.
Config::update({{"3new_source", "{\"options\": {\"new2\": \"different\"}}"}});
config = Config::getEntireConfiguration();
option = config.get<std::string>("options.new1", "");
EXPECT_EQ(option, "changed");
option = config.get<std::string>("options.new2", "");
EXPECT_EQ(option, "different");
{
ConfigDataInstance config;
auto option = config.data().get<std::string>("options.new1", "");
EXPECT_EQ(option, "changed");
option = config.data().get<std::string>("options.new2", "");
EXPECT_EQ(option, "different");
}
}
}

24
osquery/scheduler/scheduler.cpp
View File

@ -139,21 +139,25 @@ int splayValue(int original, int splayPercent) {
}
void SchedulerRunner::enter() {
// Iterate over scheduled queryies and add a splay to each.
auto schedule = Config::getScheduledQueries();
for (auto& query : schedule) {
auto old_interval = query.interval;
auto new_interval = splayValue(old_interval, FLAGS_schedule_splay_percent);
VLOG(1) << "Splay changing the interval for " << query.name << " from "
<< old_interval << " to " << new_interval;
query.interval = new_interval;
}
/**
// Iterate over scheduled queryies and add a splay to each.
for (auto& query : schedule) {
auto old_interval = query.interval;
auto new_interval = splayValue(old_interval,
FLAGS_schedule_splay_percent);
VLOG(1) << "Splay changing the interval for " << query.name << " from "
<< old_interval << " to " << new_interval;
query.interval = new_interval;
}
**/
time_t t = time(0);
struct tm* local = localtime(&t);
unsigned long int i = local->tm_sec;
for (; (timeout_ == 0) || (i <= timeout_); ++i) {
for (const auto& query : schedule) {
ConfigDataInstance config;
for (const auto& query : config.schedule()) {
if (i % query.interval == 0) {
launchQuery(query);
}

4
osquery/tables/events/darwin/file_changes.cpp
View File

@ -55,8 +55,8 @@ class FileChangesEventSubscriber
REGISTER(FileChangesEventSubscriber, "event_subscriber", "file_changes");
void FileChangesEventSubscriber::init() {
const auto& file_map = Config::getWatchedFiles();
for (const auto& element_kv : file_map) {
ConfigDataInstance config;
for (const auto& element_kv : config.files()) {
for (const auto& file : element_kv.second) {
VLOG(1) << "Added listener to: " << file;
auto mc = createSubscriptionContext();

5
osquery/tables/events/linux/file_changes.cpp
View File

@ -55,9 +55,8 @@ class FileChangesEventSubscriber
REGISTER(FileChangesEventSubscriber, "event_subscriber", "file_changes");
void FileChangesEventSubscriber::init() {
const auto& file_map = Config::getWatchedFiles();
for (const auto& element_kv : file_map) {
ConfigDataInstance config;
for (const auto& element_kv : config.files()) {
for (const auto& file : element_kv.second) {
VLOG(1) << "Added listener to: " << file;
auto mc = createSubscriptionContext();