valitydev/osquery-1
14
0
Fork 0
mirror of https://github.com/valitydev/osquery-1.git synced 2024-11-07 01:55:20 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #1897 from theopolis/remove_rdb

Browse Source 
Refactor backing storage
This commit is contained in:
Teddy Reed 2016-03-06 21:09:23 -08:00
commit 677c448dea
49 changed files with 1871 additions and 1561 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
CMakeLists.txt
View File

@ -139,12 +139,22 @@ set(OSQUERY_REQUIRE_RUNTIMES
if(DEFINED ENV{DEBUG})
set(DEBUG TRUE)
set(CMAKE_BUILD_TYPE "Debug")
add_compile_options(-g -O0)
add_compile_options(
-g
-O0
-fstandalone-debug
)
add_definitions(-DDEBUG)
WARNING_LOG("Setting DEBUG build")
elseif(DEFINED ENV{SANITIZE})
# make sanitize (cannot make debug sanitize)
add_compile_options(-g -O1 -fno-omit-frame-pointer -fno-optimize-sibling-calls)
add_compile_options(
-g
-O1
-fstandalone-debug
-fno-omit-frame-pointer
-fno-optimize-sibling-calls
)
add_definitions(-DNDEBUG)
if(DEFINED ENV{SANITIZE_THREAD})
add_compile_options(-fsanitize=thread)
@ -347,12 +357,12 @@ endif()
find_package(Glog REQUIRED)
find_package(Gflags REQUIRED)
find_package(Gtest REQUIRED)
find_package(RocksDB REQUIRED)
find_package(Thrift 0.9.3 REQUIRED)
# If using the RocksDB LITE version our code must also define ROCKSDB_LITE=1
if(ROCKSDB_LITE_FOUND)
add_definitions(-DROCKSDB_LITE=1)
if(NOT DEFINED ENV{SKIP_ROCKSDB})
set(ROCKSDB TRUE)
find_package(RocksDB REQUIRED)
endif()
# Python is used for table spec generation and formating.

228
include/osquery/database.h
View File

@ -10,6 +10,7 @@
#pragma once
#include <atomic>
#include <map>
#include <string>
#include <vector>
@ -21,6 +22,18 @@
namespace osquery {
/**
* @brief A list of supported backing storage categories: called domains.
*
* RocksDB has a concept of "column families" which are kind of like tables
* in other databases. kDomainds is populated with a list of all column
* families. If a string exists in kDomains, it's a column family in the
* database.
*
* For SQLite-backed storage these are tables using a keyed index.
*/
extern const std::vector<std::string> kDomains;
/**
* @brief A backing storage domain name, used for key/value based storage.
*
@ -46,9 +59,6 @@ extern const std::string kEvents;
*/
extern const std::string kLogs;
/// An ordered list of column type names.
extern const std::vector<std::string> kDomains;
/////////////////////////////////////////////////////////////////////////////
// Row
/////////////////////////////////////////////////////////////////////////////
@ -394,142 +404,6 @@ Status serializeQueryLogItemAsEvents(const QueryLogItem& item,
Status serializeQueryLogItemAsEventsJSON(const QueryLogItem& i,
std::vector<std::string>& items);
/////////////////////////////////////////////////////////////////////////////
// DistributedQueryRequest
/////////////////////////////////////////////////////////////////////////////
/**
* @brief Small struct containing the query and ID information for a
* distributed query
*/
struct DistributedQueryRequest {
public:
explicit DistributedQueryRequest() {}
explicit DistributedQueryRequest(const std::string& q, const std::string& i)
: query(q), id(i) {}
/// equals operator
bool operator==(const DistributedQueryRequest& comp) const {
return (comp.query == query) && (comp.id == id);
}
std::string query;
std::string id;
};
/**
* @brief Serialize a DistributedQueryRequest into a property tree
*
* @param r the DistributedQueryRequest to serialize
* @param tree the output property tree
*
* @return Status indicating the success or failure of the operation
*/
Status serializeDistributedQueryRequest(const DistributedQueryRequest& r,
boost::property_tree::ptree& tree);
/**
* @brief Serialize a DistributedQueryRequest object into a JSON string
*
* @param r the DistributedQueryRequest to serialize
* @param json the output JSON string
*
* @return Status indicating the success or failure of the operation
*/
Status serializeDistributedQueryRequestJSON(const DistributedQueryRequest& r,
std::string& json);
/**
* @brief Deserialize a DistributedQueryRequest object from a property tree
*
* @param tree the input property tree
* @param r the output DistributedQueryRequest structure
*
* @return Status indicating the success or failure of the operation
*/
Status deserializeDistributedQueryRequest(
const boost::property_tree::ptree& tree, DistributedQueryRequest& r);
/**
* @brief Deserialize a DistributedQueryRequest object from a JSON string
*
* @param json the input JSON string
* @param r the output DistributedQueryRequest structure
*
* @return Status indicating the success or failure of the operation
*/
Status deserializeDistributedQueryRequestJSON(const std::string& json,
DistributedQueryRequest& r);
/////////////////////////////////////////////////////////////////////////////
// DistributedQueryResult
/////////////////////////////////////////////////////////////////////////////
/**
* @brief Small struct containing the results of a distributed query
*/
struct DistributedQueryResult {
public:
explicit DistributedQueryResult() {}
explicit DistributedQueryResult(const DistributedQueryRequest& req,
const QueryData& res)
: request(req), results(res) {}
/// equals operator
bool operator==(const DistributedQueryResult& comp) const {
return (comp.request == request) && (comp.results == results);
}
DistributedQueryRequest request;
QueryData results;
};
/**
* @brief Serialize a DistributedQueryResult into a property tree
*
* @param r the DistributedQueryResult to serialize
* @param tree the output property tree
*
* @return Status indicating the success or failure of the operation
*/
Status serializeDistributedQueryResult(const DistributedQueryResult& r,
boost::property_tree::ptree& tree);
/**
* @brief Serialize a DistributedQueryResult object into a JSON string
*
* @param r the DistributedQueryResult to serialize
* @param json the output JSON string
*
* @return Status indicating the success or failure of the operation
*/
Status serializeDistributedQueryResultJSON(const DistributedQueryResult& r,
std::string& json);
/**
* @brief Deserialize a DistributedQueryResult object from a property tree
*
* @param tree the input property tree
* @param r the output DistributedQueryResult structure
*
* @return Status indicating the success or failure of the operation
*/
Status deserializeDistributedQueryResult(
const boost::property_tree::ptree& tree, DistributedQueryResult& r);
/**
* @brief Deserialize a DistributedQueryResult object from a JSON string
*
* @param json the input JSON string
* @param r the output DistributedQueryResult structure
*
* @return Status indicating the success or failure of the operation
*/
Status deserializeDistributedQueryResultJSON(const std::string& json,
DistributedQueryResult& r);
/////////////////////////////////////////////////////////////////////////////
/**
* @brief An osquery backing storage (database) type that persists executions.
*
@ -543,7 +417,7 @@ Status deserializeDistributedQueryResultJSON(const std::string& json,
* to removing RocksDB as a dependency for the osquery SDK.
*/
class DatabasePlugin : public Plugin {
protected:
public:
/**
* @brief Perform a domain and key lookup from the backing store.
*
@ -580,15 +454,79 @@ class DatabasePlugin : public Plugin {
/// Data removal method.
virtual Status remove(const std::string& domain, const std::string& k) = 0;
/// Key/index lookup method.
virtual Status scan(const std::string& domain,
std::vector<std::string>& results,
const std::string& prefix,
size_t max = 0) const {
return Status(0, "Not used");
}
/**
* @brief Shutdown the database and release initialization resources.
*
* Assume that a plugin may override ::tearDown and choose to close resources
* when the registry is stopping. Most plugins will implement a mutex around
* initialization and destruction and assume ::setUp and ::tearDown will
* dictate the flow in most situations.
*/
virtual ~DatabasePlugin() {}
/**
* @brief Support the registry calling API for extensions.
*
* The database plugin "fast-calls" directly to local plugins.
* Extensions cannot use an extension-local backing store so their requests
* are routed like all other plugins.
*/
Status call(const PluginRequest& request, PluginResponse& response) override;
public:
Status call(const PluginRequest& request, PluginResponse& response);
/// Database-specific workflow: reset the originally request instance.
virtual Status reset() final;
/// Database-specific workflow: perform an initialize, then reset.
bool checkDB();
/// Require all DBHandle accesses to open a read and write handle.
static void setRequireWrite(bool rw) { kDBHandleOptionRequireWrite = rw; }
/// Allow DBHandle creations.
static void setAllowOpen(bool ao) { kDBHandleOptionAllowOpen = ao; }
public:
/// Control availability of the RocksDB handle (default false).
static bool kDBHandleOptionAllowOpen;
/// The database must be opened in a R/W mode (default false).
static bool kDBHandleOptionRequireWrite;
/// A queryable mutex around database sanity checking.
static std::atomic<bool> kCheckingDB;
public:
/**
* @brief Allow the initializer to check the active database plugin.
*
* Unlink the initializer's ::initActivePlugin helper method, the database
* plugin should always be within the core. There is no need to discover
* the active plugin via the registry or extensions API.
*
* The database should setUp in preparation for accesses.
*/
static bool initPlugin();
/// Allow shutdown before exit.
static void shutdown();
protected:
/// The database was opened in a ReadOnly mode.
bool read_only_{false};
/// True if the database was started in an in-memory mode.
bool in_memory_{false};
/// Original requested path on disk.
std::string path_;
};
/**
@ -631,6 +569,12 @@ Status scanDatabaseKeys(const std::string& domain,
std::vector<std::string>& keys,
size_t max = 0);
/// Get a list of keys for a given domain.
Status scanDatabaseKeys(const std::string& domain,
std::vector<std::string>& keys,
const std::string& prefix,
size_t max = 0);
/// Allow callers to scan each column family and print each value.
void dumpDatabase();

130
include/osquery/distributed.h
View File

@ -19,6 +19,136 @@
namespace osquery {
/**
* @brief Small struct containing the query and ID information for a
* distributed query
*/
struct DistributedQueryRequest {
public:
explicit DistributedQueryRequest() {}
explicit DistributedQueryRequest(const std::string& q, const std::string& i)
: query(q), id(i) {}
/// equals operator
bool operator==(const DistributedQueryRequest& comp) const {
return (comp.query == query) && (comp.id == id);
}
std::string query;
std::string id;
};
/**
* @brief Serialize a DistributedQueryRequest into a property tree
*
* @param r the DistributedQueryRequest to serialize
* @param tree the output property tree
*
* @return Status indicating the success or failure of the operation
*/
Status serializeDistributedQueryRequest(const DistributedQueryRequest& r,
boost::property_tree::ptree& tree);
/**
* @brief Serialize a DistributedQueryRequest object into a JSON string
*
* @param r the DistributedQueryRequest to serialize
* @param json the output JSON string
*
* @return Status indicating the success or failure of the operation
*/
Status serializeDistributedQueryRequestJSON(const DistributedQueryRequest& r,
std::string& json);
/**
* @brief Deserialize a DistributedQueryRequest object from a property tree
*
* @param tree the input property tree
* @param r the output DistributedQueryRequest structure
*
* @return Status indicating the success or failure of the operation
*/
Status deserializeDistributedQueryRequest(
const boost::property_tree::ptree& tree, DistributedQueryRequest& r);
/**
* @brief Deserialize a DistributedQueryRequest object from a JSON string
*
* @param json the input JSON string
* @param r the output DistributedQueryRequest structure
*
* @return Status indicating the success or failure of the operation
*/
Status deserializeDistributedQueryRequestJSON(const std::string& json,
DistributedQueryRequest& r);
/////////////////////////////////////////////////////////////////////////////
// DistributedQueryResult
/////////////////////////////////////////////////////////////////////////////
/**
* @brief Small struct containing the results of a distributed query
*/
struct DistributedQueryResult {
public:
explicit DistributedQueryResult() {}
explicit DistributedQueryResult(const DistributedQueryRequest& req,
const QueryData& res)
: request(req), results(res) {}
/// equals operator
bool operator==(const DistributedQueryResult& comp) const {
return (comp.request == request) && (comp.results == results);
}
DistributedQueryRequest request;
QueryData results;
};
/**
* @brief Serialize a DistributedQueryResult into a property tree
*
* @param r the DistributedQueryResult to serialize
* @param tree the output property tree
*
* @return Status indicating the success or failure of the operation
*/
Status serializeDistributedQueryResult(const DistributedQueryResult& r,
boost::property_tree::ptree& tree);
/**
* @brief Serialize a DistributedQueryResult object into a JSON string
*
* @param r the DistributedQueryResult to serialize
* @param json the output JSON string
*
* @return Status indicating the success or failure of the operation
*/
Status serializeDistributedQueryResultJSON(const DistributedQueryResult& r,
std::string& json);
/**
* @brief Deserialize a DistributedQueryResult object from a property tree
*
* @param tree the input property tree
* @param r the output DistributedQueryResult structure
*
* @return Status indicating the success or failure of the operation
*/
Status deserializeDistributedQueryResult(
const boost::property_tree::ptree& tree, DistributedQueryResult& r);
/**
* @brief Deserialize a DistributedQueryResult object from a JSON string
*
* @param json the input JSON string
* @param r the output DistributedQueryResult structure
*
* @return Status indicating the success or failure of the operation
*/
Status deserializeDistributedQueryResultJSON(const std::string& json,
DistributedQueryResult& r);
class DistributedPlugin : public Plugin {
public:
/**

7
include/osquery/extensions.h
View File

@ -16,7 +16,6 @@
namespace osquery {
DECLARE_int32(worker_threads);
DECLARE_string(extensions_socket);
DECLARE_string(extensions_autoload);
DECLARE_string(extensions_timeout);
@ -41,11 +40,7 @@ typedef std::map<RouteUUID, ExtensionInfo> ExtensionList;
inline std::string getExtensionSocket(
RouteUUID uuid, const std::string& path = FLAGS_extensions_socket) {
if (uuid == 0) {
return path;
} else {
return path + "." + std::to_string(uuid);
}
return (uuid == 0) ? path : path + "." + std::to_string(uuid);
}
/// External (extensions) SQL implementation of the osquery query API.

2
include/osquery/registry.h
View File

@ -12,8 +12,8 @@
#include <map>
#include <mutex>
#include <vector>
#include <set>
#include <vector>
#include <boost/noncopyable.hpp>
#include <boost/property_tree/ptree.hpp>

18
osquery/CMakeLists.txt
View File

@ -14,10 +14,16 @@ set(OSQUERY_ADDITIONAL_LINKS "")
set(OSQUERY_ADDITIONAL_TESTS "")
set(OSQUERY_TABLES_TESTS "")
# Add all and extra for osquery code.
add_compile_options(
-Wall
-Wextra
-Wno-unused-parameter
)
# The core set of osquery libraries most discovered with find_package.
set(OSQUERY_LIBS
# This includes librocksdb[_lite] and libsnappy.
${ROCKSDB_LIBRARIES}
${THRIFT_LIBRARY}
${GLOG_LIBRARY}
${GFLAGS_LIBRARY}
@ -30,12 +36,10 @@ set(OSQUERY_LIBS
z
)
# Add all and extra for osquery code.
add_compile_options(
-Wall
-Wextra
-Wno-unused-parameter
)
# If building with RocksDB (default) append associated libraries.
if(ROCKSDB)
set(OSQUERY_LIBS ${OSQUERY_LIBS} ${ROCKSDB_LIBRARIES})
endif()
if(NOT FREEBSD)
set(OSQUERY_LIBS ${OSQUERY_LIBS} dl)

23
osquery/config/config.cpp
View File

@ -18,9 +18,9 @@
#include <osquery/config.h>
#include <osquery/database.h>
#include <osquery/filesystem.h>
#include <osquery/flags.h>
#include <osquery/hash.h>
#include <osquery/filesystem.h>
#include <osquery/logger.h>
#include <osquery/packs.h>
#include <osquery/registry.h>
@ -249,8 +249,9 @@ void Config::removeFiles(const std::string& source) {
}
}
void Config::scheduledQueries(std::function<
void(const std::string& name, const ScheduledQuery& query)> predicate) {
void Config::scheduledQueries(
std::function<void(const std::string& name, const ScheduledQuery& query)>
predicate) {
ReadLock rlock(config_schedule_mutex_);
for (const PackRef& pack : *schedule_) {
for (const auto& it : pack->getSchedule()) {
@ -675,18 +676,12 @@ Status Config::getMD5(std::string& hash) {
const std::shared_ptr<ConfigParserPlugin> Config::getParser(
const std::string& parser) {
std::shared_ptr<ConfigParserPlugin> config_parser = nullptr;
try {
auto plugin = Registry::get("config_parser", parser);
config_parser = std::dynamic_pointer_cast<ConfigParserPlugin>(plugin);
} catch (const std::out_of_range& e) {
LOG(ERROR) << "Error getting config parser plugin " << parser << ": "
<< e.what();
} catch (const std::bad_cast& e) {
LOG(ERROR) << "Error casting " << parser
<< " as a ConfigParserPlugin: " << e.what();
if (!Registry::exists("config_parser", parser, true)) {
return nullptr;
}
return config_parser;
auto plugin = Registry::get("config_parser", parser);
return std::dynamic_pointer_cast<ConfigParserPlugin>(plugin);
}
void Config::files(

63
osquery/core/init.cpp
View File

@ -30,7 +30,6 @@
#include <osquery/registry.h>
#include "osquery/core/watcher.h"
#include "osquery/database/db_handle.h"
#include "osquery/dispatcher/dispatcher.h"
#if defined(__linux__) || defined(__FreeBSD__)
@ -105,6 +104,8 @@ static inline bool hasWorkerVariable() {
volatile std::sig_atomic_t kHandledSignal{0};
static inline bool isWatcher() { return (osquery::Watcher::getWorker() > 0); }
void signalHandler(int signal) {
// Inform exit status of main threads blocked by service joins.
if (kHandledSignal == 0) {
@ -112,10 +113,8 @@ void signalHandler(int signal) {
}
// Handle signals based on a tri-state (worker, watcher, neither).
pid_t worker_pid = osquery::Watcher::getWorker();
bool is_watcher = worker_pid > 0;
if (signal == SIGHUP) {
if (!is_watcher || hasWorkerVariable()) {
if (!isWatcher() || hasWorkerVariable()) {
// Reload configuration.
}
} else if (signal == SIGTERM || signal == SIGINT || signal == SIGABRT) {
@ -127,7 +126,7 @@ void signalHandler(int signal) {
std::signal(signal, SIG_DFL);
// The watcher waits for the worker to die.
if (is_watcher) {
if (isWatcher()) {
// Bind the fate of the worker to this watcher.
osquery::Watcher::bindFates();
} else {
@ -145,7 +144,7 @@ void signalHandler(int signal) {
raise((kHandledSignal != 0) ? kHandledSignal : SIGALRM);
}
if (is_watcher) {
if (isWatcher()) {
// The signal should be proliferated through the process group.
// Otherwise the watcher could 'forward' the signal to workers and
// managed extension processes.
@ -166,7 +165,9 @@ DECLARE_string(distributed_plugin);
DECLARE_bool(disable_distributed);
DECLARE_string(config_plugin);
DECLARE_bool(config_check);
DECLARE_bool(config_dump);
DECLARE_bool(database_dump);
DECLARE_string(database_path);
ToolType kToolType = OSQUERY_TOOL_UNKNOWN;
@ -421,14 +422,27 @@ void Initializer::initActivePlugin(const std::string& type,
timeout = kExtensionInitializeLatencyUS * 10;
}
while (!Registry::setActive(type, name)) {
if (!Watcher::hasManagedExtensions() || delay > timeout) {
LOG(ERROR) << "Active " << type << " plugin not found: " << name;
osquery::shutdown(EXIT_CATASTROPHIC);
// Attempt to set the request plugin as active.
Status status;
do {
status = Registry::setActive(type, name);
if (status.ok()) {
// The plugin was found, and is not active.
return;
}
if (!Watcher::hasManagedExtensions()) {
// The plugin was found locally, and is not active, problem.
break;
}
// The plugin is not local and is not active, wait and retry.
delay += kExtensionInitializeLatencyUS;
::usleep(kExtensionInitializeLatencyUS);
}
} while (delay < timeout);
LOG(ERROR) << "Cannot activate " << name << " " << type
<< " plugin: " << status.getMessage();
osquery::shutdown(EXIT_CATASTROPHIC);
}
void Initializer::start() const {
@ -436,18 +450,26 @@ void Initializer::start() const {
osquery::loadModules();
// Pre-extension manager initialization options checking.
if (FLAGS_config_check && !Watcher::hasManagedExtensions()) {
// If the shell or daemon does not need extensions and it will exit quickly,
// prefer to disable the extension manager.
if ((FLAGS_config_check || FLAGS_config_dump) &&
!Watcher::hasManagedExtensions()) {
FLAGS_disable_extensions = true;
}
// A daemon must always have R/W access to the database.
DBHandle::setAllowOpen(true);
DBHandle::setRequireWrite(tool_ == OSQUERY_TOOL_DAEMON);
if (!DBHandle::checkDB()) {
LOG(ERROR) << RLOG(1629) << binary_
<< " initialize failed: Could not open RocksDB";
auto retcode = (isWorker()) ? EXIT_CATASTROPHIC : EXIT_FAILURE;
::exit(retcode);
// A watcher should not need access to the backing store.
// If there are spurious access then warning logs will be emitted since the
// set-allow-open will never be called.
if (!isWatcher()) {
DatabasePlugin::setAllowOpen(true);
// A daemon must always have R/W access to the database.
DatabasePlugin::setRequireWrite(tool_ == OSQUERY_TOOL_DAEMON);
if (!DatabasePlugin::initPlugin()) {
LOG(ERROR) << RLOG(1629) << binary_
<< " initialize failed: Could not initialize database";
auto retcode = (isWorker()) ? EXIT_CATASTROPHIC : EXIT_FAILURE;
::exit(retcode);
}
}
// Bind to an extensions socket and wait for registry additions.
@ -516,6 +538,7 @@ void shutdown(int retcode, bool wait) {
// Hopefully release memory used by global string constructors in gflags.
GFLAGS_NAMESPACE::ShutDownCommandLineFlags();
DatabasePlugin::shutdown();
::exit(retcode);
}
}

14
osquery/core/test_util.cpp
View File

@ -16,18 +16,20 @@
#include <signal.h>
#include <time.h>
#include <boost/property_tree/json_parser.hpp>
#include <boost/filesystem/operations.hpp>
#include <boost/property_tree/json_parser.hpp>
#include <osquery/core.h>
#include <osquery/database.h>
#include <osquery/filesystem.h>
#include <osquery/logger.h>
#include "osquery/core/test_util.h"
#include "osquery/database/db_handle.h"
namespace fs = boost::filesystem;
namespace osquery {
std::string kFakeDirectory = "";
#ifdef DARWIN
@ -80,10 +82,14 @@ void initTesting() {
FLAGS_modules_autoload = kTestWorkingDirectory + "unittests-mod.load";
FLAGS_disable_logging = true;
// Create a default DBHandle instance before unittests.
(void)DBHandle::getInstance();
// Tests need a database plugin.
// Set up the database instance for the unittests.
DatabasePlugin::setAllowOpen(true);
Registry::setActive("database", "ephemeral");
}
void shutdownTesting() { DatabasePlugin::shutdown(); }
std::map<std::string, std::string> getTestConfigMap() {
std::string content;
readFile(kTestDataPath + "test_parse_items.conf", content);

2
osquery/core/test_util.h
View File

@ -29,7 +29,7 @@ namespace osquery {
void initTesting();
/// Cleanup/stop function for tests and benchmarks.
void cleanupTesting();
void shutdownTesting();
/// Any SQL-dependent tests should use kTestQuery for a pre-populated example.
const std::string kTestQuery = "SELECT * FROM test_table";

4
osquery/core/watcher.cpp
View File

@ -11,8 +11,8 @@
#include <cstring>
#include <math.h>
#include <sys/wait.h>
#include <signal.h>
#include <sys/wait.h>
#include <boost/filesystem.hpp>
@ -431,7 +431,7 @@ void WatcherWatcherRunner::start() {
VLOG(1) << "osqueryd worker (" << getpid()
<< ") detected killed watcher (" << watcher_ << ")";
// The watcher watcher is a thread. Do not join services after removing.
::exit(EXIT_FAILURE);
raise(SIGKILL);
}
interruptableSleep(getWorkerLimit(INTERVAL) * 1000);
}

36
osquery/database/CMakeLists.txt
View File

@ -1,15 +1,41 @@
ADD_OSQUERY_LIBRARY(TRUE osquery_database
database.cpp
)
# osquery_database_internal should be an 'additional' CORE=False lib
ADD_OSQUERY_LIBRARY(TRUE osquery_database_internal
db_handle.cpp
query.cpp
# Add 'core' plugins that do not required additional libraries.
plugins/ephemeral.cpp
)
# Begin with just the SQLite database plugin.
set(OSQUERY_DATABASE_PLUGINS
plugins/sqlite.cpp
)
set(OSQUERY_DATABASE_PLUGIN_TESTS
database/tests/plugin_tests.cpp
database/plugins/tests/sqlite_tests.cpp
)
# Optionally (but by default), add the RocksDB database plugin.
if(ROCKSDB)
if(ROCKSDB_LITE_FOUND)
add_definitions(-DROCKSDB_LITE=1)
endif()
set(OSQUERY_DATABASE_PLUGINS ${OSQUERY_DATABASE_PLUGINS} plugins/rocksdb.cpp)
set(OSQUERY_DATABASE_PLUGIN_TESTS ${OSQUERY_DATABASE_PLUGIN_TESTS} database/plugins/tests/rocksdb_tests.cpp)
else()
add_definitions(-DSKIP_ROCKSDB)
endif()
# Plugins (which do not include the shim/ephemeral) are 'additional'.
ADD_OSQUERY_LIBRARY(FALSE osquery_database_plugins ${OSQUERY_DATABASE_PLUGINS})
# Non-plugin tests are core.
file(GLOB OSQUERY_DATABASE_TESTS "tests/*.cpp")
ADD_OSQUERY_TEST(TRUE ${OSQUERY_DATABASE_TESTS})
# Plugin tests are additional.
ADD_OSQUERY_TEST(FALSE ${OSQUERY_DATABASE_PLUGIN_TESTS})
file(GLOB OSQUERY_DATABASE_BENCHMARKS "benchmarks/*.cpp")
ADD_OSQUERY_BENCHMARK(${OSQUERY_DATABASE_BENCHMARKS})

27
osquery/database/benchmarks/database_benchmarks.cpp
View File

@ -10,11 +10,10 @@
#include <benchmark/benchmark.h>
#include <osquery/filesystem.h>
#include <osquery/database.h>
#include <osquery/filesystem.h>
#include "osquery/core/test_util.h"
#include "osquery/database/db_handle.h"
#include "osquery/database/query.h"
namespace osquery {
@ -76,8 +75,22 @@ static void DATABASE_query_results(benchmark::State& state) {
}
}
BENCHMARK(DATABASE_query_results)->ArgPair(1, 1)->ArgPair(10, 10)->ArgPair(10,
100);
BENCHMARK(DATABASE_query_results)
->ArgPair(1, 1)
->ArgPair(10, 10)
->ArgPair(10, 100);
static void DATABASE_get(benchmark::State& state) {
setDatabaseValue(kPersistentSettings, "benchmark", "1");
while (state.KeepRunning()) {
std::string value;
getDatabaseValue(kPersistentSettings, "benchmark", value);
}
// All benchmarks will share a single database handle.
deleteDatabaseValue(kPersistentSettings, "benchmark");
}
BENCHMARK(DATABASE_get);
static void DATABASE_store(benchmark::State& state) {
while (state.KeepRunning()) {
@ -112,12 +125,14 @@ static void DATABASE_store_append(benchmark::State& state) {
size_t k = 0;
while (state.KeepRunning()) {
setDatabaseValue(kPersistentSettings, "key" + std::to_string(k++), content);
setDatabaseValue(kPersistentSettings, "key" + std::to_string(k), content);
deleteDatabaseValue(kPersistentSettings, "key" + std::to_string(k));
k++;
}
// All benchmarks will share a single database handle.
for (size_t i = 0; i < k; ++i) {
deleteDatabaseValue(kPersistentSettings, "key" + std::to_string(i));
// deleteDatabaseValue(kPersistentSettings, "key" + std::to_string(i));
}
}

294
osquery/database/database.cpp
View File

@ -8,12 +8,7 @@
*
*/
#include <algorithm>
#include <iostream>
#include <sstream>
#include <set>
#include <string>
#include <vector>
#include <boost/lexical_cast.hpp>
#include <boost/property_tree/json_parser.hpp>
@ -27,6 +22,37 @@ namespace osquery {
CLI_FLAG(bool, database_dump, false, "Dump the contents of the backing store");
CLI_FLAG(string,
database_path,
"/var/osquery/osquery.db",
"If using a disk-based backing store, specify a path");
FLAG_ALIAS(std::string, db_path, database_path);
CLI_FLAG(bool,
database_in_memory,
false,
"Keep osquery backing-store in memory");
FLAG_ALIAS(bool, use_in_memory_database, database_in_memory);
#if defined(SKIP_ROCKSDB)
#define DATABASE_PLUGIN "sqlite"
#else
#define DATABASE_PLUGIN "rocksdb"
#endif
const std::string kInternalDatabase = DATABASE_PLUGIN;
const std::string kPersistentSettings = "configurations";
const std::string kQueries = "queries";
const std::string kEvents = "events";
const std::string kLogs = "logs";
const std::vector<std::string> kDomains = {kPersistentSettings, kQueries,
kEvents, kLogs};
bool DatabasePlugin::kDBHandleOptionAllowOpen(false);
bool DatabasePlugin::kDBHandleOptionRequireWrite(false);
std::atomic<bool> DatabasePlugin::kCheckingDB(false);
/////////////////////////////////////////////////////////////////////////////
// Row - the representation of a row in a set of database results. Row is a
// simple map where individual column names are keys, which map to the Row's
@ -360,133 +386,6 @@ Status serializeQueryLogItemAsEventsJSON(const QueryLogItem& i,
return Status(0, "OK");
}
/////////////////////////////////////////////////////////////////////////////
// DistributedQueryRequest - small struct containing the query and ID
// information for a distributed query
/////////////////////////////////////////////////////////////////////////////
Status serializeDistributedQueryRequest(const DistributedQueryRequest& r,
pt::ptree& tree) {
tree.put("query", r.query);
tree.put("id", r.id);
return Status(0, "OK");
}
Status serializeDistributedQueryRequestJSON(const DistributedQueryRequest& r,
std::string& json) {
pt::ptree tree;
auto s = serializeDistributedQueryRequest(r, tree);
if (!s.ok()) {
return s;
}
std::stringstream ss;
try {
pt::write_json(ss, tree, false);
} catch (const pt::ptree_error& e) {
return Status(1, "Error serializing JSON: " + std::string(e.what()));
}
json = ss.str();
return Status(0, "OK");
}
Status deserializeDistributedQueryRequest(const pt::ptree& tree,
DistributedQueryRequest& r) {
r.query = tree.get<std::string>("query", "");
r.id = tree.get<std::string>("id", "");
return Status(0, "OK");
}
Status deserializeDistributedQueryRequestJSON(const std::string& json,
DistributedQueryRequest& r) {
std::stringstream ss(json);
pt::ptree tree;
try {
pt::read_json(ss, tree);
} catch (const pt::ptree_error& e) {
return Status(1, "Error serializing JSON: " + std::string(e.what()));
}
return deserializeDistributedQueryRequest(tree, r);
}
/////////////////////////////////////////////////////////////////////////////
// DistributedQueryResult - small struct containing the results of a
// distributed query
/////////////////////////////////////////////////////////////////////////////
Status serializeDistributedQueryResult(const DistributedQueryResult& r,
pt::ptree& tree) {
pt::ptree request;
auto s = serializeDistributedQueryRequest(r.request, request);
if (!s.ok()) {
return s;
}
pt::ptree results;
s = serializeQueryData(r.results, results);
if (!s.ok()) {
return s;
}
tree.add_child("request", request);
tree.add_child("results", results);
return Status(0, "OK");
}
Status serializeDistributedQueryResultJSON(const DistributedQueryResult& r,
std::string& json) {
pt::ptree tree;
auto s = serializeDistributedQueryResult(r, tree);
if (!s.ok()) {
return s;
}
std::stringstream ss;
try {
pt::write_json(ss, tree, false);
} catch (const pt::ptree_error& e) {
return Status(1, "Error serializing JSON: " + std::string(e.what()));
}
json = ss.str();
return Status(0, "OK");
}
Status deserializeDistributedQueryResult(const pt::ptree& tree,
DistributedQueryResult& r) {
DistributedQueryRequest request;
auto s =
deserializeDistributedQueryRequest(tree.get_child("request"), request);
if (!s.ok()) {
return s;
}
QueryData results;
s = deserializeQueryData(tree.get_child("results"), results);
if (!s.ok()) {
return s;
}
r.request = request;
r.results = results;
return Status(0, "OK");
}
Status deserializeDistributedQueryResultJSON(const std::string& json,
DistributedQueryResult& r) {
std::stringstream ss(json);
pt::ptree tree;
try {
pt::read_json(ss, tree);
} catch (const pt::ptree_error& e) {
return Status(1, "Error serializing JSON: " + std::string(e.what()));
}
return deserializeDistributedQueryResult(tree, r);
}
/////////////////////////////////////////////////////////////////////////////
bool addUniqueRowToQueryData(QueryData& q, const Row& r) {
if (std::find(q.begin(), q.end(), r) != q.end()) {
return false;
@ -495,6 +394,41 @@ bool addUniqueRowToQueryData(QueryData& q, const Row& r) {
return true;
}
bool DatabasePlugin::initPlugin() {
// Initialize the database plugin using the flag.
return Registry::setActive("database", kInternalDatabase).ok();
}
void DatabasePlugin::shutdown() {
auto datbase_registry = Registry::registry("database");
for (auto& plugin : datbase_registry->names()) {
datbase_registry->remove(plugin);
}
}
Status DatabasePlugin::reset() {
tearDown();
return setUp();
}
bool DatabasePlugin::checkDB() {
kCheckingDB = true;
bool result = true;
try {
auto status = setUp();
if (kDBHandleOptionRequireWrite && read_only_) {
result = false;
}
tearDown();
result = status.ok();
} catch (const std::exception& e) {
VLOG(1) << "Database plugin check failed: " << e.what();
result = false;
}
kCheckingDB = false;
return result;
}
Status DatabasePlugin::call(const PluginRequest& request,
PluginResponse& response) {
if (request.count("action") == 0) {
@ -526,7 +460,7 @@ Status DatabasePlugin::call(const PluginRequest& request,
if (request.count("max") > 0) {
max = std::stoul(request.at("max"));
}
auto status = this->scan(domain, keys, max);
auto status = this->scan(domain, keys, request.at("prefix"), max);
for (const auto& key : keys) {
response.push_back({{"k", key}});
}
@ -536,51 +470,97 @@ Status DatabasePlugin::call(const PluginRequest& request,
return Status(1, "Unknown database plugin action");
}
static inline std::shared_ptr<DatabasePlugin> getDatabasePlugin() {
if (!Registry::exists("database", Registry::getActive("database"), true)) {
return nullptr;
}
auto plugin = Registry::get("database", Registry::getActive("database"));
return std::dynamic_pointer_cast<DatabasePlugin>(plugin);
}
Status getDatabaseValue(const std::string& domain,
const std::string& key,
std::string& value) {
PluginRequest request = {{"action", "get"}, {"domain", domain}, {"key", key}};
PluginResponse response;
auto status = Registry::call("database", "rocks", request, response);
if (!status.ok()) {
if (Registry::external()) {
// External registries (extensions) do not have databases active.
// It is not possible to use an extension-based database.
PluginRequest request = {
{"action", "get"}, {"domain", domain}, {"key", key}};
PluginResponse response;
auto status = Registry::call("database", request, response);
if (status.ok()) {
// Set value from the internally-known "v" key.
if (response.size() > 0 && response[0].count("v") > 0) {
value = response[0].at("v");
}
}
return status;
} else {
auto plugin = getDatabasePlugin();
return plugin->get(domain, key, value);
}
// Set value from the internally-known "v" key.
if (response.size() > 0 && response[0].count("v") > 0) {
value = response[0].at("v");
}
return status;
}
Status setDatabaseValue(const std::string& domain,
const std::string& key,
const std::string& value) {
PluginRequest request = {
{"action", "put"}, {"domain", domain}, {"key", key}, {"value", value}};
return Registry::call("database", "rocks", request);
if (Registry::external()) {
// External registries (extensions) do not have databases active.
// It is not possible to use an extension-based database.
PluginRequest request = {
{"action", "put"}, {"domain", domain}, {"key", key}, {"value", value}};
return Registry::call("database", request);
} else {
auto plugin = getDatabasePlugin();
return plugin->put(domain, key, value);
}
}
Status deleteDatabaseValue(const std::string& domain, const std::string& key) {
PluginRequest request = {
{"action", "remove"}, {"domain", domain}, {"key", key}};
return Registry::call("database", "rocks", request);
if (Registry::external()) {
// External registries (extensions) do not have databases active.
// It is not possible to use an extension-based database.
PluginRequest request = {
{"action", "remove"}, {"domain", domain}, {"key", key}};
return Registry::call("database", request);
} else {
auto plugin = getDatabasePlugin();
return plugin->remove(domain, key);
}
}
Status scanDatabaseKeys(const std::string& domain,
std::vector<std::string>& keys,
size_t max) {
PluginRequest request = {
{"action", "scan"}, {"domain", domain}, {"max", std::to_string(max)}};
PluginResponse response;
auto status = Registry::call("database", "rocks", request, response);
return scanDatabaseKeys(domain, keys, "", max);
}
for (const auto& item : response) {
if (item.count("k") > 0) {
keys.push_back(item.at("k"));
/// Get a list of keys for a given domain.
Status scanDatabaseKeys(const std::string& domain,
std::vector<std::string>& keys,
const std::string& prefix,
size_t max) {
if (Registry::external()) {
// External registries (extensions) do not have databases active.
// It is not possible to use an extension-based database.
PluginRequest request = {{"action", "scan"},
{"domain", domain},
{"prefix", prefix},
{"max", std::to_string(max)}};
PluginResponse response;
auto status = Registry::call("database", request, response);
for (const auto& item : response) {
if (item.count("k") > 0) {
keys.push_back(item.at("k"));
}
}
return status;
} else {
auto plugin = getDatabasePlugin();
return plugin->scan(domain, keys, prefix, max);
}
return status;
}
void dumpDatabase() {

400
osquery/database/db_handle.cpp
View File

@ -1,400 +0,0 @@
/*
* Copyright (c) 2014-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*
*/
#include <algorithm>
#include <mutex>
#include <stdexcept>
#include <sys/stat.h>
#include <snappy.h>
#include <rocksdb/env.h>
#include <rocksdb/options.h>
#include <osquery/database.h>
#include <osquery/filesystem.h>
#include <osquery/logger.h>
#include <osquery/status.h>
#include "osquery/database/db_handle.h"
namespace osquery {
class RocksDatabasePlugin : public DatabasePlugin {
public:
/// Data retrieval method.
Status get(const std::string& domain,
const std::string& key,
std::string& value) const override;
/// Data storage method.
Status put(const std::string& domain,
const std::string& key,
const std::string& value) override;
/// Data removal method.
Status remove(const std::string& domain, const std::string& k) override;
/// Key/index lookup method.
Status scan(const std::string& domain,
std::vector<std::string>& results,
size_t max = 0) const override;
};
/// Backing-storage provider for osquery internal/core.
REGISTER_INTERNAL(RocksDatabasePlugin, "database", "rocks");
/////////////////////////////////////////////////////////////////////////////
// Constants
/////////////////////////////////////////////////////////////////////////////
bool DBHandle::kDBHandleOptionAllowOpen = false;
bool DBHandle::kDBHandleOptionRequireWrite = false;
const std::string kPersistentSettings = "configurations";
const std::string kQueries = "queries";
const std::string kEvents = "events";
const std::string kLogs = "logs";
/**
* @brief A const vector of column families in RocksDB
*
* RocksDB has a concept of "column families" which are kind of like tables
* in other databases. kDomainds is populated with a list of all column
* families. If a string exists in kDomains, it's a column family in the
* database.
*/
const std::vector<std::string> kDomains = {
kPersistentSettings, kQueries, kEvents, kLogs};
CLI_FLAG(string,
database_path,
"/var/osquery/osquery.db",
"If using a disk-based backing store, specify a path");
FLAG_ALIAS(std::string, db_path, database_path);
CLI_FLAG(bool,
database_in_memory,
false,
"Keep osquery backing-store in memory");
FLAG_ALIAS(bool, use_in_memory_database, database_in_memory);
/////////////////////////////////////////////////////////////////////////////
// constructors and destructors
/////////////////////////////////////////////////////////////////////////////
/// A queryable mutex around database sanity checking.
static bool kCheckingDB = false;
void GlogRocksDBLogger::Logv(const char* format, va_list ap) {
// Convert RocksDB log to string and check if header or level-ed log.
char buffer[501] = {0};
vsnprintf(buffer, 500, format, ap);
va_end(ap);
if (buffer[0] != '[') {
return;
}
if (buffer[1] == 'E' || buffer[1] == 'W') {
LOG(INFO) << "RocksDB: " << buffer;
}
}
DBHandle::DBHandle(const std::string& path, bool in_memory)
: path_(path), in_memory_(in_memory) {
if (!kDBHandleOptionAllowOpen) {
LOG(WARNING) << RLOG(1629) << "Not allowed to create DBHandle instance";
}
// Set meta-data (mostly) handling options.
options_.create_if_missing = true;
options_.create_missing_column_families = true;
options_.info_log_level = rocksdb::ERROR_LEVEL;
options_.log_file_time_to_roll = 0;
options_.keep_log_file_num = 10;
options_.max_log_file_size = 1024 * 1024 * 1;
options_.stats_dump_period_sec = 0;
// Performance and optimization settings.
options_.compression = rocksdb::kNoCompression;
options_.compaction_style = rocksdb::kCompactionStyleLevel;
options_.arena_block_size = (4 * 1024);
options_.write_buffer_size = (4 * 1024) * 100; // 100 blocks.
options_.max_write_buffer_number = 2;
options_.min_write_buffer_number_to_merge = 1;
options_.max_background_compactions = 2;
options_.max_background_flushes = 2;
// Create an environment to replace the default logger.
if (logger_ == nullptr) {
logger_ = std::make_shared<GlogRocksDBLogger>();
}
options_.info_log = logger_;
column_families_.push_back(rocksdb::ColumnFamilyDescriptor(
rocksdb::kDefaultColumnFamilyName, options_));
for (const auto& cf_name : kDomains) {
column_families_.push_back(
rocksdb::ColumnFamilyDescriptor(cf_name, options_));
}
// Make the magic happen.
open();
}
void DBHandle::open() {
if (in_memory_) {
// Remove when MemEnv is included in librocksdb
// options_.env = rocksdb::NewMemEnv(rocksdb::Env::Default());
throw std::runtime_error("Cannot start in-memory RocksDB: Requires MemEnv");
}
if (pathExists(path_).ok() && !isReadable(path_).ok()) {
throw std::runtime_error("Cannot read RocksDB path: " + path_);
}
if (!kCheckingDB) {
VLOG(1) << "Opening RocksDB handle: " << path_;
}
auto s =
rocksdb::DB::Open(options_, path_, column_families_, &handles_, &db_);
if (!s.ok() || db_ == nullptr) {
if (kDBHandleOptionRequireWrite) {
// A failed open in R/W mode is a runtime error.
throw std::runtime_error(s.ToString());
}
if (!kCheckingDB) {
VLOG(1) << "Opening RocksDB failed: Continuing with read-only support";
}
#if !defined(ROCKSDB_LITE)
// RocksDB LITE does not support readonly mode.
// The database was readable but could not be opened, either (1) it is not
// writable or (2) it is already opened by another process.
// Try to open the database in a ReadOnly mode.
rocksdb::DB::OpenForReadOnly(
options_, path_, column_families_, &handles_, &db_);
#endif
// Also disable event publishers.
Flag::updateValue("disable_events", "true");
read_only_ = true;
}
// RocksDB may not create/append a directory with acceptable permissions.
if (!read_only_ && chmod(path_.c_str(), S_IRWXU) != 0) {
throw std::runtime_error("Cannot set permissions on RocksDB path: " +
path_);
}
}
DBHandle::~DBHandle() { close(); }
void DBHandle::close() {
for (auto handle : handles_) {
delete handle;
}
if (db_ != nullptr) {
delete db_;
}
}
/////////////////////////////////////////////////////////////////////////////
// getInstance methods
/////////////////////////////////////////////////////////////////////////////
DBHandleRef DBHandle::getInstance() {
return getInstance(FLAGS_database_path, FLAGS_database_in_memory);
}
bool DBHandle::checkDB() {
// Allow database instances to check if a status/sanity check was requested.
kCheckingDB = true;
try {
auto handle = DBHandle(FLAGS_database_path, FLAGS_database_in_memory);
kCheckingDB = false;
if (kDBHandleOptionRequireWrite && handle.read_only_) {
return false;
}
} catch (const std::exception& e) {
kCheckingDB = false;
VLOG(1) << e.what();
return false;
}
return true;
}
DBHandleRef DBHandle::getInstanceInMemory() { return getInstance("", true); }
DBHandleRef DBHandle::getInstanceAtPath(const std::string& path) {
return getInstance(path, false);
}
DBHandleRef DBHandle::getInstance(const std::string& path, bool in_memory) {
static DBHandleRef db_handle = DBHandleRef(new DBHandle(path, in_memory));
return db_handle;
}
void DBHandle::resetInstance(const std::string& path, bool in_memory) {
close();
path_ = path;
in_memory_ = in_memory;
open();
}
/////////////////////////////////////////////////////////////////////////////
// getters and setters
/////////////////////////////////////////////////////////////////////////////
rocksdb::DB* DBHandle::getDB() const { return db_; }
rocksdb::ColumnFamilyHandle* DBHandle::getHandleForColumnFamily(
const std::string& cf) const {
try {
for (size_t i = 0; i < kDomains.size(); i++) {
if (kDomains[i] == cf) {
return handles_[i];
}
}
} catch (const std::exception& e) {
// pass through and return nullptr
}
return nullptr;
}
/////////////////////////////////////////////////////////////////////////////
// Data manipulation methods
/////////////////////////////////////////////////////////////////////////////
Status DBHandle::Get(const std::string& domain,
const std::string& key,
std::string& value) const {
if (getDB() == nullptr) {
return Status(1, "Database not opened");
}
auto cfh = getHandleForColumnFamily(domain);
if (cfh == nullptr) {
return Status(1, "Could not get column family for " + domain);
}
auto s = getDB()->Get(rocksdb::ReadOptions(), cfh, key, &value);
return Status(s.code(), s.ToString());
}
Status DBHandle::Put(const std::string& domain,
const std::string& key,
const std::string& value) const {
if (read_only_) {
return Status(0, "Database in readonly mode");
}
auto cfh = getHandleForColumnFamily(domain);
if (cfh == nullptr) {
return Status(1, "Could not get column family for " + domain);
}
auto s = getDB()->Put(rocksdb::WriteOptions(), cfh, key, value);
if (s.code() != 0 && s.IsIOError()) {
// An error occurred, check if it is an IO error and remove the offending
// specific filename or log name.
std::string error_string = s.ToString();
size_t error_pos = error_string.find_last_of(":");
if (error_pos != std::string::npos) {
return Status(s.code(), "IOError: " + error_string.substr(error_pos + 2));
}
}
return Status(s.code(), s.ToString());
}
Status DBHandle::Delete(const std::string& domain,
const std::string& key) const {
if (read_only_) {
return Status(0, "Database in readonly mode");
}
auto cfh = getHandleForColumnFamily(domain);
if (cfh == nullptr) {
return Status(1, "Could not get column family for " + domain);
}
auto options = rocksdb::WriteOptions();
// We could sync here, but large deletes will cause multi-syncs.
// For example: event record expirations found in an expired index.
// options.sync = true;
auto s = getDB()->Delete(options, cfh, key);
return Status(s.code(), s.ToString());
}
Status DBHandle::Scan(const std::string& domain,
std::vector<std::string>& results,
size_t max) const {
// Trampoline into scan prefix with an empty prefix requirement.
return ScanPrefix(domain, results, "", max);
}
Status DBHandle::ScanPrefix(const std::string& domain,
std::vector<std::string>& results,
const std::string& prefix,
size_t max) const {
if (getDB() == nullptr) {
return Status(1, "Database not opened");
}
auto cfh = getHandleForColumnFamily(domain);
if (cfh == nullptr) {
return Status(1, "Could not get column family for " + domain);
}
auto options = rocksdb::ReadOptions();
options.verify_checksums = false;
options.fill_cache = false;
auto it = getDB()->NewIterator(rocksdb::ReadOptions(), cfh);
if (it == nullptr) {
return Status(1, "Could not get iterator for " + domain);
}
size_t count = 0;
for (it->SeekToFirst(); it->Valid(); it->Next()) {
auto key = it->key().ToString();
if (key.find(prefix) == 0) {
results.push_back(std::move(key));
if (max > 0 && ++count >= max) {
break;
}
}
}
delete it;
return Status(0, "OK");
}
Status RocksDatabasePlugin::get(const std::string& domain,
const std::string& key,
std::string& value) const {
return DBHandle::getInstance()->Get(domain, key, value);
}
Status RocksDatabasePlugin::put(const std::string& domain,
const std::string& key,
const std::string& value) {
return DBHandle::getInstance()->Put(domain, key, value);
}
Status RocksDatabasePlugin::remove(const std::string& domain,
const std::string& key) {
return DBHandle::getInstance()->Delete(domain, key);
}
Status RocksDatabasePlugin::scan(const std::string& domain,
std::vector<std::string>& results,
size_t max) const {
return DBHandle::getInstance()->Scan(domain, results, max);
}
}

296
osquery/database/db_handle.h
View File

@ -1,296 +0,0 @@
/*
* Copyright (c) 2014-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*
*/
#pragma once
#include <memory>
#include <string>
#include <vector>
#include <rocksdb/db.h>
#include <rocksdb/env.h>
#include <rocksdb/options.h>
#include <boost/noncopyable.hpp>
#include <osquery/core.h>
#include <osquery/flags.h>
namespace osquery {
DECLARE_string(database_path);
class DBHandle;
typedef std::shared_ptr<DBHandle> DBHandleRef;
class GlogRocksDBLogger : public rocksdb::Logger {
public:
// We intend to override a virtual method that is overloaded.
using rocksdb::Logger::Logv;
void Logv(const char* format, va_list ap) override;
};
/**
* @brief RAII singleton around RocksDB database access.
*
* Accessing RocksDB necessitates creating several pointers which must be
* carefully memory managed. DBHandle offers you a singleton which takes
* care of acquiring and releasing the relevant pointers and data structures
* for you.
*/
class DBHandle {
public:
/// Removes every column family handle and single DB handle/lock.
~DBHandle();
/**
* @brief The primary way to access the DBHandle singleton.
*
* DBHandle::getInstance() provides access to the DBHandle singleton.
*
* @code{.cpp}
* auto db = DBHandle::getInstance();
* std::string value;
* auto status = db->Get("default", "foo", value);
* if (status.ok()) {
* assert(value == "bar");
* }
* @endcode
*
* @return a shared pointer to an instance of DBHandle
*/
static DBHandleRef getInstance();
/**
* @brief Check the sanity of the database configuration options
*
* Create a handle to the backing store using the database configuration.
* Catch any instance creation exceptions and release the handle immediately.
*
* @return Success if a handle was created without error.
*/
static bool checkDB();
/// Require all DBHandle accesses to open a read and write handle.
static void setRequireWrite(bool rw) { kDBHandleOptionRequireWrite = rw; }
/// Allow DBHandle creations.
static void setAllowOpen(bool ao) { kDBHandleOptionAllowOpen = ao; }
public:
/////////////////////////////////////////////////////////////////////////////
// Data access methods
/////////////////////////////////////////////////////////////////////////////
/**
* @brief Get data from the database for a given domain and key.
*
* @param domain the "domain" or "column family"
* @param key the string key
* @param value the output string container, will be populated with data
*
* @return operation success or failure
*/
Status Get(const std::string& domain,
const std::string& key,
std::string& value) const;
/**
* @brief Put data into the database identified by a domain and key.
*
* @param domain the "domain" or "column family"
* @param key the string key
* @param value the data in a string container
*
* @return operation success or failure
*/
Status Put(const std::string& domain,
const std::string& key,
const std::string& value) const;
/**
* @brief Delete data from the database given a domain and key.
*
* @param domain the "domain" or "column family"
* @param key the string key
*
* @return operation success or failure
*/
Status Delete(const std::string& domain, const std::string& key) const;
/**
* @brief List the keys in a "domain"
*
* @param domain the "domain" or "column family"
* @param results an output list of all keys within the domain
* @param optional max limit the number of result keys to a max
*
* @return operation success or failure
*/
Status Scan(const std::string& domain,
std::vector<std::string>& results,
size_t max = 0) const;
/**
* @brief List the data in a "domain"
*
* @param domain the "domain" or "column family"
* @param results an output list of all keys with the given prefix
* @param prefix require each key to contain this string prefix
* @param optional max limit the number of result keys to a max
*
* @return operation success or failure
*/
Status ScanPrefix(const std::string& domain,
std::vector<std::string>& results,
const std::string& prefix,
size_t max = 0) const;
private:
/**
* @brief Default constructor
*
* DBHandle's constructor takes care of properly connecting to RocksDB and
* ensuring that all necessary column families are created. The resulting
* database handle can then be accessed via DBHandle::getDB() and the
* success of the connection can be determined by inspecting the resulting
* status code via DBHandle::getStatus()
*/
DBHandle();
/**
* @brief Internal only constructor used to create instances of DBHandle.
*
* This constructor allows you to specify a few more details about how you'd
* like DBHandle to be used. This is only used internally, so you should
* never actually use it.
*
* @param path the path to create/access the database
* @param in_memory a boolean indicating whether or not the database should
* be creating in memory or not.
*/
DBHandle(const std::string& path, bool in_memory);
/**
* @brief A method which allows you to override the database path
*
* This should only be used by unit tests. Never use it in production code.
*
* @return a shared pointer to an instance of DBHandle
*/
static DBHandleRef getInstanceAtPath(const std::string& path);
/**
* @brief A method which gets you an in-memory RocksDB instance.
*
* This should only be used by unit tests. Never use it in production code.
*
* @return a shared pointer to an instance of DBHandle
*/
static DBHandleRef getInstanceInMemory();
/**
* @brief A method which allows you to configure various aspects of RocksDB
* database options.
*
* This should only be used by unit tests. Never use it in production code.
*
* @param path the path to create/access the database
* @param in_memory a boolean indicating whether or not the database should
* be creating in memory or not.
*
* @return a shared pointer to an instance of DBHandle
*/
static DBHandleRef getInstance(const std::string& path, bool in_memory);
/// Allow friend classes, such as unit tests, to reset the instance.
void resetInstance(const std::string& path, bool in_memory);
/// Perform the DB open work.
void open();
/// Perform the DB close work.
void close();
/**
* @brief Private helper around accessing the column family handle for a
* specific column family, based on its name
*/
rocksdb::ColumnFamilyHandle* getHandleForColumnFamily(
const std::string& cf) const;
/**
* @brief Helper method which can be used to get a raw pointer to the
* underlying RocksDB database handle
*
* You probably shouldn't use this. DBHandle::getDB() should only be used
* when you're positive that it's the right thing to use.
*
* @return a pointer to the underlying RocksDB database handle
*/
rocksdb::DB* getDB() const;
public:
/// Control availability of the RocksDB handle (default false).
static bool kDBHandleOptionAllowOpen;
// The database must be opened in a R/W mode (default false).
static bool kDBHandleOptionRequireWrite;
private:
/////////////////////////////////////////////////////////////////////////////
// Private members
/////////////////////////////////////////////////////////////////////////////
/// The database handle
rocksdb::DB* db_{nullptr};
/// RocksDB logger instance.
std::shared_ptr<GlogRocksDBLogger> logger_{nullptr};
/// Column family descriptors which are used to connect to RocksDB
std::vector<rocksdb::ColumnFamilyDescriptor> column_families_;
/// A vector of pointers to column family handles
std::vector<rocksdb::ColumnFamilyHandle*> handles_;
/// The RocksDB connection options that are used to connect to RocksDB
rocksdb::Options options_;
/// The database was opened in a ReadOnly mode.
bool read_only_{false};
/// Location of RocksDB on disk, blank if in-memory is true.
std::string path_;
/// True if the database was started in an in-memory only mode.
bool in_memory_{false};
private:
friend class RocksDatabasePlugin;
friend class Query;
friend class EventSubscriberPlugin;
/////////////////////////////////////////////////////////////////////////////
// Unit tests which can access private members
/////////////////////////////////////////////////////////////////////////////
friend class DBHandleTests;
FRIEND_TEST(DBHandleTests, test_get);
FRIEND_TEST(DBHandleTests, test_put);
FRIEND_TEST(DBHandleTests, test_delete);
FRIEND_TEST(DBHandleTests, test_scan);
friend class QueryTests;
FRIEND_TEST(QueryTests, test_get_query_results);
FRIEND_TEST(QueryTests, test_is_query_name_in_database);
FRIEND_TEST(QueryTests, test_get_stored_query_names);
friend class EventsTests;
friend class EventsDatabaseTests;
};
}

101
osquery/database/plugins/ephemeral.cpp Normal file
View File

@ -0,0 +1,101 @@
/*
* Copyright (c) 2014-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*
*/
#include <osquery/database.h>
#include <osquery/logger.h>
namespace osquery {
DECLARE_string(database_path);
DECLARE_bool(database_in_memory);
class EphemeralDatabasePlugin : public DatabasePlugin {
using DBType = std::map<std::string, std::map<std::string, std::string> >;
public:
/// Data retrieval method.
Status get(const std::string& domain,
const std::string& key,
std::string& value) const override;
/// Data storage method.
Status put(const std::string& domain,
const std::string& key,
const std::string& value) override;
/// Data removal method.
Status remove(const std::string& domain, const std::string& k) override;
/// Key/index lookup method.
Status scan(const std::string& domain,
std::vector<std::string>& results,
const std::string& prefix,
size_t max = 0) const override;
public:
/// Database workflow: open and setup.
Status setUp() override {
DBType().swap(db_);
return Status(0);
}
private:
DBType db_;
};
/// Backing-storage provider for osquery internal/core.
REGISTER_INTERNAL(EphemeralDatabasePlugin, "database", "ephemeral");
Status EphemeralDatabasePlugin::get(const std::string& domain,
const std::string& key,
std::string& value) const {
if (db_.count(domain) > 0 && db_.at(domain).count(key) > 0) {
value = db_.at(domain).at(key);
return Status(0);
} else {
return Status(1);
}
}
Status EphemeralDatabasePlugin::put(const std::string& domain,
const std::string& key,
const std::string& value) {
db_[domain][key] = value;
return Status(0);
}
Status EphemeralDatabasePlugin::remove(const std::string& domain,
const std::string& k) {
db_[domain].erase(k);
return Status(0);
}
Status EphemeralDatabasePlugin::scan(const std::string& domain,
std::vector<std::string>& results,
const std::string& prefix,
size_t max) const {
if (db_.count(domain) == 0) {
return Status(0);
}
for (const auto& key : db_.at(domain)) {
if (!prefix.empty() &&
!(std::mismatch(prefix.begin(), prefix.end(), key.first.begin())
.first == prefix.end())) {
continue;
}
results.push_back(key.first);
if (max > 0 && results.size() >= max) {
break;
}
}
return Status(0);
}
}

350
osquery/database/plugins/rocksdb.cpp Normal file
View File

@ -0,0 +1,350 @@
/*
* Copyright (c) 2014-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*
*/
#include <mutex>
#include <sys/stat.h>
#include <snappy.h>
#include <rocksdb/db.h>
#include <rocksdb/env.h>
#include <rocksdb/options.h>
#include <osquery/database.h>
#include <osquery/filesystem.h>
#include <osquery/logger.h>
namespace osquery {
DECLARE_string(database_path);
DECLARE_bool(database_in_memory);
class GlogRocksDBLogger : public rocksdb::Logger {
public:
// We intend to override a virtual method that is overloaded.
using rocksdb::Logger::Logv;
void Logv(const char* format, va_list ap) override;
};
class RocksDBDatabasePlugin : public DatabasePlugin {
public:
/// Data retrieval method.
Status get(const std::string& domain,
const std::string& key,
std::string& value) const override;
/// Data storage method.
Status put(const std::string& domain,
const std::string& key,
const std::string& value) override;
/// Data removal method.
Status remove(const std::string& domain, const std::string& k) override;
/// Key/index lookup method.
Status scan(const std::string& domain,
std::vector<std::string>& results,
const std::string& prefix,
size_t max = 0) const override;
public:
/// Database workflow: open and setup.
Status setUp() override;
/// Database workflow: close and cleanup.
void tearDown() override { close(); }
/// Need to tear down open resources,
virtual ~RocksDBDatabasePlugin() { close(); }
private:
/// Obtain a close lock and release resources.
void close();
/**
* @brief Private helper around accessing the column family handle for a
* specific column family, based on its name
*/
rocksdb::ColumnFamilyHandle* getHandleForColumnFamily(
const std::string& cf) const;
/**
* @brief Helper method which can be used to get a raw pointer to the
* underlying RocksDB database handle
*
* @return a pointer to the underlying RocksDB database handle
*/
rocksdb::DB* getDB() const;
private:
bool initialized_{false};
/// The database handle
rocksdb::DB* db_{nullptr};
/// RocksDB logger instance.
std::shared_ptr<GlogRocksDBLogger> logger_{nullptr};
/// Column family descriptors which are used to connect to RocksDB
std::vector<rocksdb::ColumnFamilyDescriptor> column_families_;
/// A vector of pointers to column family handles
std::vector<rocksdb::ColumnFamilyHandle*> handles_;
/// The RocksDB connection options that are used to connect to RocksDB
rocksdb::Options options_;
/// Deconstruction mutex.
std::mutex close_mutex_;
};
/// Backing-storage provider for osquery internal/core.
REGISTER_INTERNAL(RocksDBDatabasePlugin, "database", "rocksdb");
void GlogRocksDBLogger::Logv(const char* format, va_list ap) {
// Convert RocksDB log to string and check if header or level-ed log.
std::string log_line;
{
char buffer[501] = {0};
vsnprintf(buffer, 500, format, ap);
va_end(ap);
if (buffer[0] != '[' || (buffer[1] != 'E' && buffer[1] != 'W')) {
return;
}
log_line = buffer;
}
// There is a spurious warning on first open.
if (log_line.find("Error when reading") == std::string::npos) {
LOG(INFO) << "RocksDB: " << log_line;
}
}
Status RocksDBDatabasePlugin::setUp() {
if (!kDBHandleOptionAllowOpen) {
LOG(WARNING) << RLOG(1629) << "Not allowed to create DBHandle instance";
}
if (!initialized_) {
initialized_ = true;
// Set meta-data (mostly) handling options.
options_.create_if_missing = true;
options_.create_missing_column_families = true;
options_.info_log_level = rocksdb::ERROR_LEVEL;
options_.log_file_time_to_roll = 0;
options_.keep_log_file_num = 10;
options_.max_log_file_size = 1024 * 1024 * 1;
options_.stats_dump_period_sec = 0;
// Performance and optimization settings.
options_.compression = rocksdb::kNoCompression;
options_.compaction_style = rocksdb::kCompactionStyleLevel;
options_.arena_block_size = (4 * 1024);
options_.write_buffer_size = (4 * 1024) * 100; // 100 blocks.
options_.max_write_buffer_number = 2;
options_.min_write_buffer_number_to_merge = 1;
options_.max_background_compactions = 2;
options_.max_background_flushes = 2;
// Create an environment to replace the default logger.
if (logger_ == nullptr) {
logger_ = std::make_shared<GlogRocksDBLogger>();
}
options_.info_log = logger_;
column_families_.push_back(rocksdb::ColumnFamilyDescriptor(
rocksdb::kDefaultColumnFamilyName, options_));
for (const auto& cf_name : kDomains) {
column_families_.push_back(
rocksdb::ColumnFamilyDescriptor(cf_name, options_));
}
}
// Consume the current settings.
// A configuration update may change them, but that does not affect state.
path_ = FLAGS_database_path;
in_memory_ = FLAGS_database_in_memory;
if (in_memory_) {
// Remove when MemEnv is included in librocksdb
// options_.env = rocksdb::NewMemEnv(rocksdb::Env::Default());
return Status(1, "Cannot start in-memory RocksDB: Requires MemEnv");
}
if (pathExists(path_).ok() && !isReadable(path_).ok()) {
return Status(1, "Cannot read RocksDB path: " + path_);
}
if (!DatabasePlugin::kCheckingDB) {
VLOG(1) << "Opening RocksDB handle: " << path_;
}
// Tests may trash calls to setUp, make sure subsequent calls do not leak.
close();
// Attempt to create a RocksDB instance and handles.
auto s =
rocksdb::DB::Open(options_, path_, column_families_, &handles_, &db_);
if (!s.ok() || db_ == nullptr) {
if (kDBHandleOptionRequireWrite) {
// A failed open in R/W mode is a runtime error.
return Status(1, s.ToString());
}
if (!DatabasePlugin::kCheckingDB) {
VLOG(1) << "Opening RocksDB failed: Continuing with read-only support";
}
#if !defined(ROCKSDB_LITE)
// RocksDB LITE does not support readonly mode.
// The database was readable but could not be opened, either (1) it is not
// writable or (2) it is already opened by another process.
// Try to open the database in a ReadOnly mode.
rocksdb::DB::OpenForReadOnly(options_, path_, column_families_, &handles_,
&db_);
#endif
// Also disable event publishers.
Flag::updateValue("disable_events", "true");
read_only_ = true;
}
// RocksDB may not create/append a directory with acceptable permissions.
if (!read_only_ && chmod(path_.c_str(), S_IRWXU) != 0) {
return Status(1, "Cannot set permissions on RocksDB path: " + path_);
}
return Status(0);
}
void RocksDBDatabasePlugin::close() {
std::unique_lock<std::mutex> lock(close_mutex_);
for (auto handle : handles_) {
delete handle;
}
handles_.clear();
if (db_ != nullptr) {
delete db_;
db_ = nullptr;
}
}
rocksdb::DB* RocksDBDatabasePlugin::getDB() const { return db_; }
rocksdb::ColumnFamilyHandle* RocksDBDatabasePlugin::getHandleForColumnFamily(
const std::string& cf) const {
try {
for (size_t i = 0; i < kDomains.size(); i++) {
if (kDomains[i] == cf) {
return handles_[i];
}
}
} catch (const std::exception& e) {
// pass through and return nullptr
}
return nullptr;
}
/////////////////////////////////////////////////////////////////////////////
// Data manipulation methods
/////////////////////////////////////////////////////////////////////////////
Status RocksDBDatabasePlugin::get(const std::string& domain,
const std::string& key,
std::string& value) const {
if (getDB() == nullptr) {
return Status(1, "Database not opened");
}
auto cfh = getHandleForColumnFamily(domain);
if (cfh == nullptr) {
return Status(1, "Could not get column family for " + domain);
}
auto s = getDB()->Get(rocksdb::ReadOptions(), cfh, key, &value);
return Status(s.code(), s.ToString());
}
Status RocksDBDatabasePlugin::put(const std::string& domain,
const std::string& key,
const std::string& value) {
if (read_only_) {
return Status(0, "Database in readonly mode");
}
auto cfh = getHandleForColumnFamily(domain);
if (cfh == nullptr) {
return Status(1, "Could not get column family for " + domain);
}
auto s = getDB()->Put(rocksdb::WriteOptions(), cfh, key, value);
if (s.code() != 0 && s.IsIOError()) {
// An error occurred, check if it is an IO error and remove the offending
// specific filename or log name.
std::string error_string = s.ToString();
size_t error_pos = error_string.find_last_of(":");
if (error_pos != std::string::npos) {
return Status(s.code(), "IOError: " + error_string.substr(error_pos + 2));
}
}
return Status(s.code(), s.ToString());
}
Status RocksDBDatabasePlugin::remove(const std::string& domain,
const std::string& key) {
if (read_only_) {
return Status(0, "Database in readonly mode");
}
auto cfh = getHandleForColumnFamily(domain);
if (cfh == nullptr) {
return Status(1, "Could not get column family for " + domain);
}
auto options = rocksdb::WriteOptions();
// We could sync here, but large deletes will cause multi-syncs.
// For example: event record expirations found in an expired index.
// options.sync = true;
auto s = getDB()->Delete(options, cfh, key);
return Status(s.code(), s.ToString());
}
Status RocksDBDatabasePlugin::scan(const std::string& domain,
std::vector<std::string>& results,
const std::string& prefix,
size_t max) const {
if (getDB() == nullptr) {
return Status(1, "Database not opened");
}
auto cfh = getHandleForColumnFamily(domain);
if (cfh == nullptr) {
return Status(1, "Could not get column family for " + domain);
}
auto options = rocksdb::ReadOptions();
options.verify_checksums = false;
options.fill_cache = false;
auto it = getDB()->NewIterator(rocksdb::ReadOptions(), cfh);
if (it == nullptr) {
return Status(1, "Could not get iterator for " + domain);
}
size_t count = 0;
for (it->SeekToFirst(); it->Valid(); it->Next()) {
auto key = it->key().ToString();
if (key.find(prefix) == 0) {
results.push_back(std::move(key));
if (max > 0 && ++count >= max) {
break;
}
}
}
delete it;
return Status(0, "OK");
}
}

277
osquery/database/plugins/sqlite.cpp Normal file
View File

@ -0,0 +1,277 @@
/*
* Copyright (c) 2014-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*
*/
#include <mutex>
#include <sqlite3.h>
#include <sys/stat.h>
#include <osquery/database.h>
#include <osquery/filesystem.h>
#include <osquery/logger.h>
namespace osquery {
DECLARE_string(database_path);
DECLARE_bool(database_in_memory);
const std::map<std::string, std::string> kDBSettings = {
{"synchronous", "OFF"}, {"count_changes", "OFF"},
{"default_temp_store", "2"}, {"auto_vacuum", "FULL"},
{"journal_mode", "OFF"}, {"cache_size", "1000"},
{"page_count", "1000"},
};
class SQLiteDatabasePlugin : public DatabasePlugin {
public:
/// Data retrieval method.
Status get(const std::string& domain,
const std::string& key,
std::string& value) const override;
/// Data storage method.
Status put(const std::string& domain,
const std::string& key,
const std::string& value) override;
/// Data removal method.
Status remove(const std::string& domain, const std::string& k) override;
/// Key/index lookup method.
Status scan(const std::string& domain,
std::vector<std::string>& results,
const std::string& prefix,
size_t max = 0) const override;
public:
/// Database workflow: open and setup.
Status setUp() override;
/// Database workflow: close and cleanup.
void tearDown() override { close(); }
/// Need to tear down open resources,
virtual ~SQLiteDatabasePlugin() { close(); }
private:
void close();
private:
/// The long-lived sqlite3 database.
sqlite3* db_{nullptr};
/// Deconstruction mutex.
std::mutex close_mutex_;
};
/// Backing-storage provider for osquery internal/core.
REGISTER_INTERNAL(SQLiteDatabasePlugin, "database", "sqlite");
Status SQLiteDatabasePlugin::setUp() {
if (!DatabasePlugin::kDBHandleOptionAllowOpen) {
LOG(WARNING) << RLOG(1629) << "Not allowed to create DBHandle instance";
}
// Consume the current settings.
// A configuration update may change them, but that does not affect state.
path_ = FLAGS_database_path;
in_memory_ = FLAGS_database_in_memory;
if (!in_memory_ && pathExists(path_).ok() && !isReadable(path_).ok()) {
return Status(1, "Cannot read database path: " + path_);
}
if (!DatabasePlugin::kCheckingDB) {
VLOG(1) << "Opening database handle: " << path_;
}
// Tests may trash calls to setUp, make sure subsequent calls do not leak.
close();
// Can actually try to create a DB in memory with: in_memory_.
// Open the SQLite backing storage at path_
auto result = sqlite3_open_v2(
((in_memory_) ? ":memory:" : path_.c_str()),
&db_,
(SQLITE_OPEN_FULLMUTEX | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE),
nullptr);
if (result != SQLITE_OK || db_ == nullptr) {
if (DatabasePlugin::kDBHandleOptionRequireWrite) {
close();
// A failed open in R/W mode is a runtime error.
return Status(1, "Cannot open database: " + std::to_string(result));
}
if (!DatabasePlugin::kCheckingDB) {
VLOG(1) << "Opening database failed: Continuing with read-only support";
}
read_only_ = true;
}
if (!read_only_) {
for (const auto& domain : kDomains) {
std::string q = "create table if not exists " + domain +
" (key TEXT PRIMARY KEY, value TEXT);";
result = sqlite3_exec(db_, q.c_str(), nullptr, nullptr, nullptr);
if (result != SQLITE_OK) {
close();
return Status(1, "Cannot create domain: " + domain);
}
}
std::string settings;
for (const auto& setting : kDBSettings) {
settings += "PRAGMA " + setting.first + "=" + setting.second + "; ";
}
sqlite3_exec(db_, settings.c_str(), nullptr, nullptr, nullptr);
}
// RocksDB may not create/append a directory with acceptable permissions.
if (!read_only_ && chmod(path_.c_str(), S_IRWXU) != 0) {
close();
return Status(1, "Cannot set permissions on database path: " + path_);
}
return Status(0);
}
void SQLiteDatabasePlugin::close() {
std::unique_lock<std::mutex> lock(close_mutex_);
if (db_ != nullptr) {
sqlite3_close(db_);
db_ = nullptr;
}
}
static int getData(void* argument, int argc, char* argv[], char* column[]) {
if (argument == nullptr) {
return SQLITE_MISUSE;
}
QueryData* qData = (QueryData*)argument;
Row r;
for (int i = 0; i < argc; i++) {
if (column[i] != nullptr) {
r[column[i]] = (argv[i] != nullptr) ? argv[i] : "";
}
}
(*qData).push_back(std::move(r));
return 0;
}
Status SQLiteDatabasePlugin::get(const std::string& domain,
const std::string& key,
std::string& value) const {
QueryData results;
char* err = nullptr;
std::string q = "select value from " + domain + " where key = '" + key + "';";
sqlite3_exec(db_, q.c_str(), getData, &results, &err);
if (err != nullptr) {
sqlite3_free(err);
}
// Only assign value if the query found a result.
if (results.size() > 0) {
value = std::move(results[0]["value"]);
return Status(0);
}
return Status(1);
}
static void tryVacuum(sqlite3* db) {
std::string q =
"SELECT (sum(s1.pageno + 1 == s2.pageno) * 1.0 / count(*)) < 0.01 as v "
" FROM "
"(SELECT pageno FROM dbstat ORDER BY path) AS s1,"
"(SELECT pageno FROM dbstat ORDER BY path) AS s2 WHERE "
"s1.rowid + 1 = s2.rowid; ";
QueryData results;
sqlite3_exec(db, q.c_str(), getData, &results, nullptr);
if (results.size() > 0 && results[0]["v"].back() == '1') {
sqlite3_exec(db, "vacuum;", nullptr, nullptr, nullptr);
}
}
Status SQLiteDatabasePlugin::put(const std::string& domain,
const std::string& key,
const std::string& value) {
if (read_only_) {
return Status(0, "Database in readonly mode");
}
sqlite3_stmt* stmt = nullptr;
std::string q = "insert or replace into " + domain + " values (?1, ?2);";
sqlite3_prepare_v2(db_, q.c_str(), -1, &stmt, nullptr);
sqlite3_bind_text(stmt, 1, key.c_str(), -1, SQLITE_STATIC);
sqlite3_bind_text(stmt, 2, value.c_str(), -1, SQLITE_STATIC);
auto rc = sqlite3_step(stmt);
if (rc != SQLITE_DONE) {
return Status(1);
}
sqlite3_finalize(stmt);
if (rand() % 10 == 0) {
tryVacuum(db_);
}
return Status(0);
}
Status SQLiteDatabasePlugin::remove(const std::string& domain,
const std::string& key) {
if (read_only_) {
return Status(0, "Database in readonly mode");
}
sqlite3_stmt* stmt = nullptr;
std::string q = "delete from " + domain + " where key IN (?1);";
sqlite3_prepare_v2(db_, q.c_str(), -1, &stmt, nullptr);
sqlite3_bind_text(stmt, 1, key.c_str(), -1, SQLITE_STATIC);
auto rc = sqlite3_step(stmt);
if (rc != SQLITE_DONE) {
return Status(1);
}
sqlite3_finalize(stmt);
if (rand() % 10 == 0) {
tryVacuum(db_);
}
return Status(0);
}
Status SQLiteDatabasePlugin::scan(const std::string& domain,
std::vector<std::string>& results,
const std::string& prefix,
size_t max) const {
QueryData _results;
char* err = nullptr;
std::string q =
"select key from " + domain + " where key LIKE '" + prefix + "%'";
if (max > 0) {
q += " limit " + std::to_string(max);
}
sqlite3_exec(db_, q.c_str(), getData, &_results, &err);
if (err != nullptr) {
sqlite3_free(err);
}
// Only assign value if the query found a result.
for (auto& r : _results) {
results.push_back(std::move(r["key"]));
}
return Status(0, "OK");
}
}

34
osquery/database/plugins/tests/rocksdb_tests.cpp Normal file
View File

@ -0,0 +1,34 @@
/*
* Copyright (c) 2014-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*
*/
#include <osquery/sql.h>
#include "osquery/database/tests/plugin_tests.h"
namespace osquery {
class RocksDBDatabasePluginTests : public DatabasePluginTests {
protected:
std::string name() override { return "rocksdb"; }
};
// Define the default set of database plugin operation tests.
CREATE_DATABASE_TESTS(RocksDBDatabasePluginTests);
TEST_F(RocksDBDatabasePluginTests, test_rocksdb_loglevel) {
// Make sure a log file was created.
EXPECT_FALSE(pathExists(path_ + "/LOG"));
// Make sure no log file is created.
// RocksDB logs are intercepted and forwarded to the GLog sink.
auto details = SQL::selectAllFrom("file", "path", EQUALS, path_ + "/LOG");
ASSERT_EQ(details.size(), 0U);
}
}

22
osquery/database/plugins/tests/sqlite_tests.cpp Normal file
View File

@ -0,0 +1,22 @@
/*
* Copyright (c) 2014-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*
*/
#include "osquery/database/tests/plugin_tests.h"
namespace osquery {
class SQLiteDatabasePluginTests : public DatabasePluginTests {
protected:
std::string name() override { return "sqlite"; }
};
// Define the default set of database plugin operation tests.
CREATE_DATABASE_TESTS(SQLiteDatabasePluginTests);
}

37
osquery/database/query.cpp
View File

@ -14,21 +14,13 @@
namespace osquery {
/////////////////////////////////////////////////////////////////////////////
// Data access methods
/////////////////////////////////////////////////////////////////////////////
Status Query::getPreviousQueryResults(QueryData& results) {
return getPreviousQueryResults(results, DBHandle::getInstance());
}
Status Query::getPreviousQueryResults(QueryData& results, DBHandleRef db) {
if (!isQueryNameInDatabase()) {
return Status(0, "Query name not found in database");
}
std::string raw;
auto status = db->Get(kQueries, name_, raw);
auto status = getDatabaseValue(kQueries, name_, raw);
if (!status.ok()) {
return status;
}
@ -41,41 +33,28 @@ Status Query::getPreviousQueryResults(QueryData& results, DBHandleRef db) {
}
std::vector<std::string> Query::getStoredQueryNames() {
return getStoredQueryNames(DBHandle::getInstance());
}
std::vector<std::string> Query::getStoredQueryNames(DBHandleRef db) {
std::vector<std::string> results;
db->Scan(kQueries, results);
scanDatabaseKeys(kQueries, results);
return results;
}
bool Query::isQueryNameInDatabase() {
return isQueryNameInDatabase(DBHandle::getInstance());
}
bool Query::isQueryNameInDatabase(DBHandleRef db) {
auto names = Query::getStoredQueryNames(db);
auto names = Query::getStoredQueryNames();
return std::find(names.begin(), names.end(), name_) != names.end();
}
Status Query::addNewResults(const osquery::QueryData& qd) {
return addNewResults(qd, DBHandle::getInstance());
}
Status Query::addNewResults(const QueryData& qd, DBHandleRef db) {
Status Query::addNewResults(const QueryData& qd) {
DiffResults dr;
return addNewResults(qd, dr, false, db);
return addNewResults(qd, dr, false);
}
Status Query::addNewResults(const QueryData& qd, DiffResults& dr) {
return addNewResults(qd, dr, true, DBHandle::getInstance());
return addNewResults(qd, dr, true);
}
Status Query::addNewResults(const QueryData& current_qd,
DiffResults& dr,
bool calculate_diff,
DBHandleRef db) {
bool calculate_diff) {
// Get the rows from the last run of this query name.
QueryData previous_qd;
auto status = getPreviousQueryResults(previous_qd);
@ -97,7 +76,7 @@ Status Query::addNewResults(const QueryData& current_qd,
return status;
}
status = db->Put(kQueries, name_, json);
status = setDatabaseValue(kQueries, name_, json);
if (!status.ok()) {
return status;
}

89
osquery/database/query.h
View File

@ -14,10 +14,8 @@
#include <string>
#include <vector>
#include <osquery/status.h>
#include <osquery/database.h>
#include "osquery/database/db_handle.h"
#include <osquery/status.h>
namespace osquery {
@ -58,23 +56,6 @@ class Query {
*/
Status getPreviousQueryResults(QueryData& results);
private:
/**
* @brief Serialize the data in RocksDB into a useful data structure using a
* custom database handle
*
* This method is the same as getHistoricalQueryResults, but with the
* addition of a parameter which allows you to pass a custom RocksDB
* database handle.
*
* @param hQR the output HistoricalQueryResults struct
* @param db a shared pointer to a custom DBHandle
*
* @return the success or failure of the operation
* @see getHistoricalQueryResults
*/
Status getPreviousQueryResults(QueryData& results, DBHandleRef db);
public:
/**
* @brief Get the names of all historical queries that are stored in RocksDB
@ -88,22 +69,6 @@ class Query {
*/
static std::vector<std::string> getStoredQueryNames();
private:
/**
* @brief Get the names of all historical queries that are stored in RocksDB
* using a custom database handle
*
* This method is the same as getStoredQueryNames(), but with the addition
* of a parameter which allows you to pass a custom RocksDB database handle.
*
* @param db a custom RocksDB database handle
*
* @return a vector containing the string names of all scheduled queries
*
* @see getStoredQueryNames()
*/
static std::vector<std::string> getStoredQueryNames(DBHandleRef db);
public:
/**
* @brief Accessor method for checking if a given scheduled query exists in
@ -113,20 +78,6 @@ class Query {
*/
bool isQueryNameInDatabase();
private:
/**
* @brief Accessor method for checking if a given scheduled query exists in
* the database, using a custom database handle
*
* This method is the same as isQueryNameInDatabase(), but with the addition
* of a parameter which allows you to pass a custom RocksDB database handle
*
* @param db a custom RocksDB database handle
*
* @return does the scheduled query which is already exists in the database
*/
bool isQueryNameInDatabase(DBHandleRef db);
public:
/**
* @brief Add a new set of results to the persistant storage
@ -143,24 +94,6 @@ class Query {
*/
Status addNewResults(const QueryData& qd);
private:
/**
* @brief Add a new set of results to the persistant storage using a custom
* database handle
*
* This method is the same as addNewResults(), but with the addition of a
* parameter which allows you to pass a custom RocksDB database handle
*
* @param qd the QueryData object, which has the results of the query which
* you would like to store
* @param unix_time the time that the query was executed
* @param db a custom RocksDB database handle
*
* @return an instance of osquery::Status indicating the success or failure
* of the operation
*/
Status addNewResults(const QueryData& qd, DBHandleRef db);
public:
/**
* @brief Add a new set of results to the persistent storage and get back
@ -192,8 +125,7 @@ class Query {
*/
Status addNewResults(const QueryData& qd,
DiffResults& dr,
bool calculate_diff,
DBHandleRef db);
bool calculate_diff);
public:
/**
@ -205,22 +137,6 @@ class Query {
*/
Status getCurrentResults(QueryData& qd);
private:
/**
* @brief A getter for the most recent result set for a scheduled query,
* but with the addition of a parameter which allows you to pass a custom
* RocksDB database handle.
*
* This method is the same as Query::getCurrentResults, but with addition of a
* parameter which allows you to pass a custom RocksDB database handle.
*
* @param qd the output QueryData object
* @param db a custom RocksDB database handle
*
* @return the success or failure of the operation
*/
Status getCurrentResults(QueryData& qd, DBHandleRef db);
private:
/////////////////////////////////////////////////////////////////////////////
// Private members
@ -228,6 +144,7 @@ class Query {
/// The scheduled query and internal
ScheduledQuery query_;
/// The scheduled query name.
std::string name_;

129
osquery/database/tests/db_handle_tests.cpp
View File

@ -1,129 +0,0 @@
/*
* Copyright (c) 2014-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*
*/
#include <algorithm>
#include <boost/filesystem/operations.hpp>
#include <gtest/gtest.h>
#include <osquery/filesystem.h>
#include <osquery/flags.h>
#include <osquery/logger.h>
#include <osquery/sql.h>
#include <osquery/tables.h>
#include "osquery/database/db_handle.h"
#include "osquery/core/test_util.h"
namespace osquery {
class DBHandleTests : public testing::Test {
public:
void SetUp() {
// A database instance is setup during testing initialize (initTesting).
// We need to reset that instance to test unordered expectations.
path_ = kTestWorkingDirectory + std::to_string(rand() % 10000 + 20000);
DBHandle::getInstance()->resetInstance(path_, false);
cfh_queries_ = DBHandle::getInstance()->getHandleForColumnFamily(kQueries);
cfh_foobar_ =
DBHandle::getInstance()->getHandleForColumnFamily("foobartest");
db_ = DBHandle::getInstance();
}
void TearDown() {
// Clean the transient instance and reset to the testing instance.
boost::filesystem::remove_all(path_);
auto path = Flag::getValue("database_path");
DBHandle::getInstance()->resetInstance(path, false);
}
public:
std::string path_;
rocksdb::ColumnFamilyHandle* cfh_queries_;
rocksdb::ColumnFamilyHandle* cfh_foobar_;
std::shared_ptr<DBHandle> db_;
};
TEST_F(DBHandleTests, test_singleton_on_disk) {
auto db1 = DBHandle::getInstance();
auto db2 = DBHandle::getInstance();
EXPECT_EQ(db1, db2);
}
TEST_F(DBHandleTests, test_get_handle_for_column_family) {
ASSERT_TRUE(cfh_queries_ != nullptr);
ASSERT_TRUE(cfh_foobar_ == nullptr);
}
TEST_F(DBHandleTests, test_get) {
db_->getDB()->Put(
rocksdb::WriteOptions(), cfh_queries_, "test_query_123", "{}");
std::string r;
std::string key = "test_query_123";
auto s = db_->Get(kQueries, key, r);
EXPECT_TRUE(s.ok());
EXPECT_EQ(s.toString(), "OK");
EXPECT_EQ(r, "{}");
}
TEST_F(DBHandleTests, test_put) {
auto s = db_->Put(kQueries, "test_put", "bar");
EXPECT_TRUE(s.ok());
EXPECT_EQ(s.toString(), "OK");
}
TEST_F(DBHandleTests, test_delete) {
db_->Put(kQueries, "test_delete", "baz");
auto s = db_->Delete(kQueries, "test_delete");
EXPECT_TRUE(s.ok());
EXPECT_EQ(s.toString(), "OK");
}
TEST_F(DBHandleTests, test_scan) {
db_->Put(kQueries, "test_scan_foo1", "baz");
db_->Put(kQueries, "test_scan_foo2", "baz");
db_->Put(kQueries, "test_scan_foo3", "baz");
std::vector<std::string> keys;
std::vector<std::string> expected = {
"test_scan_foo1", "test_scan_foo2", "test_scan_foo3"};
auto s = db_->Scan(kQueries, keys);
EXPECT_TRUE(s.ok());
EXPECT_EQ(s.toString(), "OK");
EXPECT_EQ(keys.size(), 3U);
for (const auto& i : expected) {
EXPECT_NE(std::find(keys.begin(), keys.end(), i), keys.end());
}
}
TEST_F(DBHandleTests, test_scan_limit) {
db_->Put(kQueries, "test_scan_foo1", "baz");
db_->Put(kQueries, "test_scan_foo2", "baz");
db_->Put(kQueries, "test_scan_foo3", "baz");
std::vector<std::string> keys;
auto s = db_->Scan(kQueries, keys, 2);
EXPECT_TRUE(s.ok());
EXPECT_EQ(s.toString(), "OK");
EXPECT_EQ(keys.size(), 2U);
}
TEST_F(DBHandleTests, test_rocksdb_loglevel) {
// Make sure a log file was created.
EXPECT_FALSE(pathExists(path_ + "/LOG"));
// Make sure no log file is created.
// RocksDB logs are intercepted and forwarded to the GLog sink.
auto details = SQL::selectAllFrom("file", "path", EQUALS, path_ + "/LOG");
ASSERT_EQ(details.size(), 0U);
}
}

88
osquery/database/tests/plugin_tests.cpp Normal file
View File

@ -0,0 +1,88 @@
/*
* Copyright (c) 2014-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*
*/
#include "osquery/database/tests/plugin_tests.h"
namespace osquery {
class EphemeralDatabasePluginTests : public DatabasePluginTests {
protected:
std::string name() override { return "ephemeral"; }
};
// Define the default set of database plugin operation tests.
CREATE_DATABASE_TESTS(EphemeralDatabasePluginTests);
void DatabasePluginTests::testPluginCheck() {
// Do not worry about multiple set-active calls.
// For testing purposes they should be idempotent.
EXPECT_TRUE(Registry::setActive("database", getName()));
// Get an instance of the database plugin and call check.
auto plugin = Registry::get("database", getName());
auto db_plugin = std::dynamic_pointer_cast<DatabasePlugin>(plugin);
EXPECT_TRUE(db_plugin->checkDB());
// Testing relies on database resetting too.
EXPECT_TRUE(db_plugin->reset());
}
void DatabasePluginTests::testPut() {
auto s = getPlugin()->put(kQueries, "test_put", "bar");
EXPECT_TRUE(s.ok());
EXPECT_EQ(s.getMessage(), "OK");
}
void DatabasePluginTests::testGet() {
getPlugin()->put(kQueries, "test_get", "bar");
std::string r;
auto s = getPlugin()->get(kQueries, "test_get", r);
EXPECT_TRUE(s.ok());
EXPECT_EQ(s.getMessage(), "OK");
EXPECT_EQ(r, "bar");
}
void DatabasePluginTests::testDelete() {
getPlugin()->put(kQueries, "test_delete", "baz");
auto s = getPlugin()->remove(kQueries, "test_delete");
EXPECT_TRUE(s.ok());
EXPECT_EQ(s.getMessage(), "OK");
}
void DatabasePluginTests::testScan() {
getPlugin()->put(kQueries, "test_scan_foo1", "baz");
getPlugin()->put(kQueries, "test_scan_foo2", "baz");
getPlugin()->put(kQueries, "test_scan_foo3", "baz");
std::vector<std::string> keys;
std::vector<std::string> expected = {"test_scan_foo1", "test_scan_foo2",
"test_scan_foo3"};
auto s = getPlugin()->scan(kQueries, keys, "");
EXPECT_TRUE(s.ok());
EXPECT_EQ(s.getMessage(), "OK");
EXPECT_EQ(keys.size(), 3U);
for (const auto& i : expected) {
EXPECT_NE(std::find(keys.begin(), keys.end(), i), keys.end());
}
}
void DatabasePluginTests::testScanLimit() {
getPlugin()->put(kQueries, "test_scan_foo1", "baz");
getPlugin()->put(kQueries, "test_scan_foo2", "baz");
getPlugin()->put(kQueries, "test_scan_foo3", "baz");
std::vector<std::string> keys;
auto s = getPlugin()->scan(kQueries, keys, "", 2);
EXPECT_TRUE(s.ok());
EXPECT_EQ(s.getMessage(), "OK");
EXPECT_EQ(keys.size(), 2U);
}
}

86
osquery/database/tests/plugin_tests.h Normal file
View File

@ -0,0 +1,86 @@
/*
* Copyright (c) 2014-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*
*/
#pragma once
#include <gtest/gtest.h>
#include <boost/filesystem/operations.hpp>
#include <osquery/database.h>
#include <osquery/flags.h>
#include "osquery/core/test_util.h"
/// The following test macros allow pretty test output.
#define CREATE_DATABASE_TESTS(n) \
TEST_F(n, test_plugin_check) { testPluginCheck(); } \
TEST_F(n, test_put) { testPut(); } \
TEST_F(n, test_get) { testGet(); } \
TEST_F(n, test_delete) { testDelete(); } \
TEST_F(n, test_scan) { testScan(); } \
TEST_F(n, test_scan_limit) { testScanLimit(); }
namespace osquery {
DECLARE_string(database_path);
class DatabasePluginTests : public testing::Test {
public:
void SetUp() override {
existing_plugin_ = Registry::getActive("database");
Registry::get("database", existing_plugin_)->tearDown();
setName(name());
path_ = FLAGS_database_path;
boost::filesystem::remove_all(path_);
auto plugin = Registry::get("database", getName());
plugin_ = std::dynamic_pointer_cast<DatabasePlugin>(plugin);
plugin_->reset();
}
void TearDown() override {
Registry::get("database", name_)->tearDown();
Registry::setActive("database", existing_plugin_);
}
protected:
/// Path to testing database.
std::string path_;
protected:
/// Require each plugin tester to implement a set name.
virtual std::string name() = 0;
private:
void setName(const std::string& name) { name_ = name; }
const std::string& getName() { return name_; }
std::shared_ptr<DatabasePlugin> getPlugin() { return plugin_; }
private:
/// Plugin name
std::string name_;
/// Plugin casted from setUp, ready to run tests.
std::shared_ptr<DatabasePlugin> plugin_{nullptr};
/// Previous active database plugin.
std::string existing_plugin_;
protected:
void testPluginCheck();
void testPut();
void testGet();
void testDelete();
void testScan();
void testScanLimit();
};
}

37
osquery/database/tests/query_tests.cpp
View File

@ -19,18 +19,9 @@
#include "osquery/core/test_util.h"
#include "osquery/database/query.h"
const std::string kTestingQueryDBPath = "/tmp/rocksdb-osquery-querytests";
namespace osquery {
class QueryTests : public testing::Test {
public:
void SetUp() { db_ = DBHandle::getInstanceAtPath(kTestingQueryDBPath); }
void TearDown() { boost::filesystem::remove_all(kTestingQueryDBPath); }
public:
std::shared_ptr<DBHandle> db_;
};
class QueryTests : public testing::Test {};
TEST_F(QueryTests, test_private_members) {
auto query = getOsqueryScheduledQuery();
@ -42,21 +33,21 @@ TEST_F(QueryTests, test_add_and_get_current_results) {
// Test adding a "current" set of results to a scheduled query instance.
auto query = getOsqueryScheduledQuery();
auto cf = Query("foobar", query);
auto status = cf.addNewResults(getTestDBExpectedResults(), db_);
auto status = cf.addNewResults(getTestDBExpectedResults());
EXPECT_TRUE(status.ok());
EXPECT_EQ(status.toString(), "OK");
// Simulate results from several schedule runs, calculate differentials.
for (auto result : getTestDBResultStream()) {
// Get the results from the previous query execution (from RocksDB).
// Get the results from the previous query execution (from the DB).
QueryData previous_qd;
auto status = cf.getPreviousQueryResults(previous_qd, db_);
auto status = cf.getPreviousQueryResults(previous_qd);
EXPECT_TRUE(status.ok());
EXPECT_EQ(status.toString(), "OK");
// Add the "current" results and output the differentials.
DiffResults dr;
auto s = cf.addNewResults(result.second, dr, true, db_);
auto s = cf.addNewResults(result.second, dr, true);
EXPECT_TRUE(s.ok());
// Call the diffing utility directly.
@ -65,7 +56,7 @@ TEST_F(QueryTests, test_add_and_get_current_results) {
// After Query::addNewResults the previous results are now current.
QueryData qd;
cf.getPreviousQueryResults(qd, db_);
cf.getPreviousQueryResults(qd);
EXPECT_EQ(qd, result.second);
}
}
@ -74,13 +65,13 @@ TEST_F(QueryTests, test_get_query_results) {
// Grab an expected set of query data and add it as the previous result.
auto encoded_qd = getSerializedQueryDataJSON();
auto query = getOsqueryScheduledQuery();
auto status = db_->Put(kQueries, "foobar", encoded_qd.first);
auto status = setDatabaseValue(kQueries, "foobar", encoded_qd.first);
EXPECT_TRUE(status.ok());
// Use the Query retrieval API to check the now "previous" result.
QueryData previous_qd;
auto cf = Query("foobar", query);
status = cf.getPreviousQueryResults(previous_qd, db_);
status = cf.getPreviousQueryResults(previous_qd);
EXPECT_TRUE(status.ok());
}
@ -89,31 +80,31 @@ TEST_F(QueryTests, test_query_name_not_found_in_db) {
QueryData previous_qd;
auto query = getOsqueryScheduledQuery();
auto cf = Query("not_a_real_query", query);
auto status = cf.getPreviousQueryResults(previous_qd, db_);
EXPECT_TRUE(status.toString() == "Query name not found in database");
auto status = cf.getPreviousQueryResults(previous_qd);
EXPECT_TRUE(status.ok());
EXPECT_EQ(status.toString(), "Query name not found in database");
}
TEST_F(QueryTests, test_is_query_name_in_database) {
auto query = getOsqueryScheduledQuery();
auto cf = Query("foobar", query);
auto encoded_qd = getSerializedQueryDataJSON();
auto status = db_->Put(kQueries, "foobar", encoded_qd.first);
auto status = setDatabaseValue(kQueries, "foobar", encoded_qd.first);
EXPECT_TRUE(status.ok());
// Now test that the query name exists.
EXPECT_TRUE(cf.isQueryNameInDatabase(db_));
EXPECT_TRUE(cf.isQueryNameInDatabase());
}
TEST_F(QueryTests, test_get_stored_query_names) {
auto query = getOsqueryScheduledQuery();
auto cf = Query("foobar", query);
auto encoded_qd = getSerializedQueryDataJSON();
auto status = db_->Put(kQueries, "foobar", encoded_qd.first);
auto status = setDatabaseValue(kQueries, "foobar", encoded_qd.first);
EXPECT_TRUE(status.ok());
// Stored query names is a factory method included alongside every query.
// It will include the set of query names with existing "previous" results.
auto names = cf.getStoredQueryNames(db_);
auto names = cf.getStoredQueryNames();
auto in_vector = std::find(names.begin(), names.end(), "foobar");
EXPECT_NE(in_vector, names.end());
}

105
osquery/database/tests/results_tests.cpp
View File

@ -136,111 +136,6 @@ TEST_F(ResultsTests, test_deserialize_query_log_item_json) {
EXPECT_EQ(output, results.second);
}
TEST_F(ResultsTests, test_serialize_distributed_query_request) {
DistributedQueryRequest r;
r.query = "foo";
r.id = "bar";
pt::ptree tree;
auto s = serializeDistributedQueryRequest(r, tree);
EXPECT_TRUE(s.ok());
EXPECT_EQ(tree.get<std::string>("query"), "foo");
EXPECT_EQ(tree.get<std::string>("id"), "bar");
}
TEST_F(ResultsTests, test_deserialize_distributed_query_request) {
pt::ptree tree;
tree.put<std::string>("query", "foo");
tree.put<std::string>("id", "bar");
DistributedQueryRequest r;
auto s = deserializeDistributedQueryRequest(tree, r);
EXPECT_TRUE(s.ok());
EXPECT_EQ(r.query, "foo");
EXPECT_EQ(r.id, "bar");
}
TEST_F(ResultsTests, test_deserialize_distributed_query_request_json) {
auto json =
"{"
" \"query\": \"foo\","
" \"id\": \"bar\""
"}";
DistributedQueryRequest r;
auto s = deserializeDistributedQueryRequestJSON(json, r);
EXPECT_TRUE(s.ok());
EXPECT_EQ(r.query, "foo");
EXPECT_EQ(r.id, "bar");
}
TEST_F(ResultsTests, test_serialize_distributed_query_result) {
DistributedQueryResult r;
r.request.query = "foo";
r.request.id = "bar";
Row r1;
r1["foo"] = "bar";
r.results = {r1};
pt::ptree tree;
auto s = serializeDistributedQueryResult(r, tree);
EXPECT_TRUE(s.ok());
EXPECT_EQ(tree.get<std::string>("request.query"), "foo");
EXPECT_EQ(tree.get<std::string>("request.id"), "bar");
auto& results = tree.get_child("results");
for (const auto& q : results) {
for (const auto& row : q.second) {
EXPECT_EQ(row.first, "foo");
EXPECT_EQ(q.second.get<std::string>(row.first), "bar");
}
}
}
TEST_F(ResultsTests, test_deserialize_distributed_query_result) {
pt::ptree request;
request.put<std::string>("id", "foo");
request.put<std::string>("query", "bar");
pt::ptree row;
row.put<std::string>("foo", "bar");
pt::ptree results;
results.push_back(std::make_pair("", row));
pt::ptree query_result;
query_result.put_child("request", request);
query_result.put_child("results", results);
DistributedQueryResult r;
auto s = deserializeDistributedQueryResult(query_result, r);
EXPECT_TRUE(s.ok());
EXPECT_EQ(r.request.id, "foo");
EXPECT_EQ(r.request.query, "bar");
EXPECT_EQ(r.results[0]["foo"], "bar");
}
TEST_F(ResultsTests, test_deserialize_distributed_query_result_json) {
auto json =
"{"
" \"request\": {"
" \"id\": \"foo\","
" \"query\": \"bar\""
" },"
" \"results\": ["
" {"
" \"foo\": \"bar\""
" }"
" ]"
"}";
DistributedQueryResult r;
auto s = deserializeDistributedQueryResultJSON(json, r);
EXPECT_TRUE(s.ok());
EXPECT_EQ(r.request.id, "foo");
EXPECT_EQ(r.request.query, "bar");
EXPECT_EQ(r.results[0]["foo"], "bar");
}
TEST_F(ResultsTests, test_adding_duplicate_rows_to_query_data) {
Row r1, r2, r3;
r1["foo"] = "bar";

4
osquery/devtools/shell.cpp
View File

@ -15,11 +15,11 @@
#include <signal.h>
#include <stdio.h>
#include <sys/time.h>
#include <sys/resource.h>
#include <sys/time.h>
#include <readline/readline.h>
#include <readline/history.h>
#include <readline/readline.h>
#include <sqlite3.h>

4
osquery/dispatcher/scheduler.cpp
View File

@ -47,8 +47,8 @@ inline SQL monitor(const std::string& name, const ScheduledQuery& query) {
}
}
// Always called while processes table is working.
Config::getInstance().recordQueryPerformance(
name, t1 - t0, size, r0[0], r1[0]);
Config::getInstance().recordQueryPerformance(name, t1 - t0, size, r0[0],
r1[0]);
}
return sql;
}

121
osquery/distributed/distributed.cpp
View File

@ -18,7 +18,6 @@
#include <osquery/logger.h>
#include <osquery/sql.h>
namespace pt = boost::property_tree;
namespace osquery {
@ -187,4 +186,124 @@ DistributedQueryRequest Distributed::popRequest() {
queries_.erase(queries_.begin());
return q;
}
Status serializeDistributedQueryRequest(const DistributedQueryRequest& r,
pt::ptree& tree) {
tree.put("query", r.query);
tree.put("id", r.id);
return Status(0, "OK");
}
Status serializeDistributedQueryRequestJSON(const DistributedQueryRequest& r,
std::string& json) {
pt::ptree tree;
auto s = serializeDistributedQueryRequest(r, tree);
if (!s.ok()) {
return s;
}
std::stringstream ss;
try {
pt::write_json(ss, tree, false);
} catch (const pt::ptree_error& e) {
return Status(1, "Error serializing JSON: " + std::string(e.what()));
}
json = ss.str();
return Status(0, "OK");
}
Status deserializeDistributedQueryRequest(const pt::ptree& tree,
DistributedQueryRequest& r) {
r.query = tree.get<std::string>("query", "");
r.id = tree.get<std::string>("id", "");
return Status(0, "OK");
}
Status deserializeDistributedQueryRequestJSON(const std::string& json,
DistributedQueryRequest& r) {
std::stringstream ss(json);
pt::ptree tree;
try {
pt::read_json(ss, tree);
} catch (const pt::ptree_error& e) {
return Status(1, "Error serializing JSON: " + std::string(e.what()));
}
return deserializeDistributedQueryRequest(tree, r);
}
/////////////////////////////////////////////////////////////////////////////
// DistributedQueryResult - small struct containing the results of a
// distributed query
/////////////////////////////////////////////////////////////////////////////
Status serializeDistributedQueryResult(const DistributedQueryResult& r,
pt::ptree& tree) {
pt::ptree request;
auto s = serializeDistributedQueryRequest(r.request, request);
if (!s.ok()) {
return s;
}
pt::ptree results;
s = serializeQueryData(r.results, results);
if (!s.ok()) {
return s;
}
tree.add_child("request", request);
tree.add_child("results", results);
return Status(0, "OK");
}
Status serializeDistributedQueryResultJSON(const DistributedQueryResult& r,
std::string& json) {
pt::ptree tree;
auto s = serializeDistributedQueryResult(r, tree);
if (!s.ok()) {
return s;
}
std::stringstream ss;
try {
pt::write_json(ss, tree, false);
} catch (const pt::ptree_error& e) {
return Status(1, "Error serializing JSON: " + std::string(e.what()));
}
json = ss.str();
return Status(0, "OK");
}
Status deserializeDistributedQueryResult(const pt::ptree& tree,
DistributedQueryResult& r) {
DistributedQueryRequest request;
auto s =
deserializeDistributedQueryRequest(tree.get_child("request"), request);
if (!s.ok()) {
return s;
}
QueryData results;
s = deserializeQueryData(tree.get_child("results"), results);
if (!s.ok()) {
return s;
}
r.request = request;
r.results = results;
return Status(0, "OK");
}
Status deserializeDistributedQueryResultJSON(const std::string& json,
DistributedQueryResult& r) {
std::stringstream ss(json);
pt::ptree tree;
try {
pt::read_json(ss, tree);
} catch (const pt::ptree_error& e) {
return Status(1, "Error serializing JSON: " + std::string(e.what()));
}
return deserializeDistributedQueryResult(tree, r);
}
}

105
osquery/distributed/tests/distributed_tests.cpp
View File

@ -65,6 +65,111 @@ class DistributedTests : public testing::Test {
std::string distributed_tls_write_endpoint_;
};
TEST_F(DistributedTests, test_serialize_distributed_query_request) {
DistributedQueryRequest r;
r.query = "foo";
r.id = "bar";
pt::ptree tree;
auto s = serializeDistributedQueryRequest(r, tree);
EXPECT_TRUE(s.ok());
EXPECT_EQ(tree.get<std::string>("query"), "foo");
EXPECT_EQ(tree.get<std::string>("id"), "bar");
}
TEST_F(DistributedTests, test_deserialize_distributed_query_request) {
pt::ptree tree;
tree.put<std::string>("query", "foo");
tree.put<std::string>("id", "bar");
DistributedQueryRequest r;
auto s = deserializeDistributedQueryRequest(tree, r);
EXPECT_TRUE(s.ok());
EXPECT_EQ(r.query, "foo");
EXPECT_EQ(r.id, "bar");
}
TEST_F(DistributedTests, test_deserialize_distributed_query_request_json) {
auto json =
"{"
" \"query\": \"foo\","
" \"id\": \"bar\""
"}";
DistributedQueryRequest r;
auto s = deserializeDistributedQueryRequestJSON(json, r);
EXPECT_TRUE(s.ok());
EXPECT_EQ(r.query, "foo");
EXPECT_EQ(r.id, "bar");
}
TEST_F(DistributedTests, test_serialize_distributed_query_result) {
DistributedQueryResult r;
r.request.query = "foo";
r.request.id = "bar";
Row r1;
r1["foo"] = "bar";
r.results = {r1};
pt::ptree tree;
auto s = serializeDistributedQueryResult(r, tree);
EXPECT_TRUE(s.ok());
EXPECT_EQ(tree.get<std::string>("request.query"), "foo");
EXPECT_EQ(tree.get<std::string>("request.id"), "bar");
auto& results = tree.get_child("results");
for (const auto& q : results) {
for (const auto& row : q.second) {
EXPECT_EQ(row.first, "foo");
EXPECT_EQ(q.second.get<std::string>(row.first), "bar");
}
}
}
TEST_F(DistributedTests, test_deserialize_distributed_query_result) {
pt::ptree request;
request.put<std::string>("id", "foo");
request.put<std::string>("query", "bar");
pt::ptree row;
row.put<std::string>("foo", "bar");
pt::ptree results;
results.push_back(std::make_pair("", row));
pt::ptree query_result;
query_result.put_child("request", request);
query_result.put_child("results", results);
DistributedQueryResult r;
auto s = deserializeDistributedQueryResult(query_result, r);
EXPECT_TRUE(s.ok());
EXPECT_EQ(r.request.id, "foo");
EXPECT_EQ(r.request.query, "bar");
EXPECT_EQ(r.results[0]["foo"], "bar");
}
TEST_F(DistributedTests, test_deserialize_distributed_query_result_json) {
auto json =
"{"
" \"request\": {"
" \"id\": \"foo\","
" \"query\": \"bar\""
" },"
" \"results\": ["
" {"
" \"foo\": \"bar\""
" }"
" ]"
"}";
DistributedQueryResult r;
auto s = deserializeDistributedQueryResultJSON(json, r);
EXPECT_TRUE(s.ok());
EXPECT_EQ(r.request.id, "foo");
EXPECT_EQ(r.request.query, "bar");
EXPECT_EQ(r.results[0]["foo"], "bar");
}
TEST_F(DistributedTests, test_workflow) {
auto dist = Distributed();
auto s = dist.pullUpdates();

12
osquery/events/benchmarks/events_benchmarks.cpp
View File

@ -58,9 +58,13 @@ class BenchmarkEventSubscriber
}
void clearRows() {
auto ee = expire_events_;
auto et = expire_time_;
expire_events_ = true;
expire_time_ = -1;
getIndexes(0, 0);
expire_events_ = ee;
expire_time_ = et;
}
void benchmarkGet(int low, int high) { auto results = get(low, high); }
@ -113,7 +117,7 @@ BENCHMARK(EVENTS_add_events);
static void EVENTS_retrieve_events(benchmark::State& state) {
auto sub = std::make_shared<BenchmarkEventSubscriber>();
for (int i = 0; i < 10000; i++) {
for (int i = 0; i < 1000; i++) {
sub->benchmarkAdd(i++);
}
@ -125,8 +129,8 @@ static void EVENTS_retrieve_events(benchmark::State& state) {
}
BENCHMARK(EVENTS_retrieve_events)
->ArgPair(0, 10)
->ArgPair(0, 50)
->ArgPair(0, 100)
->ArgPair(0, 500)
->ArgPair(0, 1000)
->ArgPair(0, 10000);
->ArgPair(0, 1000);
}

53
osquery/events/events.cpp
View File

@ -21,7 +21,6 @@
#include <osquery/logger.h>
#include "osquery/core/conversions.h"
#include "osquery/database/db_handle.h"
namespace osquery {
@ -92,9 +91,8 @@ QueryData EventSubscriberPlugin::genTable(QueryContext& context) {
// Store the optimize time such that it can be restored if the daemon is
// restarted.
auto db = DBHandle::getInstance();
auto index_key = "optimize." + dbNamespace();
db->Put(kEvents, index_key, std::to_string(optimize_time_));
setDatabaseValue(kEvents, index_key, std::to_string(optimize_time_));
}
return get(start, stop);
@ -135,7 +133,6 @@ void EventPublisherPlugin::fire(const EventContextRef& ec, EventTime time) {
std::set<std::string> EventSubscriberPlugin::getIndexes(EventTime start,
EventTime stop,
size_t list_key) {
auto db = DBHandle::getInstance();
auto index_key = "indexes." + dbNamespace();
std::set<std::string> indexes;
@ -152,7 +149,7 @@ std::set<std::string> EventSubscriberPlugin::getIndexes(EventTime start,
std::string time_list;
auto list_type = boost::lexical_cast<std::string>(size);
db->Get(kEvents, index_key + "." + list_type, time_list);
getDatabaseValue(kEvents, index_key + "." + list_type, time_list);
if (time_list.empty()) {
// No events in this binning size.
return indexes;
@ -240,7 +237,6 @@ std::set<std::string> EventSubscriberPlugin::getIndexes(EventTime start,
void EventSubscriberPlugin::expireRecords(const std::string& list_type,
const std::string& index,
bool all) {
auto db = DBHandle::getInstance();
auto record_key = "records." + dbNamespace();
auto data_key = "data." + dbNamespace();
@ -250,7 +246,7 @@ void EventSubscriberPlugin::expireRecords(const std::string& list_type,
auto expired_records = getRecords({list_type + "." + index});
for (const auto& record : expired_records) {
if (all) {
db->Delete(kEvents, data_key + "." + record.first);
deleteDatabaseValue(kEvents, data_key + "." + record.first);
} else if (record.second > expire_time_) {
persisting_records.push_back(record.first + ":" +
std::to_string(record.second));
@ -259,10 +255,11 @@ void EventSubscriberPlugin::expireRecords(const std::string& list_type,
// Either drop or overwrite the record list.
if (all) {
db->Delete(kEvents, record_key + "." + list_type + "." + index);
deleteDatabaseValue(kEvents, record_key + "." + list_type + "." + index);
} else {
auto new_records = boost::algorithm::join(persisting_records, ",");
db->Put(kEvents, record_key + "." + list_type + "." + index, new_records);
setDatabaseValue(
kEvents, record_key + "." + list_type + "." + index, new_records);
}
}
@ -270,7 +267,6 @@ void EventSubscriberPlugin::expireIndexes(
const std::string& list_type,
const std::vector<std::string>& indexes,
const std::vector<std::string>& expirations) {
auto db = DBHandle::getInstance();
auto index_key = "indexes." + dbNamespace();
// Construct a mutable list of persisting indexes to rewrite as records.
@ -285,16 +281,15 @@ void EventSubscriberPlugin::expireIndexes(
// Update the list of indexes with the non-expired indexes.
auto new_indexes = boost::algorithm::join(persisting_indexes, ",");
db->Put(kEvents, index_key + "." + list_type, new_indexes);
setDatabaseValue(kEvents, index_key + "." + list_type, new_indexes);
}
void EventSubscriberPlugin::expireCheck() {
auto db = DBHandle::getInstance();
auto data_key = "data." + dbNamespace();
auto eid_key = "eid." + dbNamespace();
std::vector<std::string> keys;
db->ScanPrefix(kEvents, keys, data_key);
scanDatabaseKeys(kEvents, keys, data_key);
if (keys.size() <= FLAGS_events_max) {
return;
}
@ -305,13 +300,13 @@ void EventSubscriberPlugin::expireCheck() {
// Inspect the N-FLAGS_events_max -th event's value and expire before the
// time within the content.
std::string last_key;
db->Get(kEvents, eid_key, last_key);
getDatabaseValue(kEvents, eid_key, last_key);
// The EID is the next-index.
size_t max_key = boost::lexical_cast<size_t>(last_key) - FLAGS_events_max - 1;
// Convert the key index into a time using the content.
std::string content;
db->Get(kEvents, data_key + "." + std::to_string(max_key), content);
getDatabaseValue(kEvents, data_key + "." + std::to_string(max_key), content);
// Decode the value into a row structure to extract the time.
Row r;
@ -332,7 +327,6 @@ void EventSubscriberPlugin::expireCheck() {
std::vector<EventRecord> EventSubscriberPlugin::getRecords(
const std::set<std::string>& indexes) {
auto db = DBHandle::getInstance();
auto record_key = "records." + dbNamespace();
std::vector<EventRecord> records;
@ -340,7 +334,7 @@ std::vector<EventRecord> EventSubscriberPlugin::getRecords(
std::vector<std::string> bin_records;
{
std::string record_value;
db->Get(kEvents, record_key + "." + index, record_value);
getDatabaseValue(kEvents, record_key + "." + index, record_value);
if (record_value.empty()) {
// There are actually no events in this bin, interesting error case.
continue;
@ -364,7 +358,6 @@ std::vector<EventRecord> EventSubscriberPlugin::getRecords(
Status EventSubscriberPlugin::recordEvent(EventID& eid, EventTime time) {
Status status;
auto db = DBHandle::getInstance();
std::string time_value = boost::lexical_cast<std::string>(time);
// The record is identified by the event type then module name.
@ -386,27 +379,29 @@ Status EventSubscriberPlugin::recordEvent(EventID& eid, EventTime time) {
boost::lock_guard<boost::mutex> lock(event_record_lock_);
// Append the record (eid, unix_time) to the list bin.
std::string record_value;
status = db->Get(
status = getDatabaseValue(
kEvents, record_key + "." + list_key + "." + list_id, record_value);
if (record_value.length() == 0) {
// This is a new list_id for list_key, append the ID to the indirect
// lookup for this list_key.
std::string index_value;
status = db->Get(kEvents, index_key + "." + list_key, index_value);
status =
getDatabaseValue(kEvents, index_key + "." + list_key, index_value);
if (index_value.length() == 0) {
// A new index.
index_value = list_id;
} else {
index_value += "," + list_id;
}
status = db->Put(kEvents, index_key + "." + list_key, index_value);
status =
setDatabaseValue(kEvents, index_key + "." + list_key, index_value);
record_value = eid + ":" + time_value;
} else {
// Tokenize a record using ',' and the EID/time using ':'.
record_value += "," + eid + ":" + time_value;
}
status = db->Put(
status = setDatabaseValue(
kEvents, record_key + "." + list_key + "." + list_id, record_value);
if (!status.ok()) {
LOG(ERROR) << "Could not put Event Record key: " << record_key;
@ -419,7 +414,6 @@ Status EventSubscriberPlugin::recordEvent(EventID& eid, EventTime time) {
EventID EventSubscriberPlugin::getEventID() {
Status status;
auto db = DBHandle::getInstance();
// First get an event ID from the meta key.
std::string eid_key = "eid." + dbNamespace();
std::string last_eid_value;
@ -427,14 +421,14 @@ EventID EventSubscriberPlugin::getEventID() {
{
boost::lock_guard<boost::mutex> lock(event_id_lock_);
status = db->Get(kEvents, eid_key, last_eid_value);
status = getDatabaseValue(kEvents, eid_key, last_eid_value);
if (!status.ok() || last_eid_value.empty()) {
last_eid_value = "0";
}
last_eid_ = boost::lexical_cast<size_t>(last_eid_value) + 1;
eid_value = boost::lexical_cast<std::string>(last_eid_);
status = db->Put(kEvents, eid_key, eid_value);
status = setDatabaseValue(kEvents, eid_key, eid_value);
}
if (!status.ok()) {
@ -448,7 +442,6 @@ QueryData EventSubscriberPlugin::get(EventTime start, EventTime stop) {
QueryData results;
// Get the records for this time range.
auto db = DBHandle::getInstance();
auto indexes = getIndexes(start, stop);
auto records = getRecords(indexes);
std::string events_key = "data." + dbNamespace();
@ -464,7 +457,7 @@ QueryData EventSubscriberPlugin::get(EventTime start, EventTime stop) {
std::string data_value;
for (const auto& record : mapped_records) {
Row r;
auto status = db->Get(kEvents, record, data_value);
auto status = getDatabaseValue(kEvents, record, data_value);
if (data_value.length() == 0) {
// There is no record here, interesting error case.
continue;
@ -485,7 +478,6 @@ QueryData EventSubscriberPlugin::get(EventTime start, EventTime stop) {
}
Status EventSubscriberPlugin::add(Row& r, EventTime event_time) {
auto db = DBHandle::getInstance();
// Get and increment the EID for this module.
EventID eid = getEventID();
// Without encouraging a missing event time, do not support a 0-time.
@ -506,7 +498,7 @@ Status EventSubscriberPlugin::add(Row& r, EventTime event_time) {
// Store the event data.
std::string event_key = "data." + dbNamespace() + "." + eid;
status = db->Put(kEvents, event_key, data);
status = setDatabaseValue(kEvents, event_key, data);
// Record the event in the indexing bins, using the index time.
recordEvent(eid, event_time);
return status;
@ -692,10 +684,9 @@ Status EventFactory::registerEventSubscriber(const PluginRef& sub) {
// Restore optimize times for a daemon.
if (kToolType == OSQUERY_TOOL_DAEMON && FLAGS_events_optimize) {
auto db = DBHandle::getInstance();
auto index_key = "optimize." + specialized_sub->dbNamespace();
std::string content;
if (db->Get(kEvents, index_key, content)) {
if (getDatabaseValue(kEvents, index_key, content)) {
long long optimize_time = 0;
safeStrtoll(content, 10, optimize_time);
specialized_sub->optimize_time_ = static_cast<EventTime>(optimize_time);

6
osquery/events/tests/events_database_tests.cpp
View File

@ -19,8 +19,6 @@
#include <osquery/flags.h>
#include <osquery/tables.h>
#include "osquery/database/db_handle.h"
namespace osquery {
DECLARE_uint64(events_expiry);
@ -209,8 +207,8 @@ TEST_F(EventsDatabaseTests, test_gentable) {
// The optimize time should have been written to the database.
// It should be the same as the current (relative) optimize time.
std::string content;
getDatabaseValue(
"events", "optimize.DBFakePublisher.DBFakeSubscriber", content);
getDatabaseValue("events", "optimize.DBFakePublisher.DBFakeSubscriber",
content);
EXPECT_EQ(std::to_string(sub->optimize_time_), content);
keys.clear();

30
osquery/extensions/interface.cpp
View File

@ -44,7 +44,6 @@ void ExtensionHandler::call(ExtensionResponse& _return,
_return.status.code = status.getCode();
_return.status.message = status.getMessage();
_return.status.uuid = uuid_;
if (status.ok()) {
for (const auto& response_item : response) {
// Translate a PluginResponse to an ExtensionPluginResponse.
@ -176,18 +175,18 @@ bool ExtensionManagerHandler::exists(const std::string& name) {
ExtensionRunnerCore::~ExtensionRunnerCore() { remove(path_); }
void ExtensionRunnerCore::stop() {
boost::lock_guard<boost::mutex> lock(service_start_);
service_stopping_ = true;
if (transport_ != nullptr) {
transport_->interrupt();
{
std::unique_lock<std::mutex> lock(service_start_);
service_stopping_ = true;
if (transport_ != nullptr) {
transport_->interrupt();
transport_->interruptChildren();
}
}
{
// In most cases the service thread has started before the stop request.
boost::lock_guard<boost::mutex> lock(service_run_);
if (server_ != nullptr) {
server_->stop();
}
// In most cases the service thread has started before the stop request.
if (server_ != nullptr) {
server_->stop();
}
}
@ -202,7 +201,7 @@ inline void removeStalePaths(const std::string& manager) {
void ExtensionRunnerCore::startServer(TProcessorRef processor) {
{
boost::lock_guard<boost::mutex> lock(service_start_);
std::unique_lock<std::mutex> lock(service_start_);
// A request to stop the service may occur before the thread starts.
if (service_stopping_) {
return;
@ -221,10 +220,7 @@ void ExtensionRunnerCore::startServer(TProcessorRef processor) {
processor, transport_, transport_fac, protocol_fac));
}
{
boost::lock_guard<boost::mutex> lock(service_run_);
server_->serve();
}
server_->serve();
}
void ExtensionRunner::start() {
@ -243,7 +239,7 @@ void ExtensionRunner::start() {
ExtensionManagerRunner::~ExtensionManagerRunner() {
// Only attempt to remove stale paths if the server was started.
boost::lock_guard<boost::mutex> lock(service_start_);
std::unique_lock<std::mutex> lock(service_start_);
if (server_ != nullptr) {
removeStalePaths(path_);
}

6
osquery/extensions/interface.h
View File

@ -261,11 +261,7 @@ class ExtensionRunnerCore : public InternalRunnable {
TThreadedServerRef server_{nullptr};
/// Protect the service start and stop, this mutex protects server creation.
boost::mutex service_start_;
private:
/// Protect the service start and stop, this mutex protects the transport.
boost::mutex service_run_;
std::mutex service_start_;
/// Record a dispatcher's request to stop the service.
bool service_stopping_{false};

19
osquery/logger/CMakeLists.txt
View File

@ -1,19 +1,16 @@
ADD_OSQUERY_LIBRARY(TRUE osquery_logger
logger.cpp
)
file(GLOB OSQUERY_LOGGER "*.cpp")
ADD_OSQUERY_LIBRARY(TRUE osquery_logger ${OSQUERY_LOGGER})
add_dependencies(osquery_logger libglog)
ADD_OSQUERY_LIBRARY(FALSE osquery_logger_plugins
plugins/filesystem.cpp
plugins/tls.cpp
plugins/syslog.cpp
)
file(GLOB OSQUERY_LOGGER_PLUGINS "plugins/*.cpp")
ADD_OSQUERY_LIBRARY(FALSE osquery_logger_plugins ${OSQUERY_LOGGER_PLUGINS})
file(GLOB OSQUERY_LOGGER_TESTS "tests/*.cpp")
ADD_OSQUERY_TEST(TRUE ${OSQUERY_LOGGER_TESTS})
# Keep the logger testing in the additional to test filesystem logging.
# There is a significant difference between the Glog-backed filesystem plugin
# and other, which use a Glog sink. They must be tested in tandem.
file(GLOB OSQUERY_LOGGER_TESTS "tests/*.cpp")
ADD_OSQUERY_TEST(TRUE ${OSQUERY_LOGGER_TESTS})
file(GLOB OSQUERY_LOGGER_PLUGIN_TESTS "plugins/tests/*.cpp")
ADD_OSQUERY_TEST(FALSE ${OSQUERY_LOGGER_PLUGIN_TESTS})

27
osquery/logger/plugins/tls.cpp
View File

@ -8,21 +8,20 @@
*
*/
#include <vector>
#include <string>
#include <vector>
#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/json_parser.hpp>
#include <boost/property_tree/ptree.hpp>
#include <osquery/enroll.h>
#include <osquery/flags.h>
#include <osquery/registry.h>
#include "osquery/remote/requests.h"
#include "osquery/remote/transports/tls.h"
#include "osquery/remote/serializers/json.h"
#include "osquery/remote/transports/tls.h"
#include "osquery/remote/utility.h"
#include "osquery/database/db_handle.h"
#include "osquery/logger/plugins/tls.h"
@ -144,8 +143,7 @@ Status TLSLogForwarderRunner::send(std::vector<std::string>& log_data,
// Read each logged line into JSON and populate a list of lines.
// The result list will use the 'data' key.
pt::ptree children;
iterate(log_data,
([&children](std::string& item) {
iterate(log_data, ([&children](std::string& item) {
pt::ptree child;
try {
std::stringstream input;
@ -169,21 +167,16 @@ Status TLSLogForwarderRunner::send(std::vector<std::string>& log_data,
}
void TLSLogForwarderRunner::check() {
// Instead of using the 'help' database API, prefer to interact with the
// DBHandle directly for additional performance.
auto handle = DBHandle::getInstance();
// Get a list of all the buffered log items, with a max of 1024 lines.
std::vector<std::string> indexes;
auto status = handle->Scan(kLogs, indexes, kTLSMaxLogLines);
auto status = scanDatabaseKeys(kLogs, indexes, kTLSMaxLogLines);
// For each index, accumulate the log line into the result or status set.
std::vector<std::string> results, statuses;
iterate(indexes,
([&handle, &results, &statuses](std::string& index) {
iterate(indexes, ([&results, &statuses](std::string& index) {
std::string value;
auto& target = ((index.at(0) == 'r') ? results : statuses);
if (handle->Get(kLogs, index, value)) {
if (getDatabaseValue(kLogs, index, value)) {
// Enforce a max log line size for TLS logging.
if (value.size() > FLAGS_logger_tls_max) {
LOG(WARNING) << "Line exceeds TLS logger max: " << value.size();
@ -201,8 +194,7 @@ void TLSLogForwarderRunner::check() {
<< status.getMessage() << ")";
} else {
// Clear the results logs once they were sent.
iterate(indexes,
([&results](std::string& index) {
iterate(indexes, ([&results](std::string& index) {
if (index.at(0) != 'r') {
return;
}
@ -218,8 +210,7 @@ void TLSLogForwarderRunner::check() {
<< status.getMessage() << ")";
} else {
// Clear the status logs once they were sent.
iterate(indexes,
([&results](std::string& index) {
iterate(indexes, ([&results](std::string& index) {
if (index.at(0) != 's') {
return;
}

5
osquery/main/tests.cpp
View File

@ -19,5 +19,8 @@ int main(int argc, char* argv[]) {
testing::InitGoogleTest(&argc, argv);
// Optionally enable Goggle Logging
// google::InitGoogleLogging(argv[0]);
return RUN_ALL_TESTS();
auto result = RUN_ALL_TESTS();
osquery::shutdownTesting();
return result;
}

38
osquery/sql/benchmarks/sql_benchmarks.cpp
View File

@ -62,6 +62,40 @@ static void SQL_virtual_table_internal(benchmark::State& state) {
BENCHMARK(SQL_virtual_table_internal);
static void SQL_virtual_table_internal_global(benchmark::State& state) {
Registry::add<BenchmarkTablePlugin>("table", "benchmark");
PluginResponse res;
Registry::call("table", "benchmark", {{"action", "columns"}}, res);
while (state.KeepRunning()) {
// Get a connection to the persistent database.
auto dbc = SQLiteDBManager::get();
attachTableInternal("benchmark", columnDefinition(res), dbc);
QueryData results;
queryInternal("select * from benchmark", results, dbc->db());
}
}
BENCHMARK(SQL_virtual_table_internal_global);
static void SQL_virtual_table_internal_unique(benchmark::State& state) {
Registry::add<BenchmarkTablePlugin>("table", "benchmark");
PluginResponse res;
Registry::call("table", "benchmark", {{"action", "columns"}}, res);
while (state.KeepRunning()) {
// Get a new database connection (to a unique database).
auto dbc = SQLiteDBManager::getUnique();
attachTableInternal("benchmark", columnDefinition(res), dbc);
QueryData results;
queryInternal("select * from benchmark", results, dbc->db());
}
}
BENCHMARK(SQL_virtual_table_internal_unique);
class BenchmarkLongTablePlugin : public TablePlugin {
private:
TableColumns columns() const {
@ -83,7 +117,7 @@ static void SQL_virtual_table_internal_long(benchmark::State& state) {
Registry::call("table", "long_benchmark", {{"action", "columns"}}, res);
// Attach a sample virtual table.
auto dbc = SQLiteDBManager::get();
auto dbc = SQLiteDBManager::getUnique();
attachTableInternal("long_benchmark", columnDefinition(res), dbc);
while (state.KeepRunning()) {
@ -123,7 +157,7 @@ static void SQL_virtual_table_internal_wide(benchmark::State& state) {
Registry::call("table", "wide_benchmark", {{"action", "columns"}}, res);
// Attach a sample virtual table.
auto dbc = SQLiteDBManager::get();
auto dbc = SQLiteDBManager::getUnique();
attachTableInternal("wide_benchmark", columnDefinition(res), dbc);
while (state.KeepRunning()) {

26
osquery/sql/sqlite_util.cpp
View File

@ -9,8 +9,8 @@
*/
#include <osquery/core.h>
#include <osquery/logger.h>
#include <osquery/flags.h>
#include <osquery/logger.h>
#include <osquery/sql.h>
#include "osquery/sql/sqlite_util.h"
@ -49,6 +49,13 @@ const std::map<int, std::string> kSQLiteReturnCodes = {
{101, "SQLITE_DONE"},
};
const std::map<std::string, std::string> kMemoryDBSettings = {
{"synchronous", "OFF"}, {"count_changes", "OFF"},
{"default_temp_store", "0"}, {"auto_vacuum", "FULL"},
{"journal_mode", "OFF"}, {"cache_size", "0"},
{"page_count", "0"},
};
#define OpComparator(x) \
{ x, QueryPlanner::Opcode(OpReg::P2, INTEGER_TYPE) }
#define Arithmetic(x) \
@ -140,9 +147,19 @@ SQLiteDBInstance::SQLiteDBInstance(sqlite3*& db, std::mutex& mtx)
}
}
static inline void openOptimized(sqlite3*& db) {
sqlite3_open(":memory:", &db);
std::string settings;
for (const auto& setting : kMemoryDBSettings) {
settings += "PRAGMA " + setting.first + "=" + setting.second + "; ";
}
sqlite3_exec(db, settings.c_str(), nullptr, nullptr, nullptr);
}
void SQLiteDBInstance::init() {
primary_ = false;
sqlite3_open(":memory:", &db_);
openOptimized(db_);
}
void SQLiteDBInstance::addAffectedTable(VirtualTableContent* table) {
@ -175,7 +192,7 @@ SQLiteDBInstance::~SQLiteDBInstance() {
}
SQLiteDBManager::SQLiteDBManager() : db_(nullptr) {
sqlite3_soft_heap_limit64(SQLITE_SOFT_HEAP_LIMIT);
sqlite3_soft_heap_limit64(1);
setDisabledTables(Flag::getValue("disable_tables"));
}
@ -202,7 +219,7 @@ SQLiteDBInstanceRef SQLiteDBManager::getConnection(bool primary) {
if (self.db_ == nullptr) {
// Create primary SQLite DB instance.
sqlite3_open(":memory:", &self.db_);
openOptimized(self.db_);
self.connection_ = SQLiteDBInstanceRef(new SQLiteDBInstance(self.db_));
attachVirtualTables(self.connection_);
}
@ -301,7 +318,6 @@ Status queryInternal(const std::string& q, QueryData& results, sqlite3* db) {
sqlite3_free(err);
return Status(1, "Error running query: " + error_string);
}
return Status(0, "OK");
}

6
osquery/sql/virtual_table.h
View File

@ -10,6 +10,8 @@
#pragma once
#include <boost/noncopyable.hpp>
#include <osquery/tables.h>
#include "osquery/core/conversions.h"
@ -22,7 +24,7 @@ namespace osquery {
*
* Only used in the SQLite virtual table module methods.
*/
struct BaseCursor {
struct BaseCursor : private boost::noncopyable {
/// SQLite virtual table cursor.
sqlite3_vtab_cursor base;
/// Track cursors for optional planner output.
@ -41,7 +43,7 @@ struct BaseCursor {
* Only used in the SQLite virtual table module methods.
* This adds each table plugin class to the state tracking in SQLite.
*/
struct VirtualTable {
struct VirtualTable : private boost::noncopyable {
/// The SQLite-provided virtual table structure.
sqlite3_vtab base;

6
osquery/tables/system/linux/processes.cpp
View File

@ -8,8 +8,8 @@
*
*/
#include <string>
#include <map>
#include <string>
#include <stdlib.h>
#include <unistd.h>
@ -17,9 +17,9 @@
#include <boost/algorithm/string/trim.hpp>
#include <osquery/core.h>
#include <osquery/tables.h>
#include <osquery/filesystem.h>
#include <osquery/logger.h>
#include <osquery/tables.h>
#include "osquery/core/conversions.h"
@ -143,7 +143,7 @@ void genProcessMap(const std::string& pid, QueryData& results) {
// BSS with name in pathname.
r["pseudo"] = (fields[4] == "0" && !r["path"].empty()) ? "1" : "0";
results.push_back(r);
results.push_back(std::move(r));
}
}

2
third-party

@ -1 +1 @@
Subproject commit bd9d658ff4fbc642384376663c1f9ea4f66be7be
Subproject commit 467917469b31f567f5f6a755641cfe8179a68394

30
tools/tests/stress.py
View File

@ -87,15 +87,21 @@ def audit(args):
def single(args):
start_time = time.time()
proc = subprocess.Popen(args,
shell=True,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
if ARGS.verbose:
proc = subprocess.Popen(args, shell=True)
else:
proc = subprocess.Popen(args,
shell=True,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
if ARGS.verbose:
print("PID: %d" % (proc.pid))
stdout, stderr = proc.communicate()
end_time = time.time() - start_time
if proc.returncode is not 0:
print(stdout)
print(stderr)
if not ARGS.verbose:
print(stdout)
print(stderr)
print("%s Test failed. (total %6.4fs)" % (
red("FAILED"), end_time))
sys.exit(proc.returncode)
@ -129,14 +135,16 @@ if __name__ == "__main__":
help="Arguments to pass to test binary")
parser.add_argument("--stat", action="store_true", default=False,
help="Only print numerical values")
parser.add_argument("--verbose", action="store_true", default=False,
help="Do not consume stderr/stdout")
parser.add_argument("run", nargs="?", help="Run specific test binary")
args = parser.parse_args()
ARGS = parser.parse_args()
# A baseline was requested, first run baselines then normal.
if args.baseline:
if ARGS.baseline:
print("Running baseline tests...")
stress(vars(args))
args.baseline = False
stress(vars(ARGS))
ARGS.baseline = False
print("Finished. Running tests...")
stress(vars(args))
stress(vars(ARGS))

6
tools/tests/test_base.py
View File

@ -165,8 +165,10 @@ class ProcRunner(object):
pid = 0
try:
if self.silent:
self.proc = subprocess.Popen([self.path] + self.args,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
self.proc = subprocess.Popen(
[self.path] + self.args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
else:
self.proc = subprocess.Popen([self.path] + self.args)
pid = self.proc.pid

1
tools/tests/test_extensions.py
View File

@ -303,6 +303,7 @@ class ExtensionTests(test_base.ProcessGenerator, unittest.TestCase):
# Now start a daemon
daemon = self._run_daemon({
"disable_watchdog": True,
"verbose": True,
"extensions_timeout": EXTENSION_TIMEOUT,
"extensions_socket": extension.options["extensions_socket"],
})

6
tools/tests/test_osqueryi.py
View File

@ -69,7 +69,7 @@ class OsqueryiTest(unittest.TestCase):
self.assertEqual(proc.proc.poll(), 0)
@test_base.flaky
def test_config_check_failure(self):
def test_config_check_failure_invalid_path(self):
'''Test that a missing config fails'''
proc = test_base.TimeoutRunner([
self.binary,
@ -83,6 +83,8 @@ class OsqueryiTest(unittest.TestCase):
print(proc.stderr)
self.assertEqual(proc.proc.poll(), 1)
@test_base.flaky
def test_config_check_failure_valid_path(self):
# Now with a valid path, but invalid content.
proc = test_base.TimeoutRunner([
self.binary,
@ -94,6 +96,8 @@ class OsqueryiTest(unittest.TestCase):
self.assertEqual(proc.proc.poll(), 1)
self.assertNotEqual(proc.stderr, "")
@test_base.flaky
def test_config_check_failure_missing_plugin(self):
# Finally with a missing config plugin
proc = test_base.TimeoutRunner([
self.binary,