diff --git a/CMakeLists.txt b/CMakeLists.txt index f39e9f8a..eccb733a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -139,12 +139,22 @@ set(OSQUERY_REQUIRE_RUNTIMES if(DEFINED ENV{DEBUG}) set(DEBUG TRUE) set(CMAKE_BUILD_TYPE "Debug") - add_compile_options(-g -O0) + add_compile_options( + -g + -O0 + -fstandalone-debug + ) add_definitions(-DDEBUG) WARNING_LOG("Setting DEBUG build") elseif(DEFINED ENV{SANITIZE}) # make sanitize (cannot make debug sanitize) - add_compile_options(-g -O1 -fno-omit-frame-pointer -fno-optimize-sibling-calls) + add_compile_options( + -g + -O1 + -fstandalone-debug + -fno-omit-frame-pointer + -fno-optimize-sibling-calls + ) add_definitions(-DNDEBUG) if(DEFINED ENV{SANITIZE_THREAD}) add_compile_options(-fsanitize=thread) @@ -347,12 +357,12 @@ endif() find_package(Glog REQUIRED) find_package(Gflags REQUIRED) find_package(Gtest REQUIRED) -find_package(RocksDB REQUIRED) find_package(Thrift 0.9.3 REQUIRED) # If using the RocksDB LITE version our code must also define ROCKSDB_LITE=1 -if(ROCKSDB_LITE_FOUND) - add_definitions(-DROCKSDB_LITE=1) +if(NOT DEFINED ENV{SKIP_ROCKSDB}) + set(ROCKSDB TRUE) + find_package(RocksDB REQUIRED) endif() # Python is used for table spec generation and formating. diff --git a/include/osquery/database.h b/include/osquery/database.h index d3a88faa..8a30d5ef 100644 --- a/include/osquery/database.h +++ b/include/osquery/database.h @@ -10,6 +10,7 @@ #pragma once +#include #include #include #include @@ -21,6 +22,18 @@ namespace osquery { +/** + * @brief A list of supported backing storage categories: called domains. + * + * RocksDB has a concept of "column families" which are kind of like tables + * in other databases. kDomainds is populated with a list of all column + * families. If a string exists in kDomains, it's a column family in the + * database. + * + * For SQLite-backed storage these are tables using a keyed index. + */ +extern const std::vector kDomains; + /** * @brief A backing storage domain name, used for key/value based storage. * @@ -46,9 +59,6 @@ extern const std::string kEvents; */ extern const std::string kLogs; -/// An ordered list of column type names. -extern const std::vector kDomains; - ///////////////////////////////////////////////////////////////////////////// // Row ///////////////////////////////////////////////////////////////////////////// @@ -394,142 +404,6 @@ Status serializeQueryLogItemAsEvents(const QueryLogItem& item, Status serializeQueryLogItemAsEventsJSON(const QueryLogItem& i, std::vector& items); -///////////////////////////////////////////////////////////////////////////// -// DistributedQueryRequest -///////////////////////////////////////////////////////////////////////////// - -/** - * @brief Small struct containing the query and ID information for a - * distributed query - */ -struct DistributedQueryRequest { - public: - explicit DistributedQueryRequest() {} - explicit DistributedQueryRequest(const std::string& q, const std::string& i) - : query(q), id(i) {} - - /// equals operator - bool operator==(const DistributedQueryRequest& comp) const { - return (comp.query == query) && (comp.id == id); - } - - std::string query; - std::string id; -}; - -/** - * @brief Serialize a DistributedQueryRequest into a property tree - * - * @param r the DistributedQueryRequest to serialize - * @param tree the output property tree - * - * @return Status indicating the success or failure of the operation - */ -Status serializeDistributedQueryRequest(const DistributedQueryRequest& r, - boost::property_tree::ptree& tree); - -/** - * @brief Serialize a DistributedQueryRequest object into a JSON string - * - * @param r the DistributedQueryRequest to serialize - * @param json the output JSON string - * - * @return Status indicating the success or failure of the operation - */ -Status serializeDistributedQueryRequestJSON(const DistributedQueryRequest& r, - std::string& json); - -/** - * @brief Deserialize a DistributedQueryRequest object from a property tree - * - * @param tree the input property tree - * @param r the output DistributedQueryRequest structure - * - * @return Status indicating the success or failure of the operation - */ -Status deserializeDistributedQueryRequest( - const boost::property_tree::ptree& tree, DistributedQueryRequest& r); - -/** - * @brief Deserialize a DistributedQueryRequest object from a JSON string - * - * @param json the input JSON string - * @param r the output DistributedQueryRequest structure - * - * @return Status indicating the success or failure of the operation - */ -Status deserializeDistributedQueryRequestJSON(const std::string& json, - DistributedQueryRequest& r); - -///////////////////////////////////////////////////////////////////////////// -// DistributedQueryResult -///////////////////////////////////////////////////////////////////////////// - -/** - * @brief Small struct containing the results of a distributed query - */ -struct DistributedQueryResult { - public: - explicit DistributedQueryResult() {} - explicit DistributedQueryResult(const DistributedQueryRequest& req, - const QueryData& res) - : request(req), results(res) {} - - /// equals operator - bool operator==(const DistributedQueryResult& comp) const { - return (comp.request == request) && (comp.results == results); - } - - DistributedQueryRequest request; - QueryData results; -}; - -/** - * @brief Serialize a DistributedQueryResult into a property tree - * - * @param r the DistributedQueryResult to serialize - * @param tree the output property tree - * - * @return Status indicating the success or failure of the operation - */ -Status serializeDistributedQueryResult(const DistributedQueryResult& r, - boost::property_tree::ptree& tree); - -/** - * @brief Serialize a DistributedQueryResult object into a JSON string - * - * @param r the DistributedQueryResult to serialize - * @param json the output JSON string - * - * @return Status indicating the success or failure of the operation - */ -Status serializeDistributedQueryResultJSON(const DistributedQueryResult& r, - std::string& json); - -/** - * @brief Deserialize a DistributedQueryResult object from a property tree - * - * @param tree the input property tree - * @param r the output DistributedQueryResult structure - * - * @return Status indicating the success or failure of the operation - */ -Status deserializeDistributedQueryResult( - const boost::property_tree::ptree& tree, DistributedQueryResult& r); - -/** - * @brief Deserialize a DistributedQueryResult object from a JSON string - * - * @param json the input JSON string - * @param r the output DistributedQueryResult structure - * - * @return Status indicating the success or failure of the operation - */ -Status deserializeDistributedQueryResultJSON(const std::string& json, - DistributedQueryResult& r); - -///////////////////////////////////////////////////////////////////////////// - /** * @brief An osquery backing storage (database) type that persists executions. * @@ -543,7 +417,7 @@ Status deserializeDistributedQueryResultJSON(const std::string& json, * to removing RocksDB as a dependency for the osquery SDK. */ class DatabasePlugin : public Plugin { - protected: + public: /** * @brief Perform a domain and key lookup from the backing store. * @@ -580,15 +454,79 @@ class DatabasePlugin : public Plugin { /// Data removal method. virtual Status remove(const std::string& domain, const std::string& k) = 0; - /// Key/index lookup method. virtual Status scan(const std::string& domain, std::vector& results, + const std::string& prefix, size_t max = 0) const { return Status(0, "Not used"); } + /** + * @brief Shutdown the database and release initialization resources. + * + * Assume that a plugin may override ::tearDown and choose to close resources + * when the registry is stopping. Most plugins will implement a mutex around + * initialization and destruction and assume ::setUp and ::tearDown will + * dictate the flow in most situations. + */ + virtual ~DatabasePlugin() {} + + /** + * @brief Support the registry calling API for extensions. + * + * The database plugin "fast-calls" directly to local plugins. + * Extensions cannot use an extension-local backing store so their requests + * are routed like all other plugins. + */ + Status call(const PluginRequest& request, PluginResponse& response) override; + public: - Status call(const PluginRequest& request, PluginResponse& response); + /// Database-specific workflow: reset the originally request instance. + virtual Status reset() final; + + /// Database-specific workflow: perform an initialize, then reset. + bool checkDB(); + + /// Require all DBHandle accesses to open a read and write handle. + static void setRequireWrite(bool rw) { kDBHandleOptionRequireWrite = rw; } + + /// Allow DBHandle creations. + static void setAllowOpen(bool ao) { kDBHandleOptionAllowOpen = ao; } + + public: + /// Control availability of the RocksDB handle (default false). + static bool kDBHandleOptionAllowOpen; + + /// The database must be opened in a R/W mode (default false). + static bool kDBHandleOptionRequireWrite; + + /// A queryable mutex around database sanity checking. + static std::atomic kCheckingDB; + + public: + /** + * @brief Allow the initializer to check the active database plugin. + * + * Unlink the initializer's ::initActivePlugin helper method, the database + * plugin should always be within the core. There is no need to discover + * the active plugin via the registry or extensions API. + * + * The database should setUp in preparation for accesses. + */ + static bool initPlugin(); + + /// Allow shutdown before exit. + static void shutdown(); + + protected: + /// The database was opened in a ReadOnly mode. + bool read_only_{false}; + + /// True if the database was started in an in-memory mode. + bool in_memory_{false}; + + /// Original requested path on disk. + std::string path_; }; /** @@ -631,6 +569,12 @@ Status scanDatabaseKeys(const std::string& domain, std::vector& keys, size_t max = 0); +/// Get a list of keys for a given domain. +Status scanDatabaseKeys(const std::string& domain, + std::vector& keys, + const std::string& prefix, + size_t max = 0); + /// Allow callers to scan each column family and print each value. void dumpDatabase(); diff --git a/include/osquery/distributed.h b/include/osquery/distributed.h index 53014b98..faf0bea6 100644 --- a/include/osquery/distributed.h +++ b/include/osquery/distributed.h @@ -19,6 +19,136 @@ namespace osquery { +/** + * @brief Small struct containing the query and ID information for a + * distributed query + */ +struct DistributedQueryRequest { + public: + explicit DistributedQueryRequest() {} + explicit DistributedQueryRequest(const std::string& q, const std::string& i) + : query(q), id(i) {} + + /// equals operator + bool operator==(const DistributedQueryRequest& comp) const { + return (comp.query == query) && (comp.id == id); + } + + std::string query; + std::string id; +}; + +/** + * @brief Serialize a DistributedQueryRequest into a property tree + * + * @param r the DistributedQueryRequest to serialize + * @param tree the output property tree + * + * @return Status indicating the success or failure of the operation + */ +Status serializeDistributedQueryRequest(const DistributedQueryRequest& r, + boost::property_tree::ptree& tree); + +/** + * @brief Serialize a DistributedQueryRequest object into a JSON string + * + * @param r the DistributedQueryRequest to serialize + * @param json the output JSON string + * + * @return Status indicating the success or failure of the operation + */ +Status serializeDistributedQueryRequestJSON(const DistributedQueryRequest& r, + std::string& json); + +/** + * @brief Deserialize a DistributedQueryRequest object from a property tree + * + * @param tree the input property tree + * @param r the output DistributedQueryRequest structure + * + * @return Status indicating the success or failure of the operation + */ +Status deserializeDistributedQueryRequest( + const boost::property_tree::ptree& tree, DistributedQueryRequest& r); + +/** + * @brief Deserialize a DistributedQueryRequest object from a JSON string + * + * @param json the input JSON string + * @param r the output DistributedQueryRequest structure + * + * @return Status indicating the success or failure of the operation + */ +Status deserializeDistributedQueryRequestJSON(const std::string& json, + DistributedQueryRequest& r); + +///////////////////////////////////////////////////////////////////////////// +// DistributedQueryResult +///////////////////////////////////////////////////////////////////////////// + +/** + * @brief Small struct containing the results of a distributed query + */ +struct DistributedQueryResult { + public: + explicit DistributedQueryResult() {} + explicit DistributedQueryResult(const DistributedQueryRequest& req, + const QueryData& res) + : request(req), results(res) {} + + /// equals operator + bool operator==(const DistributedQueryResult& comp) const { + return (comp.request == request) && (comp.results == results); + } + + DistributedQueryRequest request; + QueryData results; +}; + +/** + * @brief Serialize a DistributedQueryResult into a property tree + * + * @param r the DistributedQueryResult to serialize + * @param tree the output property tree + * + * @return Status indicating the success or failure of the operation + */ +Status serializeDistributedQueryResult(const DistributedQueryResult& r, + boost::property_tree::ptree& tree); + +/** + * @brief Serialize a DistributedQueryResult object into a JSON string + * + * @param r the DistributedQueryResult to serialize + * @param json the output JSON string + * + * @return Status indicating the success or failure of the operation + */ +Status serializeDistributedQueryResultJSON(const DistributedQueryResult& r, + std::string& json); + +/** + * @brief Deserialize a DistributedQueryResult object from a property tree + * + * @param tree the input property tree + * @param r the output DistributedQueryResult structure + * + * @return Status indicating the success or failure of the operation + */ +Status deserializeDistributedQueryResult( + const boost::property_tree::ptree& tree, DistributedQueryResult& r); + +/** + * @brief Deserialize a DistributedQueryResult object from a JSON string + * + * @param json the input JSON string + * @param r the output DistributedQueryResult structure + * + * @return Status indicating the success or failure of the operation + */ +Status deserializeDistributedQueryResultJSON(const std::string& json, + DistributedQueryResult& r); + class DistributedPlugin : public Plugin { public: /** diff --git a/include/osquery/extensions.h b/include/osquery/extensions.h index f531fdab..a00aa0ed 100644 --- a/include/osquery/extensions.h +++ b/include/osquery/extensions.h @@ -16,7 +16,6 @@ namespace osquery { -DECLARE_int32(worker_threads); DECLARE_string(extensions_socket); DECLARE_string(extensions_autoload); DECLARE_string(extensions_timeout); @@ -41,11 +40,7 @@ typedef std::map ExtensionList; inline std::string getExtensionSocket( RouteUUID uuid, const std::string& path = FLAGS_extensions_socket) { - if (uuid == 0) { - return path; - } else { - return path + "." + std::to_string(uuid); - } + return (uuid == 0) ? path : path + "." + std::to_string(uuid); } /// External (extensions) SQL implementation of the osquery query API. diff --git a/include/osquery/registry.h b/include/osquery/registry.h index f9265686..1b95e82b 100644 --- a/include/osquery/registry.h +++ b/include/osquery/registry.h @@ -12,8 +12,8 @@ #include #include -#include #include +#include #include #include diff --git a/osquery/CMakeLists.txt b/osquery/CMakeLists.txt index fbcc791b..13ff186b 100644 --- a/osquery/CMakeLists.txt +++ b/osquery/CMakeLists.txt @@ -14,10 +14,16 @@ set(OSQUERY_ADDITIONAL_LINKS "") set(OSQUERY_ADDITIONAL_TESTS "") set(OSQUERY_TABLES_TESTS "") +# Add all and extra for osquery code. +add_compile_options( + -Wall + -Wextra + -Wno-unused-parameter +) + # The core set of osquery libraries most discovered with find_package. set(OSQUERY_LIBS # This includes librocksdb[_lite] and libsnappy. - ${ROCKSDB_LIBRARIES} ${THRIFT_LIBRARY} ${GLOG_LIBRARY} ${GFLAGS_LIBRARY} @@ -30,12 +36,10 @@ set(OSQUERY_LIBS z ) -# Add all and extra for osquery code. -add_compile_options( - -Wall - -Wextra - -Wno-unused-parameter -) +# If building with RocksDB (default) append associated libraries. +if(ROCKSDB) + set(OSQUERY_LIBS ${OSQUERY_LIBS} ${ROCKSDB_LIBRARIES}) +endif() if(NOT FREEBSD) set(OSQUERY_LIBS ${OSQUERY_LIBS} dl) diff --git a/osquery/config/config.cpp b/osquery/config/config.cpp index 08037f65..fc39b065 100644 --- a/osquery/config/config.cpp +++ b/osquery/config/config.cpp @@ -18,9 +18,9 @@ #include #include +#include #include #include -#include #include #include #include @@ -249,8 +249,9 @@ void Config::removeFiles(const std::string& source) { } } -void Config::scheduledQueries(std::function< - void(const std::string& name, const ScheduledQuery& query)> predicate) { +void Config::scheduledQueries( + std::function + predicate) { ReadLock rlock(config_schedule_mutex_); for (const PackRef& pack : *schedule_) { for (const auto& it : pack->getSchedule()) { @@ -675,18 +676,12 @@ Status Config::getMD5(std::string& hash) { const std::shared_ptr Config::getParser( const std::string& parser) { - std::shared_ptr config_parser = nullptr; - try { - auto plugin = Registry::get("config_parser", parser); - config_parser = std::dynamic_pointer_cast(plugin); - } catch (const std::out_of_range& e) { - LOG(ERROR) << "Error getting config parser plugin " << parser << ": " - << e.what(); - } catch (const std::bad_cast& e) { - LOG(ERROR) << "Error casting " << parser - << " as a ConfigParserPlugin: " << e.what(); + if (!Registry::exists("config_parser", parser, true)) { + return nullptr; } - return config_parser; + + auto plugin = Registry::get("config_parser", parser); + return std::dynamic_pointer_cast(plugin); } void Config::files( diff --git a/osquery/core/init.cpp b/osquery/core/init.cpp index 73d752c2..a407e575 100644 --- a/osquery/core/init.cpp +++ b/osquery/core/init.cpp @@ -30,7 +30,6 @@ #include #include "osquery/core/watcher.h" -#include "osquery/database/db_handle.h" #include "osquery/dispatcher/dispatcher.h" #if defined(__linux__) || defined(__FreeBSD__) @@ -105,6 +104,8 @@ static inline bool hasWorkerVariable() { volatile std::sig_atomic_t kHandledSignal{0}; +static inline bool isWatcher() { return (osquery::Watcher::getWorker() > 0); } + void signalHandler(int signal) { // Inform exit status of main threads blocked by service joins. if (kHandledSignal == 0) { @@ -112,10 +113,8 @@ void signalHandler(int signal) { } // Handle signals based on a tri-state (worker, watcher, neither). - pid_t worker_pid = osquery::Watcher::getWorker(); - bool is_watcher = worker_pid > 0; if (signal == SIGHUP) { - if (!is_watcher || hasWorkerVariable()) { + if (!isWatcher() || hasWorkerVariable()) { // Reload configuration. } } else if (signal == SIGTERM || signal == SIGINT || signal == SIGABRT) { @@ -127,7 +126,7 @@ void signalHandler(int signal) { std::signal(signal, SIG_DFL); // The watcher waits for the worker to die. - if (is_watcher) { + if (isWatcher()) { // Bind the fate of the worker to this watcher. osquery::Watcher::bindFates(); } else { @@ -145,7 +144,7 @@ void signalHandler(int signal) { raise((kHandledSignal != 0) ? kHandledSignal : SIGALRM); } - if (is_watcher) { + if (isWatcher()) { // The signal should be proliferated through the process group. // Otherwise the watcher could 'forward' the signal to workers and // managed extension processes. @@ -166,7 +165,9 @@ DECLARE_string(distributed_plugin); DECLARE_bool(disable_distributed); DECLARE_string(config_plugin); DECLARE_bool(config_check); +DECLARE_bool(config_dump); DECLARE_bool(database_dump); +DECLARE_string(database_path); ToolType kToolType = OSQUERY_TOOL_UNKNOWN; @@ -421,14 +422,27 @@ void Initializer::initActivePlugin(const std::string& type, timeout = kExtensionInitializeLatencyUS * 10; } - while (!Registry::setActive(type, name)) { - if (!Watcher::hasManagedExtensions() || delay > timeout) { - LOG(ERROR) << "Active " << type << " plugin not found: " << name; - osquery::shutdown(EXIT_CATASTROPHIC); + // Attempt to set the request plugin as active. + Status status; + do { + status = Registry::setActive(type, name); + if (status.ok()) { + // The plugin was found, and is not active. + return; } + + if (!Watcher::hasManagedExtensions()) { + // The plugin was found locally, and is not active, problem. + break; + } + // The plugin is not local and is not active, wait and retry. delay += kExtensionInitializeLatencyUS; ::usleep(kExtensionInitializeLatencyUS); - } + } while (delay < timeout); + + LOG(ERROR) << "Cannot activate " << name << " " << type + << " plugin: " << status.getMessage(); + osquery::shutdown(EXIT_CATASTROPHIC); } void Initializer::start() const { @@ -436,18 +450,26 @@ void Initializer::start() const { osquery::loadModules(); // Pre-extension manager initialization options checking. - if (FLAGS_config_check && !Watcher::hasManagedExtensions()) { + // If the shell or daemon does not need extensions and it will exit quickly, + // prefer to disable the extension manager. + if ((FLAGS_config_check || FLAGS_config_dump) && + !Watcher::hasManagedExtensions()) { FLAGS_disable_extensions = true; } - // A daemon must always have R/W access to the database. - DBHandle::setAllowOpen(true); - DBHandle::setRequireWrite(tool_ == OSQUERY_TOOL_DAEMON); - if (!DBHandle::checkDB()) { - LOG(ERROR) << RLOG(1629) << binary_ - << " initialize failed: Could not open RocksDB"; - auto retcode = (isWorker()) ? EXIT_CATASTROPHIC : EXIT_FAILURE; - ::exit(retcode); + // A watcher should not need access to the backing store. + // If there are spurious access then warning logs will be emitted since the + // set-allow-open will never be called. + if (!isWatcher()) { + DatabasePlugin::setAllowOpen(true); + // A daemon must always have R/W access to the database. + DatabasePlugin::setRequireWrite(tool_ == OSQUERY_TOOL_DAEMON); + if (!DatabasePlugin::initPlugin()) { + LOG(ERROR) << RLOG(1629) << binary_ + << " initialize failed: Could not initialize database"; + auto retcode = (isWorker()) ? EXIT_CATASTROPHIC : EXIT_FAILURE; + ::exit(retcode); + } } // Bind to an extensions socket and wait for registry additions. @@ -516,6 +538,7 @@ void shutdown(int retcode, bool wait) { // Hopefully release memory used by global string constructors in gflags. GFLAGS_NAMESPACE::ShutDownCommandLineFlags(); + DatabasePlugin::shutdown(); ::exit(retcode); } } diff --git a/osquery/core/test_util.cpp b/osquery/core/test_util.cpp index 38b09395..bdea6bad 100644 --- a/osquery/core/test_util.cpp +++ b/osquery/core/test_util.cpp @@ -16,18 +16,20 @@ #include #include -#include #include +#include +#include +#include #include #include #include "osquery/core/test_util.h" -#include "osquery/database/db_handle.h" namespace fs = boost::filesystem; namespace osquery { + std::string kFakeDirectory = ""; #ifdef DARWIN @@ -80,10 +82,14 @@ void initTesting() { FLAGS_modules_autoload = kTestWorkingDirectory + "unittests-mod.load"; FLAGS_disable_logging = true; - // Create a default DBHandle instance before unittests. - (void)DBHandle::getInstance(); + // Tests need a database plugin. + // Set up the database instance for the unittests. + DatabasePlugin::setAllowOpen(true); + Registry::setActive("database", "ephemeral"); } +void shutdownTesting() { DatabasePlugin::shutdown(); } + std::map getTestConfigMap() { std::string content; readFile(kTestDataPath + "test_parse_items.conf", content); diff --git a/osquery/core/test_util.h b/osquery/core/test_util.h index 31edf8de..b8bd174a 100644 --- a/osquery/core/test_util.h +++ b/osquery/core/test_util.h @@ -29,7 +29,7 @@ namespace osquery { void initTesting(); /// Cleanup/stop function for tests and benchmarks. -void cleanupTesting(); +void shutdownTesting(); /// Any SQL-dependent tests should use kTestQuery for a pre-populated example. const std::string kTestQuery = "SELECT * FROM test_table"; diff --git a/osquery/core/watcher.cpp b/osquery/core/watcher.cpp index 9a56036f..1a4904ea 100644 --- a/osquery/core/watcher.cpp +++ b/osquery/core/watcher.cpp @@ -11,8 +11,8 @@ #include #include -#include #include +#include #include @@ -431,7 +431,7 @@ void WatcherWatcherRunner::start() { VLOG(1) << "osqueryd worker (" << getpid() << ") detected killed watcher (" << watcher_ << ")"; // The watcher watcher is a thread. Do not join services after removing. - ::exit(EXIT_FAILURE); + raise(SIGKILL); } interruptableSleep(getWorkerLimit(INTERVAL) * 1000); } diff --git a/osquery/database/CMakeLists.txt b/osquery/database/CMakeLists.txt index 5687595b..f965ce3e 100644 --- a/osquery/database/CMakeLists.txt +++ b/osquery/database/CMakeLists.txt @@ -1,15 +1,41 @@ ADD_OSQUERY_LIBRARY(TRUE osquery_database database.cpp -) - -# osquery_database_internal should be an 'additional' CORE=False lib -ADD_OSQUERY_LIBRARY(TRUE osquery_database_internal - db_handle.cpp query.cpp + + # Add 'core' plugins that do not required additional libraries. + plugins/ephemeral.cpp ) +# Begin with just the SQLite database plugin. +set(OSQUERY_DATABASE_PLUGINS + plugins/sqlite.cpp +) + +set(OSQUERY_DATABASE_PLUGIN_TESTS + database/tests/plugin_tests.cpp + database/plugins/tests/sqlite_tests.cpp +) + +# Optionally (but by default), add the RocksDB database plugin. +if(ROCKSDB) + if(ROCKSDB_LITE_FOUND) + add_definitions(-DROCKSDB_LITE=1) + endif() + set(OSQUERY_DATABASE_PLUGINS ${OSQUERY_DATABASE_PLUGINS} plugins/rocksdb.cpp) + set(OSQUERY_DATABASE_PLUGIN_TESTS ${OSQUERY_DATABASE_PLUGIN_TESTS} database/plugins/tests/rocksdb_tests.cpp) +else() + add_definitions(-DSKIP_ROCKSDB) +endif() + +# Plugins (which do not include the shim/ephemeral) are 'additional'. +ADD_OSQUERY_LIBRARY(FALSE osquery_database_plugins ${OSQUERY_DATABASE_PLUGINS}) + +# Non-plugin tests are core. file(GLOB OSQUERY_DATABASE_TESTS "tests/*.cpp") ADD_OSQUERY_TEST(TRUE ${OSQUERY_DATABASE_TESTS}) +# Plugin tests are additional. +ADD_OSQUERY_TEST(FALSE ${OSQUERY_DATABASE_PLUGIN_TESTS}) + file(GLOB OSQUERY_DATABASE_BENCHMARKS "benchmarks/*.cpp") ADD_OSQUERY_BENCHMARK(${OSQUERY_DATABASE_BENCHMARKS}) diff --git a/osquery/database/benchmarks/database_benchmarks.cpp b/osquery/database/benchmarks/database_benchmarks.cpp index ad089451..22f1ed5b 100644 --- a/osquery/database/benchmarks/database_benchmarks.cpp +++ b/osquery/database/benchmarks/database_benchmarks.cpp @@ -10,11 +10,10 @@ #include -#include #include +#include #include "osquery/core/test_util.h" -#include "osquery/database/db_handle.h" #include "osquery/database/query.h" namespace osquery { @@ -76,8 +75,22 @@ static void DATABASE_query_results(benchmark::State& state) { } } -BENCHMARK(DATABASE_query_results)->ArgPair(1, 1)->ArgPair(10, 10)->ArgPair(10, - 100); +BENCHMARK(DATABASE_query_results) + ->ArgPair(1, 1) + ->ArgPair(10, 10) + ->ArgPair(10, 100); + +static void DATABASE_get(benchmark::State& state) { + setDatabaseValue(kPersistentSettings, "benchmark", "1"); + while (state.KeepRunning()) { + std::string value; + getDatabaseValue(kPersistentSettings, "benchmark", value); + } + // All benchmarks will share a single database handle. + deleteDatabaseValue(kPersistentSettings, "benchmark"); +} + +BENCHMARK(DATABASE_get); static void DATABASE_store(benchmark::State& state) { while (state.KeepRunning()) { @@ -112,12 +125,14 @@ static void DATABASE_store_append(benchmark::State& state) { size_t k = 0; while (state.KeepRunning()) { - setDatabaseValue(kPersistentSettings, "key" + std::to_string(k++), content); + setDatabaseValue(kPersistentSettings, "key" + std::to_string(k), content); + deleteDatabaseValue(kPersistentSettings, "key" + std::to_string(k)); + k++; } // All benchmarks will share a single database handle. for (size_t i = 0; i < k; ++i) { - deleteDatabaseValue(kPersistentSettings, "key" + std::to_string(i)); + // deleteDatabaseValue(kPersistentSettings, "key" + std::to_string(i)); } } diff --git a/osquery/database/database.cpp b/osquery/database/database.cpp index 37d45230..1933e7b1 100644 --- a/osquery/database/database.cpp +++ b/osquery/database/database.cpp @@ -8,12 +8,7 @@ * */ -#include -#include -#include #include -#include -#include #include #include @@ -27,6 +22,37 @@ namespace osquery { CLI_FLAG(bool, database_dump, false, "Dump the contents of the backing store"); +CLI_FLAG(string, + database_path, + "/var/osquery/osquery.db", + "If using a disk-based backing store, specify a path"); +FLAG_ALIAS(std::string, db_path, database_path); + +CLI_FLAG(bool, + database_in_memory, + false, + "Keep osquery backing-store in memory"); +FLAG_ALIAS(bool, use_in_memory_database, database_in_memory); + +#if defined(SKIP_ROCKSDB) +#define DATABASE_PLUGIN "sqlite" +#else +#define DATABASE_PLUGIN "rocksdb" +#endif +const std::string kInternalDatabase = DATABASE_PLUGIN; + +const std::string kPersistentSettings = "configurations"; +const std::string kQueries = "queries"; +const std::string kEvents = "events"; +const std::string kLogs = "logs"; + +const std::vector kDomains = {kPersistentSettings, kQueries, + kEvents, kLogs}; + +bool DatabasePlugin::kDBHandleOptionAllowOpen(false); +bool DatabasePlugin::kDBHandleOptionRequireWrite(false); +std::atomic DatabasePlugin::kCheckingDB(false); + ///////////////////////////////////////////////////////////////////////////// // Row - the representation of a row in a set of database results. Row is a // simple map where individual column names are keys, which map to the Row's @@ -360,133 +386,6 @@ Status serializeQueryLogItemAsEventsJSON(const QueryLogItem& i, return Status(0, "OK"); } -///////////////////////////////////////////////////////////////////////////// -// DistributedQueryRequest - small struct containing the query and ID -// information for a distributed query -///////////////////////////////////////////////////////////////////////////// - -Status serializeDistributedQueryRequest(const DistributedQueryRequest& r, - pt::ptree& tree) { - tree.put("query", r.query); - tree.put("id", r.id); - return Status(0, "OK"); -} - -Status serializeDistributedQueryRequestJSON(const DistributedQueryRequest& r, - std::string& json) { - pt::ptree tree; - auto s = serializeDistributedQueryRequest(r, tree); - if (!s.ok()) { - return s; - } - std::stringstream ss; - try { - pt::write_json(ss, tree, false); - } catch (const pt::ptree_error& e) { - return Status(1, "Error serializing JSON: " + std::string(e.what())); - } - json = ss.str(); - - return Status(0, "OK"); -} - -Status deserializeDistributedQueryRequest(const pt::ptree& tree, - DistributedQueryRequest& r) { - r.query = tree.get("query", ""); - r.id = tree.get("id", ""); - return Status(0, "OK"); -} - -Status deserializeDistributedQueryRequestJSON(const std::string& json, - DistributedQueryRequest& r) { - std::stringstream ss(json); - pt::ptree tree; - try { - pt::read_json(ss, tree); - } catch (const pt::ptree_error& e) { - return Status(1, "Error serializing JSON: " + std::string(e.what())); - } - return deserializeDistributedQueryRequest(tree, r); -} - -///////////////////////////////////////////////////////////////////////////// -// DistributedQueryResult - small struct containing the results of a -// distributed query -///////////////////////////////////////////////////////////////////////////// - -Status serializeDistributedQueryResult(const DistributedQueryResult& r, - pt::ptree& tree) { - pt::ptree request; - auto s = serializeDistributedQueryRequest(r.request, request); - if (!s.ok()) { - return s; - } - - pt::ptree results; - s = serializeQueryData(r.results, results); - if (!s.ok()) { - return s; - } - - tree.add_child("request", request); - tree.add_child("results", results); - - return Status(0, "OK"); -} - -Status serializeDistributedQueryResultJSON(const DistributedQueryResult& r, - std::string& json) { - pt::ptree tree; - auto s = serializeDistributedQueryResult(r, tree); - if (!s.ok()) { - return s; - } - std::stringstream ss; - try { - pt::write_json(ss, tree, false); - } catch (const pt::ptree_error& e) { - return Status(1, "Error serializing JSON: " + std::string(e.what())); - } - json = ss.str(); - - return Status(0, "OK"); -} - -Status deserializeDistributedQueryResult(const pt::ptree& tree, - DistributedQueryResult& r) { - DistributedQueryRequest request; - auto s = - deserializeDistributedQueryRequest(tree.get_child("request"), request); - if (!s.ok()) { - return s; - } - - QueryData results; - s = deserializeQueryData(tree.get_child("results"), results); - if (!s.ok()) { - return s; - } - - r.request = request; - r.results = results; - - return Status(0, "OK"); -} - -Status deserializeDistributedQueryResultJSON(const std::string& json, - DistributedQueryResult& r) { - std::stringstream ss(json); - pt::ptree tree; - try { - pt::read_json(ss, tree); - } catch (const pt::ptree_error& e) { - return Status(1, "Error serializing JSON: " + std::string(e.what())); - } - return deserializeDistributedQueryResult(tree, r); -} - -///////////////////////////////////////////////////////////////////////////// - bool addUniqueRowToQueryData(QueryData& q, const Row& r) { if (std::find(q.begin(), q.end(), r) != q.end()) { return false; @@ -495,6 +394,41 @@ bool addUniqueRowToQueryData(QueryData& q, const Row& r) { return true; } +bool DatabasePlugin::initPlugin() { + // Initialize the database plugin using the flag. + return Registry::setActive("database", kInternalDatabase).ok(); +} + +void DatabasePlugin::shutdown() { + auto datbase_registry = Registry::registry("database"); + for (auto& plugin : datbase_registry->names()) { + datbase_registry->remove(plugin); + } +} + +Status DatabasePlugin::reset() { + tearDown(); + return setUp(); +} + +bool DatabasePlugin::checkDB() { + kCheckingDB = true; + bool result = true; + try { + auto status = setUp(); + if (kDBHandleOptionRequireWrite && read_only_) { + result = false; + } + tearDown(); + result = status.ok(); + } catch (const std::exception& e) { + VLOG(1) << "Database plugin check failed: " << e.what(); + result = false; + } + kCheckingDB = false; + return result; +} + Status DatabasePlugin::call(const PluginRequest& request, PluginResponse& response) { if (request.count("action") == 0) { @@ -526,7 +460,7 @@ Status DatabasePlugin::call(const PluginRequest& request, if (request.count("max") > 0) { max = std::stoul(request.at("max")); } - auto status = this->scan(domain, keys, max); + auto status = this->scan(domain, keys, request.at("prefix"), max); for (const auto& key : keys) { response.push_back({{"k", key}}); } @@ -536,51 +470,97 @@ Status DatabasePlugin::call(const PluginRequest& request, return Status(1, "Unknown database plugin action"); } +static inline std::shared_ptr getDatabasePlugin() { + if (!Registry::exists("database", Registry::getActive("database"), true)) { + return nullptr; + } + + auto plugin = Registry::get("database", Registry::getActive("database")); + return std::dynamic_pointer_cast(plugin); +} + Status getDatabaseValue(const std::string& domain, const std::string& key, std::string& value) { - PluginRequest request = {{"action", "get"}, {"domain", domain}, {"key", key}}; - PluginResponse response; - auto status = Registry::call("database", "rocks", request, response); - if (!status.ok()) { + if (Registry::external()) { + // External registries (extensions) do not have databases active. + // It is not possible to use an extension-based database. + PluginRequest request = { + {"action", "get"}, {"domain", domain}, {"key", key}}; + PluginResponse response; + auto status = Registry::call("database", request, response); + if (status.ok()) { + // Set value from the internally-known "v" key. + if (response.size() > 0 && response[0].count("v") > 0) { + value = response[0].at("v"); + } + } return status; + } else { + auto plugin = getDatabasePlugin(); + return plugin->get(domain, key, value); } - - // Set value from the internally-known "v" key. - if (response.size() > 0 && response[0].count("v") > 0) { - value = response[0].at("v"); - } - return status; } Status setDatabaseValue(const std::string& domain, const std::string& key, const std::string& value) { - PluginRequest request = { - {"action", "put"}, {"domain", domain}, {"key", key}, {"value", value}}; - return Registry::call("database", "rocks", request); + if (Registry::external()) { + // External registries (extensions) do not have databases active. + // It is not possible to use an extension-based database. + PluginRequest request = { + {"action", "put"}, {"domain", domain}, {"key", key}, {"value", value}}; + return Registry::call("database", request); + } else { + auto plugin = getDatabasePlugin(); + return plugin->put(domain, key, value); + } } Status deleteDatabaseValue(const std::string& domain, const std::string& key) { - PluginRequest request = { - {"action", "remove"}, {"domain", domain}, {"key", key}}; - return Registry::call("database", "rocks", request); + if (Registry::external()) { + // External registries (extensions) do not have databases active. + // It is not possible to use an extension-based database. + PluginRequest request = { + {"action", "remove"}, {"domain", domain}, {"key", key}}; + return Registry::call("database", request); + } else { + auto plugin = getDatabasePlugin(); + return plugin->remove(domain, key); + } } Status scanDatabaseKeys(const std::string& domain, std::vector& keys, size_t max) { - PluginRequest request = { - {"action", "scan"}, {"domain", domain}, {"max", std::to_string(max)}}; - PluginResponse response; - auto status = Registry::call("database", "rocks", request, response); + return scanDatabaseKeys(domain, keys, "", max); +} - for (const auto& item : response) { - if (item.count("k") > 0) { - keys.push_back(item.at("k")); +/// Get a list of keys for a given domain. +Status scanDatabaseKeys(const std::string& domain, + std::vector& keys, + const std::string& prefix, + size_t max) { + if (Registry::external()) { + // External registries (extensions) do not have databases active. + // It is not possible to use an extension-based database. + PluginRequest request = {{"action", "scan"}, + {"domain", domain}, + {"prefix", prefix}, + {"max", std::to_string(max)}}; + PluginResponse response; + auto status = Registry::call("database", request, response); + + for (const auto& item : response) { + if (item.count("k") > 0) { + keys.push_back(item.at("k")); + } } + return status; + } else { + auto plugin = getDatabasePlugin(); + return plugin->scan(domain, keys, prefix, max); } - return status; } void dumpDatabase() { diff --git a/osquery/database/db_handle.cpp b/osquery/database/db_handle.cpp deleted file mode 100644 index 9d9fb9fc..00000000 --- a/osquery/database/db_handle.cpp +++ /dev/null @@ -1,400 +0,0 @@ -/* - * Copyright (c) 2014-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - * - */ - -#include -#include -#include - -#include - -#include - -#include -#include - -#include -#include -#include -#include - -#include "osquery/database/db_handle.h" - -namespace osquery { - -class RocksDatabasePlugin : public DatabasePlugin { - public: - /// Data retrieval method. - Status get(const std::string& domain, - const std::string& key, - std::string& value) const override; - - /// Data storage method. - Status put(const std::string& domain, - const std::string& key, - const std::string& value) override; - - /// Data removal method. - Status remove(const std::string& domain, const std::string& k) override; - - /// Key/index lookup method. - Status scan(const std::string& domain, - std::vector& results, - size_t max = 0) const override; -}; - -/// Backing-storage provider for osquery internal/core. -REGISTER_INTERNAL(RocksDatabasePlugin, "database", "rocks"); - -///////////////////////////////////////////////////////////////////////////// -// Constants -///////////////////////////////////////////////////////////////////////////// - -bool DBHandle::kDBHandleOptionAllowOpen = false; -bool DBHandle::kDBHandleOptionRequireWrite = false; - -const std::string kPersistentSettings = "configurations"; -const std::string kQueries = "queries"; -const std::string kEvents = "events"; -const std::string kLogs = "logs"; - -/** - * @brief A const vector of column families in RocksDB - * - * RocksDB has a concept of "column families" which are kind of like tables - * in other databases. kDomainds is populated with a list of all column - * families. If a string exists in kDomains, it's a column family in the - * database. - */ -const std::vector kDomains = { - kPersistentSettings, kQueries, kEvents, kLogs}; - -CLI_FLAG(string, - database_path, - "/var/osquery/osquery.db", - "If using a disk-based backing store, specify a path"); -FLAG_ALIAS(std::string, db_path, database_path); - -CLI_FLAG(bool, - database_in_memory, - false, - "Keep osquery backing-store in memory"); -FLAG_ALIAS(bool, use_in_memory_database, database_in_memory); - -///////////////////////////////////////////////////////////////////////////// -// constructors and destructors -///////////////////////////////////////////////////////////////////////////// - -/// A queryable mutex around database sanity checking. -static bool kCheckingDB = false; - -void GlogRocksDBLogger::Logv(const char* format, va_list ap) { - // Convert RocksDB log to string and check if header or level-ed log. - char buffer[501] = {0}; - vsnprintf(buffer, 500, format, ap); - va_end(ap); - if (buffer[0] != '[') { - return; - } - - if (buffer[1] == 'E' || buffer[1] == 'W') { - LOG(INFO) << "RocksDB: " << buffer; - } -} - -DBHandle::DBHandle(const std::string& path, bool in_memory) - : path_(path), in_memory_(in_memory) { - if (!kDBHandleOptionAllowOpen) { - LOG(WARNING) << RLOG(1629) << "Not allowed to create DBHandle instance"; - } - - // Set meta-data (mostly) handling options. - options_.create_if_missing = true; - options_.create_missing_column_families = true; - options_.info_log_level = rocksdb::ERROR_LEVEL; - options_.log_file_time_to_roll = 0; - options_.keep_log_file_num = 10; - options_.max_log_file_size = 1024 * 1024 * 1; - options_.stats_dump_period_sec = 0; - - // Performance and optimization settings. - options_.compression = rocksdb::kNoCompression; - options_.compaction_style = rocksdb::kCompactionStyleLevel; - options_.arena_block_size = (4 * 1024); - options_.write_buffer_size = (4 * 1024) * 100; // 100 blocks. - options_.max_write_buffer_number = 2; - options_.min_write_buffer_number_to_merge = 1; - options_.max_background_compactions = 2; - options_.max_background_flushes = 2; - - // Create an environment to replace the default logger. - if (logger_ == nullptr) { - logger_ = std::make_shared(); - } - options_.info_log = logger_; - - column_families_.push_back(rocksdb::ColumnFamilyDescriptor( - rocksdb::kDefaultColumnFamilyName, options_)); - - for (const auto& cf_name : kDomains) { - column_families_.push_back( - rocksdb::ColumnFamilyDescriptor(cf_name, options_)); - } - - // Make the magic happen. - open(); -} - -void DBHandle::open() { - if (in_memory_) { - // Remove when MemEnv is included in librocksdb - // options_.env = rocksdb::NewMemEnv(rocksdb::Env::Default()); - throw std::runtime_error("Cannot start in-memory RocksDB: Requires MemEnv"); - } - - if (pathExists(path_).ok() && !isReadable(path_).ok()) { - throw std::runtime_error("Cannot read RocksDB path: " + path_); - } - - if (!kCheckingDB) { - VLOG(1) << "Opening RocksDB handle: " << path_; - } - - auto s = - rocksdb::DB::Open(options_, path_, column_families_, &handles_, &db_); - if (!s.ok() || db_ == nullptr) { - if (kDBHandleOptionRequireWrite) { - // A failed open in R/W mode is a runtime error. - throw std::runtime_error(s.ToString()); - } - - if (!kCheckingDB) { - VLOG(1) << "Opening RocksDB failed: Continuing with read-only support"; - } -#if !defined(ROCKSDB_LITE) - // RocksDB LITE does not support readonly mode. - // The database was readable but could not be opened, either (1) it is not - // writable or (2) it is already opened by another process. - // Try to open the database in a ReadOnly mode. - rocksdb::DB::OpenForReadOnly( - options_, path_, column_families_, &handles_, &db_); -#endif - // Also disable event publishers. - Flag::updateValue("disable_events", "true"); - read_only_ = true; - } - - // RocksDB may not create/append a directory with acceptable permissions. - if (!read_only_ && chmod(path_.c_str(), S_IRWXU) != 0) { - throw std::runtime_error("Cannot set permissions on RocksDB path: " + - path_); - } -} - -DBHandle::~DBHandle() { close(); } - -void DBHandle::close() { - for (auto handle : handles_) { - delete handle; - } - - if (db_ != nullptr) { - delete db_; - } -} - -///////////////////////////////////////////////////////////////////////////// -// getInstance methods -///////////////////////////////////////////////////////////////////////////// - -DBHandleRef DBHandle::getInstance() { - return getInstance(FLAGS_database_path, FLAGS_database_in_memory); -} - -bool DBHandle::checkDB() { - // Allow database instances to check if a status/sanity check was requested. - kCheckingDB = true; - try { - auto handle = DBHandle(FLAGS_database_path, FLAGS_database_in_memory); - kCheckingDB = false; - if (kDBHandleOptionRequireWrite && handle.read_only_) { - return false; - } - } catch (const std::exception& e) { - kCheckingDB = false; - VLOG(1) << e.what(); - return false; - } - return true; -} - -DBHandleRef DBHandle::getInstanceInMemory() { return getInstance("", true); } - -DBHandleRef DBHandle::getInstanceAtPath(const std::string& path) { - return getInstance(path, false); -} - -DBHandleRef DBHandle::getInstance(const std::string& path, bool in_memory) { - static DBHandleRef db_handle = DBHandleRef(new DBHandle(path, in_memory)); - return db_handle; -} - -void DBHandle::resetInstance(const std::string& path, bool in_memory) { - close(); - - path_ = path; - in_memory_ = in_memory; - open(); -} - -///////////////////////////////////////////////////////////////////////////// -// getters and setters -///////////////////////////////////////////////////////////////////////////// - -rocksdb::DB* DBHandle::getDB() const { return db_; } - -rocksdb::ColumnFamilyHandle* DBHandle::getHandleForColumnFamily( - const std::string& cf) const { - try { - for (size_t i = 0; i < kDomains.size(); i++) { - if (kDomains[i] == cf) { - return handles_[i]; - } - } - } catch (const std::exception& e) { - // pass through and return nullptr - } - return nullptr; -} - -///////////////////////////////////////////////////////////////////////////// -// Data manipulation methods -///////////////////////////////////////////////////////////////////////////// - -Status DBHandle::Get(const std::string& domain, - const std::string& key, - std::string& value) const { - if (getDB() == nullptr) { - return Status(1, "Database not opened"); - } - auto cfh = getHandleForColumnFamily(domain); - if (cfh == nullptr) { - return Status(1, "Could not get column family for " + domain); - } - auto s = getDB()->Get(rocksdb::ReadOptions(), cfh, key, &value); - return Status(s.code(), s.ToString()); -} - -Status DBHandle::Put(const std::string& domain, - const std::string& key, - const std::string& value) const { - if (read_only_) { - return Status(0, "Database in readonly mode"); - } - - auto cfh = getHandleForColumnFamily(domain); - if (cfh == nullptr) { - return Status(1, "Could not get column family for " + domain); - } - auto s = getDB()->Put(rocksdb::WriteOptions(), cfh, key, value); - if (s.code() != 0 && s.IsIOError()) { - // An error occurred, check if it is an IO error and remove the offending - // specific filename or log name. - std::string error_string = s.ToString(); - size_t error_pos = error_string.find_last_of(":"); - if (error_pos != std::string::npos) { - return Status(s.code(), "IOError: " + error_string.substr(error_pos + 2)); - } - } - return Status(s.code(), s.ToString()); -} - -Status DBHandle::Delete(const std::string& domain, - const std::string& key) const { - if (read_only_) { - return Status(0, "Database in readonly mode"); - } - - auto cfh = getHandleForColumnFamily(domain); - if (cfh == nullptr) { - return Status(1, "Could not get column family for " + domain); - } - auto options = rocksdb::WriteOptions(); - - // We could sync here, but large deletes will cause multi-syncs. - // For example: event record expirations found in an expired index. - // options.sync = true; - auto s = getDB()->Delete(options, cfh, key); - return Status(s.code(), s.ToString()); -} - -Status DBHandle::Scan(const std::string& domain, - std::vector& results, - size_t max) const { - // Trampoline into scan prefix with an empty prefix requirement. - return ScanPrefix(domain, results, "", max); -} - -Status DBHandle::ScanPrefix(const std::string& domain, - std::vector& results, - const std::string& prefix, - size_t max) const { - if (getDB() == nullptr) { - return Status(1, "Database not opened"); - } - - auto cfh = getHandleForColumnFamily(domain); - if (cfh == nullptr) { - return Status(1, "Could not get column family for " + domain); - } - auto options = rocksdb::ReadOptions(); - options.verify_checksums = false; - options.fill_cache = false; - auto it = getDB()->NewIterator(rocksdb::ReadOptions(), cfh); - if (it == nullptr) { - return Status(1, "Could not get iterator for " + domain); - } - - size_t count = 0; - for (it->SeekToFirst(); it->Valid(); it->Next()) { - auto key = it->key().ToString(); - if (key.find(prefix) == 0) { - results.push_back(std::move(key)); - if (max > 0 && ++count >= max) { - break; - } - } - } - delete it; - return Status(0, "OK"); -} - -Status RocksDatabasePlugin::get(const std::string& domain, - const std::string& key, - std::string& value) const { - return DBHandle::getInstance()->Get(domain, key, value); -} - -Status RocksDatabasePlugin::put(const std::string& domain, - const std::string& key, - const std::string& value) { - return DBHandle::getInstance()->Put(domain, key, value); -} - -Status RocksDatabasePlugin::remove(const std::string& domain, - const std::string& key) { - return DBHandle::getInstance()->Delete(domain, key); -} - -Status RocksDatabasePlugin::scan(const std::string& domain, - std::vector& results, - size_t max) const { - return DBHandle::getInstance()->Scan(domain, results, max); -} -} diff --git a/osquery/database/db_handle.h b/osquery/database/db_handle.h deleted file mode 100644 index 9b2a56b3..00000000 --- a/osquery/database/db_handle.h +++ /dev/null @@ -1,296 +0,0 @@ -/* - * Copyright (c) 2014-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - * - */ - -#pragma once - -#include -#include -#include - -#include -#include -#include - -#include - -#include -#include - -namespace osquery { - -DECLARE_string(database_path); - -class DBHandle; -typedef std::shared_ptr DBHandleRef; - -class GlogRocksDBLogger : public rocksdb::Logger { - public: - // We intend to override a virtual method that is overloaded. - using rocksdb::Logger::Logv; - void Logv(const char* format, va_list ap) override; -}; - -/** - * @brief RAII singleton around RocksDB database access. - * - * Accessing RocksDB necessitates creating several pointers which must be - * carefully memory managed. DBHandle offers you a singleton which takes - * care of acquiring and releasing the relevant pointers and data structures - * for you. - */ -class DBHandle { - public: - /// Removes every column family handle and single DB handle/lock. - ~DBHandle(); - - /** - * @brief The primary way to access the DBHandle singleton. - * - * DBHandle::getInstance() provides access to the DBHandle singleton. - * - * @code{.cpp} - * auto db = DBHandle::getInstance(); - * std::string value; - * auto status = db->Get("default", "foo", value); - * if (status.ok()) { - * assert(value == "bar"); - * } - * @endcode - * - * @return a shared pointer to an instance of DBHandle - */ - static DBHandleRef getInstance(); - - /** - * @brief Check the sanity of the database configuration options - * - * Create a handle to the backing store using the database configuration. - * Catch any instance creation exceptions and release the handle immediately. - * - * @return Success if a handle was created without error. - */ - static bool checkDB(); - - /// Require all DBHandle accesses to open a read and write handle. - static void setRequireWrite(bool rw) { kDBHandleOptionRequireWrite = rw; } - - /// Allow DBHandle creations. - static void setAllowOpen(bool ao) { kDBHandleOptionAllowOpen = ao; } - - public: - ///////////////////////////////////////////////////////////////////////////// - // Data access methods - ///////////////////////////////////////////////////////////////////////////// - - /** - * @brief Get data from the database for a given domain and key. - * - * @param domain the "domain" or "column family" - * @param key the string key - * @param value the output string container, will be populated with data - * - * @return operation success or failure - */ - Status Get(const std::string& domain, - const std::string& key, - std::string& value) const; - - /** - * @brief Put data into the database identified by a domain and key. - * - * @param domain the "domain" or "column family" - * @param key the string key - * @param value the data in a string container - * - * @return operation success or failure - */ - Status Put(const std::string& domain, - const std::string& key, - const std::string& value) const; - - /** - * @brief Delete data from the database given a domain and key. - * - * @param domain the "domain" or "column family" - * @param key the string key - * - * @return operation success or failure - */ - Status Delete(const std::string& domain, const std::string& key) const; - - /** - * @brief List the keys in a "domain" - * - * @param domain the "domain" or "column family" - * @param results an output list of all keys within the domain - * @param optional max limit the number of result keys to a max - * - * @return operation success or failure - */ - Status Scan(const std::string& domain, - std::vector& results, - size_t max = 0) const; - - /** - * @brief List the data in a "domain" - * - * @param domain the "domain" or "column family" - * @param results an output list of all keys with the given prefix - * @param prefix require each key to contain this string prefix - * @param optional max limit the number of result keys to a max - * - * @return operation success or failure - */ - Status ScanPrefix(const std::string& domain, - std::vector& results, - const std::string& prefix, - size_t max = 0) const; - - private: - /** - * @brief Default constructor - * - * DBHandle's constructor takes care of properly connecting to RocksDB and - * ensuring that all necessary column families are created. The resulting - * database handle can then be accessed via DBHandle::getDB() and the - * success of the connection can be determined by inspecting the resulting - * status code via DBHandle::getStatus() - */ - DBHandle(); - - /** - * @brief Internal only constructor used to create instances of DBHandle. - * - * This constructor allows you to specify a few more details about how you'd - * like DBHandle to be used. This is only used internally, so you should - * never actually use it. - * - * @param path the path to create/access the database - * @param in_memory a boolean indicating whether or not the database should - * be creating in memory or not. - */ - DBHandle(const std::string& path, bool in_memory); - - /** - * @brief A method which allows you to override the database path - * - * This should only be used by unit tests. Never use it in production code. - * - * @return a shared pointer to an instance of DBHandle - */ - static DBHandleRef getInstanceAtPath(const std::string& path); - - /** - * @brief A method which gets you an in-memory RocksDB instance. - * - * This should only be used by unit tests. Never use it in production code. - * - * @return a shared pointer to an instance of DBHandle - */ - static DBHandleRef getInstanceInMemory(); - - /** - * @brief A method which allows you to configure various aspects of RocksDB - * database options. - * - * This should only be used by unit tests. Never use it in production code. - * - * @param path the path to create/access the database - * @param in_memory a boolean indicating whether or not the database should - * be creating in memory or not. - * - * @return a shared pointer to an instance of DBHandle - */ - static DBHandleRef getInstance(const std::string& path, bool in_memory); - - /// Allow friend classes, such as unit tests, to reset the instance. - void resetInstance(const std::string& path, bool in_memory); - - /// Perform the DB open work. - void open(); - - /// Perform the DB close work. - void close(); - - /** - * @brief Private helper around accessing the column family handle for a - * specific column family, based on its name - */ - rocksdb::ColumnFamilyHandle* getHandleForColumnFamily( - const std::string& cf) const; - - /** - * @brief Helper method which can be used to get a raw pointer to the - * underlying RocksDB database handle - * - * You probably shouldn't use this. DBHandle::getDB() should only be used - * when you're positive that it's the right thing to use. - * - * @return a pointer to the underlying RocksDB database handle - */ - rocksdb::DB* getDB() const; - - public: - /// Control availability of the RocksDB handle (default false). - static bool kDBHandleOptionAllowOpen; - // The database must be opened in a R/W mode (default false). - static bool kDBHandleOptionRequireWrite; - - private: - ///////////////////////////////////////////////////////////////////////////// - // Private members - ///////////////////////////////////////////////////////////////////////////// - - /// The database handle - rocksdb::DB* db_{nullptr}; - - /// RocksDB logger instance. - std::shared_ptr logger_{nullptr}; - - /// Column family descriptors which are used to connect to RocksDB - std::vector column_families_; - - /// A vector of pointers to column family handles - std::vector handles_; - - /// The RocksDB connection options that are used to connect to RocksDB - rocksdb::Options options_; - - /// The database was opened in a ReadOnly mode. - bool read_only_{false}; - - /// Location of RocksDB on disk, blank if in-memory is true. - std::string path_; - - /// True if the database was started in an in-memory only mode. - bool in_memory_{false}; - - private: - friend class RocksDatabasePlugin; - friend class Query; - friend class EventSubscriberPlugin; - - ///////////////////////////////////////////////////////////////////////////// - // Unit tests which can access private members - ///////////////////////////////////////////////////////////////////////////// - - friend class DBHandleTests; - FRIEND_TEST(DBHandleTests, test_get); - FRIEND_TEST(DBHandleTests, test_put); - FRIEND_TEST(DBHandleTests, test_delete); - FRIEND_TEST(DBHandleTests, test_scan); - friend class QueryTests; - FRIEND_TEST(QueryTests, test_get_query_results); - FRIEND_TEST(QueryTests, test_is_query_name_in_database); - FRIEND_TEST(QueryTests, test_get_stored_query_names); - friend class EventsTests; - friend class EventsDatabaseTests; -}; -} diff --git a/osquery/database/plugins/ephemeral.cpp b/osquery/database/plugins/ephemeral.cpp new file mode 100644 index 00000000..45490385 --- /dev/null +++ b/osquery/database/plugins/ephemeral.cpp @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2014-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ + +#include +#include + +namespace osquery { + +DECLARE_string(database_path); +DECLARE_bool(database_in_memory); + +class EphemeralDatabasePlugin : public DatabasePlugin { + using DBType = std::map >; + + public: + /// Data retrieval method. + Status get(const std::string& domain, + const std::string& key, + std::string& value) const override; + + /// Data storage method. + Status put(const std::string& domain, + const std::string& key, + const std::string& value) override; + + /// Data removal method. + Status remove(const std::string& domain, const std::string& k) override; + + /// Key/index lookup method. + Status scan(const std::string& domain, + std::vector& results, + const std::string& prefix, + size_t max = 0) const override; + + public: + /// Database workflow: open and setup. + Status setUp() override { + DBType().swap(db_); + return Status(0); + } + + private: + DBType db_; +}; + +/// Backing-storage provider for osquery internal/core. +REGISTER_INTERNAL(EphemeralDatabasePlugin, "database", "ephemeral"); + +Status EphemeralDatabasePlugin::get(const std::string& domain, + const std::string& key, + std::string& value) const { + if (db_.count(domain) > 0 && db_.at(domain).count(key) > 0) { + value = db_.at(domain).at(key); + return Status(0); + } else { + return Status(1); + } +} + +Status EphemeralDatabasePlugin::put(const std::string& domain, + const std::string& key, + const std::string& value) { + db_[domain][key] = value; + return Status(0); +} + +Status EphemeralDatabasePlugin::remove(const std::string& domain, + const std::string& k) { + db_[domain].erase(k); + return Status(0); +} + +Status EphemeralDatabasePlugin::scan(const std::string& domain, + std::vector& results, + const std::string& prefix, + size_t max) const { + if (db_.count(domain) == 0) { + return Status(0); + } + + for (const auto& key : db_.at(domain)) { + if (!prefix.empty() && + !(std::mismatch(prefix.begin(), prefix.end(), key.first.begin()) + .first == prefix.end())) { + continue; + } + results.push_back(key.first); + if (max > 0 && results.size() >= max) { + break; + } + } + return Status(0); +} +} diff --git a/osquery/database/plugins/rocksdb.cpp b/osquery/database/plugins/rocksdb.cpp new file mode 100644 index 00000000..c4828ed7 --- /dev/null +++ b/osquery/database/plugins/rocksdb.cpp @@ -0,0 +1,350 @@ +/* + * Copyright (c) 2014-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ + +#include + +#include + +#include + +#include +#include +#include + +#include +#include +#include + +namespace osquery { + +DECLARE_string(database_path); +DECLARE_bool(database_in_memory); + +class GlogRocksDBLogger : public rocksdb::Logger { + public: + // We intend to override a virtual method that is overloaded. + using rocksdb::Logger::Logv; + void Logv(const char* format, va_list ap) override; +}; + +class RocksDBDatabasePlugin : public DatabasePlugin { + public: + /// Data retrieval method. + Status get(const std::string& domain, + const std::string& key, + std::string& value) const override; + + /// Data storage method. + Status put(const std::string& domain, + const std::string& key, + const std::string& value) override; + + /// Data removal method. + Status remove(const std::string& domain, const std::string& k) override; + + /// Key/index lookup method. + Status scan(const std::string& domain, + std::vector& results, + const std::string& prefix, + size_t max = 0) const override; + + public: + /// Database workflow: open and setup. + Status setUp() override; + + /// Database workflow: close and cleanup. + void tearDown() override { close(); } + + /// Need to tear down open resources, + virtual ~RocksDBDatabasePlugin() { close(); } + + private: + /// Obtain a close lock and release resources. + void close(); + + /** + * @brief Private helper around accessing the column family handle for a + * specific column family, based on its name + */ + rocksdb::ColumnFamilyHandle* getHandleForColumnFamily( + const std::string& cf) const; + + /** + * @brief Helper method which can be used to get a raw pointer to the + * underlying RocksDB database handle + * + * @return a pointer to the underlying RocksDB database handle + */ + rocksdb::DB* getDB() const; + + private: + bool initialized_{false}; + + /// The database handle + rocksdb::DB* db_{nullptr}; + + /// RocksDB logger instance. + std::shared_ptr logger_{nullptr}; + + /// Column family descriptors which are used to connect to RocksDB + std::vector column_families_; + + /// A vector of pointers to column family handles + std::vector handles_; + + /// The RocksDB connection options that are used to connect to RocksDB + rocksdb::Options options_; + + /// Deconstruction mutex. + std::mutex close_mutex_; +}; + +/// Backing-storage provider for osquery internal/core. +REGISTER_INTERNAL(RocksDBDatabasePlugin, "database", "rocksdb"); + +void GlogRocksDBLogger::Logv(const char* format, va_list ap) { + // Convert RocksDB log to string and check if header or level-ed log. + std::string log_line; + { + char buffer[501] = {0}; + vsnprintf(buffer, 500, format, ap); + va_end(ap); + if (buffer[0] != '[' || (buffer[1] != 'E' && buffer[1] != 'W')) { + return; + } + + log_line = buffer; + } + + // There is a spurious warning on first open. + if (log_line.find("Error when reading") == std::string::npos) { + LOG(INFO) << "RocksDB: " << log_line; + } +} + +Status RocksDBDatabasePlugin::setUp() { + if (!kDBHandleOptionAllowOpen) { + LOG(WARNING) << RLOG(1629) << "Not allowed to create DBHandle instance"; + } + + if (!initialized_) { + initialized_ = true; + // Set meta-data (mostly) handling options. + options_.create_if_missing = true; + options_.create_missing_column_families = true; + options_.info_log_level = rocksdb::ERROR_LEVEL; + options_.log_file_time_to_roll = 0; + options_.keep_log_file_num = 10; + options_.max_log_file_size = 1024 * 1024 * 1; + options_.stats_dump_period_sec = 0; + + // Performance and optimization settings. + options_.compression = rocksdb::kNoCompression; + options_.compaction_style = rocksdb::kCompactionStyleLevel; + options_.arena_block_size = (4 * 1024); + options_.write_buffer_size = (4 * 1024) * 100; // 100 blocks. + options_.max_write_buffer_number = 2; + options_.min_write_buffer_number_to_merge = 1; + options_.max_background_compactions = 2; + options_.max_background_flushes = 2; + + // Create an environment to replace the default logger. + if (logger_ == nullptr) { + logger_ = std::make_shared(); + } + options_.info_log = logger_; + + column_families_.push_back(rocksdb::ColumnFamilyDescriptor( + rocksdb::kDefaultColumnFamilyName, options_)); + + for (const auto& cf_name : kDomains) { + column_families_.push_back( + rocksdb::ColumnFamilyDescriptor(cf_name, options_)); + } + } + + // Consume the current settings. + // A configuration update may change them, but that does not affect state. + path_ = FLAGS_database_path; + in_memory_ = FLAGS_database_in_memory; + + if (in_memory_) { + // Remove when MemEnv is included in librocksdb + // options_.env = rocksdb::NewMemEnv(rocksdb::Env::Default()); + return Status(1, "Cannot start in-memory RocksDB: Requires MemEnv"); + } + + if (pathExists(path_).ok() && !isReadable(path_).ok()) { + return Status(1, "Cannot read RocksDB path: " + path_); + } + + if (!DatabasePlugin::kCheckingDB) { + VLOG(1) << "Opening RocksDB handle: " << path_; + } + + // Tests may trash calls to setUp, make sure subsequent calls do not leak. + close(); + + // Attempt to create a RocksDB instance and handles. + auto s = + rocksdb::DB::Open(options_, path_, column_families_, &handles_, &db_); + if (!s.ok() || db_ == nullptr) { + if (kDBHandleOptionRequireWrite) { + // A failed open in R/W mode is a runtime error. + return Status(1, s.ToString()); + } + + if (!DatabasePlugin::kCheckingDB) { + VLOG(1) << "Opening RocksDB failed: Continuing with read-only support"; + } +#if !defined(ROCKSDB_LITE) + // RocksDB LITE does not support readonly mode. + // The database was readable but could not be opened, either (1) it is not + // writable or (2) it is already opened by another process. + // Try to open the database in a ReadOnly mode. + rocksdb::DB::OpenForReadOnly(options_, path_, column_families_, &handles_, + &db_); +#endif + // Also disable event publishers. + Flag::updateValue("disable_events", "true"); + read_only_ = true; + } + + // RocksDB may not create/append a directory with acceptable permissions. + if (!read_only_ && chmod(path_.c_str(), S_IRWXU) != 0) { + return Status(1, "Cannot set permissions on RocksDB path: " + path_); + } + return Status(0); +} + +void RocksDBDatabasePlugin::close() { + std::unique_lock lock(close_mutex_); + for (auto handle : handles_) { + delete handle; + } + handles_.clear(); + + if (db_ != nullptr) { + delete db_; + db_ = nullptr; + } +} + +rocksdb::DB* RocksDBDatabasePlugin::getDB() const { return db_; } + +rocksdb::ColumnFamilyHandle* RocksDBDatabasePlugin::getHandleForColumnFamily( + const std::string& cf) const { + try { + for (size_t i = 0; i < kDomains.size(); i++) { + if (kDomains[i] == cf) { + return handles_[i]; + } + } + } catch (const std::exception& e) { + // pass through and return nullptr + } + return nullptr; +} + +///////////////////////////////////////////////////////////////////////////// +// Data manipulation methods +///////////////////////////////////////////////////////////////////////////// + +Status RocksDBDatabasePlugin::get(const std::string& domain, + const std::string& key, + std::string& value) const { + if (getDB() == nullptr) { + return Status(1, "Database not opened"); + } + auto cfh = getHandleForColumnFamily(domain); + if (cfh == nullptr) { + return Status(1, "Could not get column family for " + domain); + } + auto s = getDB()->Get(rocksdb::ReadOptions(), cfh, key, &value); + return Status(s.code(), s.ToString()); +} + +Status RocksDBDatabasePlugin::put(const std::string& domain, + const std::string& key, + const std::string& value) { + if (read_only_) { + return Status(0, "Database in readonly mode"); + } + + auto cfh = getHandleForColumnFamily(domain); + if (cfh == nullptr) { + return Status(1, "Could not get column family for " + domain); + } + auto s = getDB()->Put(rocksdb::WriteOptions(), cfh, key, value); + if (s.code() != 0 && s.IsIOError()) { + // An error occurred, check if it is an IO error and remove the offending + // specific filename or log name. + std::string error_string = s.ToString(); + size_t error_pos = error_string.find_last_of(":"); + if (error_pos != std::string::npos) { + return Status(s.code(), "IOError: " + error_string.substr(error_pos + 2)); + } + } + return Status(s.code(), s.ToString()); +} + +Status RocksDBDatabasePlugin::remove(const std::string& domain, + const std::string& key) { + if (read_only_) { + return Status(0, "Database in readonly mode"); + } + + auto cfh = getHandleForColumnFamily(domain); + if (cfh == nullptr) { + return Status(1, "Could not get column family for " + domain); + } + auto options = rocksdb::WriteOptions(); + + // We could sync here, but large deletes will cause multi-syncs. + // For example: event record expirations found in an expired index. + // options.sync = true; + auto s = getDB()->Delete(options, cfh, key); + return Status(s.code(), s.ToString()); +} + +Status RocksDBDatabasePlugin::scan(const std::string& domain, + std::vector& results, + const std::string& prefix, + size_t max) const { + if (getDB() == nullptr) { + return Status(1, "Database not opened"); + } + + auto cfh = getHandleForColumnFamily(domain); + if (cfh == nullptr) { + return Status(1, "Could not get column family for " + domain); + } + auto options = rocksdb::ReadOptions(); + options.verify_checksums = false; + options.fill_cache = false; + auto it = getDB()->NewIterator(rocksdb::ReadOptions(), cfh); + if (it == nullptr) { + return Status(1, "Could not get iterator for " + domain); + } + + size_t count = 0; + for (it->SeekToFirst(); it->Valid(); it->Next()) { + auto key = it->key().ToString(); + if (key.find(prefix) == 0) { + results.push_back(std::move(key)); + if (max > 0 && ++count >= max) { + break; + } + } + } + delete it; + return Status(0, "OK"); +} +} diff --git a/osquery/database/plugins/sqlite.cpp b/osquery/database/plugins/sqlite.cpp new file mode 100644 index 00000000..97a110cc --- /dev/null +++ b/osquery/database/plugins/sqlite.cpp @@ -0,0 +1,277 @@ +/* + * Copyright (c) 2014-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ + +#include + +#include + +#include + +#include +#include +#include + +namespace osquery { + +DECLARE_string(database_path); +DECLARE_bool(database_in_memory); + +const std::map kDBSettings = { + {"synchronous", "OFF"}, {"count_changes", "OFF"}, + {"default_temp_store", "2"}, {"auto_vacuum", "FULL"}, + {"journal_mode", "OFF"}, {"cache_size", "1000"}, + {"page_count", "1000"}, +}; + +class SQLiteDatabasePlugin : public DatabasePlugin { + public: + /// Data retrieval method. + Status get(const std::string& domain, + const std::string& key, + std::string& value) const override; + + /// Data storage method. + Status put(const std::string& domain, + const std::string& key, + const std::string& value) override; + + /// Data removal method. + Status remove(const std::string& domain, const std::string& k) override; + + /// Key/index lookup method. + Status scan(const std::string& domain, + std::vector& results, + const std::string& prefix, + size_t max = 0) const override; + + public: + /// Database workflow: open and setup. + Status setUp() override; + + /// Database workflow: close and cleanup. + void tearDown() override { close(); } + + /// Need to tear down open resources, + virtual ~SQLiteDatabasePlugin() { close(); } + + private: + void close(); + + private: + /// The long-lived sqlite3 database. + sqlite3* db_{nullptr}; + + /// Deconstruction mutex. + std::mutex close_mutex_; +}; + +/// Backing-storage provider for osquery internal/core. +REGISTER_INTERNAL(SQLiteDatabasePlugin, "database", "sqlite"); + +Status SQLiteDatabasePlugin::setUp() { + if (!DatabasePlugin::kDBHandleOptionAllowOpen) { + LOG(WARNING) << RLOG(1629) << "Not allowed to create DBHandle instance"; + } + + // Consume the current settings. + // A configuration update may change them, but that does not affect state. + path_ = FLAGS_database_path; + in_memory_ = FLAGS_database_in_memory; + + if (!in_memory_ && pathExists(path_).ok() && !isReadable(path_).ok()) { + return Status(1, "Cannot read database path: " + path_); + } + + if (!DatabasePlugin::kCheckingDB) { + VLOG(1) << "Opening database handle: " << path_; + } + + // Tests may trash calls to setUp, make sure subsequent calls do not leak. + close(); + + // Can actually try to create a DB in memory with: in_memory_. + // Open the SQLite backing storage at path_ + auto result = sqlite3_open_v2( + ((in_memory_) ? ":memory:" : path_.c_str()), + &db_, + (SQLITE_OPEN_FULLMUTEX | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE), + nullptr); + + if (result != SQLITE_OK || db_ == nullptr) { + if (DatabasePlugin::kDBHandleOptionRequireWrite) { + close(); + // A failed open in R/W mode is a runtime error. + return Status(1, "Cannot open database: " + std::to_string(result)); + } + + if (!DatabasePlugin::kCheckingDB) { + VLOG(1) << "Opening database failed: Continuing with read-only support"; + } + + read_only_ = true; + } + + if (!read_only_) { + for (const auto& domain : kDomains) { + std::string q = "create table if not exists " + domain + + " (key TEXT PRIMARY KEY, value TEXT);"; + result = sqlite3_exec(db_, q.c_str(), nullptr, nullptr, nullptr); + if (result != SQLITE_OK) { + close(); + return Status(1, "Cannot create domain: " + domain); + } + } + + std::string settings; + for (const auto& setting : kDBSettings) { + settings += "PRAGMA " + setting.first + "=" + setting.second + "; "; + } + sqlite3_exec(db_, settings.c_str(), nullptr, nullptr, nullptr); + } + + // RocksDB may not create/append a directory with acceptable permissions. + if (!read_only_ && chmod(path_.c_str(), S_IRWXU) != 0) { + close(); + return Status(1, "Cannot set permissions on database path: " + path_); + } + return Status(0); +} + +void SQLiteDatabasePlugin::close() { + std::unique_lock lock(close_mutex_); + if (db_ != nullptr) { + sqlite3_close(db_); + db_ = nullptr; + } +} + +static int getData(void* argument, int argc, char* argv[], char* column[]) { + if (argument == nullptr) { + return SQLITE_MISUSE; + } + + QueryData* qData = (QueryData*)argument; + Row r; + for (int i = 0; i < argc; i++) { + if (column[i] != nullptr) { + r[column[i]] = (argv[i] != nullptr) ? argv[i] : ""; + } + } + (*qData).push_back(std::move(r)); + return 0; +} + +Status SQLiteDatabasePlugin::get(const std::string& domain, + const std::string& key, + std::string& value) const { + QueryData results; + char* err = nullptr; + std::string q = "select value from " + domain + " where key = '" + key + "';"; + sqlite3_exec(db_, q.c_str(), getData, &results, &err); + if (err != nullptr) { + sqlite3_free(err); + } + + // Only assign value if the query found a result. + if (results.size() > 0) { + value = std::move(results[0]["value"]); + return Status(0); + } + return Status(1); +} + +static void tryVacuum(sqlite3* db) { + std::string q = + "SELECT (sum(s1.pageno + 1 == s2.pageno) * 1.0 / count(*)) < 0.01 as v " + " FROM " + "(SELECT pageno FROM dbstat ORDER BY path) AS s1," + "(SELECT pageno FROM dbstat ORDER BY path) AS s2 WHERE " + "s1.rowid + 1 = s2.rowid; "; + + QueryData results; + sqlite3_exec(db, q.c_str(), getData, &results, nullptr); + if (results.size() > 0 && results[0]["v"].back() == '1') { + sqlite3_exec(db, "vacuum;", nullptr, nullptr, nullptr); + } +} + +Status SQLiteDatabasePlugin::put(const std::string& domain, + const std::string& key, + const std::string& value) { + if (read_only_) { + return Status(0, "Database in readonly mode"); + } + + sqlite3_stmt* stmt = nullptr; + std::string q = "insert or replace into " + domain + " values (?1, ?2);"; + sqlite3_prepare_v2(db_, q.c_str(), -1, &stmt, nullptr); + + sqlite3_bind_text(stmt, 1, key.c_str(), -1, SQLITE_STATIC); + sqlite3_bind_text(stmt, 2, value.c_str(), -1, SQLITE_STATIC); + auto rc = sqlite3_step(stmt); + if (rc != SQLITE_DONE) { + return Status(1); + } + + sqlite3_finalize(stmt); + if (rand() % 10 == 0) { + tryVacuum(db_); + } + return Status(0); +} + +Status SQLiteDatabasePlugin::remove(const std::string& domain, + const std::string& key) { + if (read_only_) { + return Status(0, "Database in readonly mode"); + } + + sqlite3_stmt* stmt = nullptr; + std::string q = "delete from " + domain + " where key IN (?1);"; + sqlite3_prepare_v2(db_, q.c_str(), -1, &stmt, nullptr); + + sqlite3_bind_text(stmt, 1, key.c_str(), -1, SQLITE_STATIC); + auto rc = sqlite3_step(stmt); + if (rc != SQLITE_DONE) { + return Status(1); + } + + sqlite3_finalize(stmt); + if (rand() % 10 == 0) { + tryVacuum(db_); + } + return Status(0); +} + +Status SQLiteDatabasePlugin::scan(const std::string& domain, + std::vector& results, + const std::string& prefix, + size_t max) const { + QueryData _results; + char* err = nullptr; + + std::string q = + "select key from " + domain + " where key LIKE '" + prefix + "%'"; + if (max > 0) { + q += " limit " + std::to_string(max); + } + sqlite3_exec(db_, q.c_str(), getData, &_results, &err); + if (err != nullptr) { + sqlite3_free(err); + } + + // Only assign value if the query found a result. + for (auto& r : _results) { + results.push_back(std::move(r["key"])); + } + + return Status(0, "OK"); +} +} diff --git a/osquery/database/plugins/tests/rocksdb_tests.cpp b/osquery/database/plugins/tests/rocksdb_tests.cpp new file mode 100644 index 00000000..20b15f1d --- /dev/null +++ b/osquery/database/plugins/tests/rocksdb_tests.cpp @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2014-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ + +#include + +#include "osquery/database/tests/plugin_tests.h" + +namespace osquery { + +class RocksDBDatabasePluginTests : public DatabasePluginTests { + protected: + std::string name() override { return "rocksdb"; } +}; + +// Define the default set of database plugin operation tests. +CREATE_DATABASE_TESTS(RocksDBDatabasePluginTests); + +TEST_F(RocksDBDatabasePluginTests, test_rocksdb_loglevel) { + // Make sure a log file was created. + EXPECT_FALSE(pathExists(path_ + "/LOG")); + + // Make sure no log file is created. + // RocksDB logs are intercepted and forwarded to the GLog sink. + auto details = SQL::selectAllFrom("file", "path", EQUALS, path_ + "/LOG"); + ASSERT_EQ(details.size(), 0U); +} +} diff --git a/osquery/database/plugins/tests/sqlite_tests.cpp b/osquery/database/plugins/tests/sqlite_tests.cpp new file mode 100644 index 00000000..9d7276ed --- /dev/null +++ b/osquery/database/plugins/tests/sqlite_tests.cpp @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2014-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ + +#include "osquery/database/tests/plugin_tests.h" + +namespace osquery { + +class SQLiteDatabasePluginTests : public DatabasePluginTests { + protected: + std::string name() override { return "sqlite"; } +}; + +// Define the default set of database plugin operation tests. +CREATE_DATABASE_TESTS(SQLiteDatabasePluginTests); +} diff --git a/osquery/database/query.cpp b/osquery/database/query.cpp index 4ca4a0f9..1c5d5953 100644 --- a/osquery/database/query.cpp +++ b/osquery/database/query.cpp @@ -14,21 +14,13 @@ namespace osquery { -///////////////////////////////////////////////////////////////////////////// -// Data access methods -///////////////////////////////////////////////////////////////////////////// - Status Query::getPreviousQueryResults(QueryData& results) { - return getPreviousQueryResults(results, DBHandle::getInstance()); -} - -Status Query::getPreviousQueryResults(QueryData& results, DBHandleRef db) { if (!isQueryNameInDatabase()) { return Status(0, "Query name not found in database"); } std::string raw; - auto status = db->Get(kQueries, name_, raw); + auto status = getDatabaseValue(kQueries, name_, raw); if (!status.ok()) { return status; } @@ -41,41 +33,28 @@ Status Query::getPreviousQueryResults(QueryData& results, DBHandleRef db) { } std::vector Query::getStoredQueryNames() { - return getStoredQueryNames(DBHandle::getInstance()); -} - -std::vector Query::getStoredQueryNames(DBHandleRef db) { std::vector results; - db->Scan(kQueries, results); + scanDatabaseKeys(kQueries, results); return results; } bool Query::isQueryNameInDatabase() { - return isQueryNameInDatabase(DBHandle::getInstance()); -} - -bool Query::isQueryNameInDatabase(DBHandleRef db) { - auto names = Query::getStoredQueryNames(db); + auto names = Query::getStoredQueryNames(); return std::find(names.begin(), names.end(), name_) != names.end(); } -Status Query::addNewResults(const osquery::QueryData& qd) { - return addNewResults(qd, DBHandle::getInstance()); -} - -Status Query::addNewResults(const QueryData& qd, DBHandleRef db) { +Status Query::addNewResults(const QueryData& qd) { DiffResults dr; - return addNewResults(qd, dr, false, db); + return addNewResults(qd, dr, false); } Status Query::addNewResults(const QueryData& qd, DiffResults& dr) { - return addNewResults(qd, dr, true, DBHandle::getInstance()); + return addNewResults(qd, dr, true); } Status Query::addNewResults(const QueryData& current_qd, DiffResults& dr, - bool calculate_diff, - DBHandleRef db) { + bool calculate_diff) { // Get the rows from the last run of this query name. QueryData previous_qd; auto status = getPreviousQueryResults(previous_qd); @@ -97,7 +76,7 @@ Status Query::addNewResults(const QueryData& current_qd, return status; } - status = db->Put(kQueries, name_, json); + status = setDatabaseValue(kQueries, name_, json); if (!status.ok()) { return status; } diff --git a/osquery/database/query.h b/osquery/database/query.h index 302e436f..6701d4c2 100644 --- a/osquery/database/query.h +++ b/osquery/database/query.h @@ -14,10 +14,8 @@ #include #include -#include #include - -#include "osquery/database/db_handle.h" +#include namespace osquery { @@ -58,23 +56,6 @@ class Query { */ Status getPreviousQueryResults(QueryData& results); - private: - /** - * @brief Serialize the data in RocksDB into a useful data structure using a - * custom database handle - * - * This method is the same as getHistoricalQueryResults, but with the - * addition of a parameter which allows you to pass a custom RocksDB - * database handle. - * - * @param hQR the output HistoricalQueryResults struct - * @param db a shared pointer to a custom DBHandle - * - * @return the success or failure of the operation - * @see getHistoricalQueryResults - */ - Status getPreviousQueryResults(QueryData& results, DBHandleRef db); - public: /** * @brief Get the names of all historical queries that are stored in RocksDB @@ -88,22 +69,6 @@ class Query { */ static std::vector getStoredQueryNames(); - private: - /** - * @brief Get the names of all historical queries that are stored in RocksDB - * using a custom database handle - * - * This method is the same as getStoredQueryNames(), but with the addition - * of a parameter which allows you to pass a custom RocksDB database handle. - * - * @param db a custom RocksDB database handle - * - * @return a vector containing the string names of all scheduled queries - * - * @see getStoredQueryNames() - */ - static std::vector getStoredQueryNames(DBHandleRef db); - public: /** * @brief Accessor method for checking if a given scheduled query exists in @@ -113,20 +78,6 @@ class Query { */ bool isQueryNameInDatabase(); - private: - /** - * @brief Accessor method for checking if a given scheduled query exists in - * the database, using a custom database handle - * - * This method is the same as isQueryNameInDatabase(), but with the addition - * of a parameter which allows you to pass a custom RocksDB database handle - * - * @param db a custom RocksDB database handle - * - * @return does the scheduled query which is already exists in the database - */ - bool isQueryNameInDatabase(DBHandleRef db); - public: /** * @brief Add a new set of results to the persistant storage @@ -143,24 +94,6 @@ class Query { */ Status addNewResults(const QueryData& qd); - private: - /** - * @brief Add a new set of results to the persistant storage using a custom - * database handle - * - * This method is the same as addNewResults(), but with the addition of a - * parameter which allows you to pass a custom RocksDB database handle - * - * @param qd the QueryData object, which has the results of the query which - * you would like to store - * @param unix_time the time that the query was executed - * @param db a custom RocksDB database handle - * - * @return an instance of osquery::Status indicating the success or failure - * of the operation - */ - Status addNewResults(const QueryData& qd, DBHandleRef db); - public: /** * @brief Add a new set of results to the persistent storage and get back @@ -192,8 +125,7 @@ class Query { */ Status addNewResults(const QueryData& qd, DiffResults& dr, - bool calculate_diff, - DBHandleRef db); + bool calculate_diff); public: /** @@ -205,22 +137,6 @@ class Query { */ Status getCurrentResults(QueryData& qd); - private: - /** - * @brief A getter for the most recent result set for a scheduled query, - * but with the addition of a parameter which allows you to pass a custom - * RocksDB database handle. - * - * This method is the same as Query::getCurrentResults, but with addition of a - * parameter which allows you to pass a custom RocksDB database handle. - * - * @param qd the output QueryData object - * @param db a custom RocksDB database handle - * - * @return the success or failure of the operation - */ - Status getCurrentResults(QueryData& qd, DBHandleRef db); - private: ///////////////////////////////////////////////////////////////////////////// // Private members @@ -228,6 +144,7 @@ class Query { /// The scheduled query and internal ScheduledQuery query_; + /// The scheduled query name. std::string name_; diff --git a/osquery/database/tests/db_handle_tests.cpp b/osquery/database/tests/db_handle_tests.cpp deleted file mode 100644 index e45223e1..00000000 --- a/osquery/database/tests/db_handle_tests.cpp +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright (c) 2014-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - * - */ - -#include - -#include - -#include - -#include -#include -#include -#include -#include - -#include "osquery/database/db_handle.h" -#include "osquery/core/test_util.h" - -namespace osquery { - -class DBHandleTests : public testing::Test { - public: - void SetUp() { - // A database instance is setup during testing initialize (initTesting). - // We need to reset that instance to test unordered expectations. - path_ = kTestWorkingDirectory + std::to_string(rand() % 10000 + 20000); - DBHandle::getInstance()->resetInstance(path_, false); - - cfh_queries_ = DBHandle::getInstance()->getHandleForColumnFamily(kQueries); - cfh_foobar_ = - DBHandle::getInstance()->getHandleForColumnFamily("foobartest"); - db_ = DBHandle::getInstance(); - } - - void TearDown() { - // Clean the transient instance and reset to the testing instance. - boost::filesystem::remove_all(path_); - auto path = Flag::getValue("database_path"); - DBHandle::getInstance()->resetInstance(path, false); - } - - public: - std::string path_; - rocksdb::ColumnFamilyHandle* cfh_queries_; - rocksdb::ColumnFamilyHandle* cfh_foobar_; - std::shared_ptr db_; -}; - -TEST_F(DBHandleTests, test_singleton_on_disk) { - auto db1 = DBHandle::getInstance(); - auto db2 = DBHandle::getInstance(); - EXPECT_EQ(db1, db2); -} - -TEST_F(DBHandleTests, test_get_handle_for_column_family) { - ASSERT_TRUE(cfh_queries_ != nullptr); - ASSERT_TRUE(cfh_foobar_ == nullptr); -} - -TEST_F(DBHandleTests, test_get) { - db_->getDB()->Put( - rocksdb::WriteOptions(), cfh_queries_, "test_query_123", "{}"); - std::string r; - std::string key = "test_query_123"; - auto s = db_->Get(kQueries, key, r); - EXPECT_TRUE(s.ok()); - EXPECT_EQ(s.toString(), "OK"); - EXPECT_EQ(r, "{}"); -} - -TEST_F(DBHandleTests, test_put) { - auto s = db_->Put(kQueries, "test_put", "bar"); - EXPECT_TRUE(s.ok()); - EXPECT_EQ(s.toString(), "OK"); -} - -TEST_F(DBHandleTests, test_delete) { - db_->Put(kQueries, "test_delete", "baz"); - auto s = db_->Delete(kQueries, "test_delete"); - EXPECT_TRUE(s.ok()); - EXPECT_EQ(s.toString(), "OK"); -} - -TEST_F(DBHandleTests, test_scan) { - db_->Put(kQueries, "test_scan_foo1", "baz"); - db_->Put(kQueries, "test_scan_foo2", "baz"); - db_->Put(kQueries, "test_scan_foo3", "baz"); - - std::vector keys; - std::vector expected = { - "test_scan_foo1", "test_scan_foo2", "test_scan_foo3"}; - auto s = db_->Scan(kQueries, keys); - EXPECT_TRUE(s.ok()); - EXPECT_EQ(s.toString(), "OK"); - EXPECT_EQ(keys.size(), 3U); - for (const auto& i : expected) { - EXPECT_NE(std::find(keys.begin(), keys.end(), i), keys.end()); - } -} - -TEST_F(DBHandleTests, test_scan_limit) { - db_->Put(kQueries, "test_scan_foo1", "baz"); - db_->Put(kQueries, "test_scan_foo2", "baz"); - db_->Put(kQueries, "test_scan_foo3", "baz"); - - std::vector keys; - auto s = db_->Scan(kQueries, keys, 2); - EXPECT_TRUE(s.ok()); - EXPECT_EQ(s.toString(), "OK"); - EXPECT_EQ(keys.size(), 2U); -} - -TEST_F(DBHandleTests, test_rocksdb_loglevel) { - // Make sure a log file was created. - EXPECT_FALSE(pathExists(path_ + "/LOG")); - - // Make sure no log file is created. - // RocksDB logs are intercepted and forwarded to the GLog sink. - auto details = SQL::selectAllFrom("file", "path", EQUALS, path_ + "/LOG"); - ASSERT_EQ(details.size(), 0U); -} -} diff --git a/osquery/database/tests/plugin_tests.cpp b/osquery/database/tests/plugin_tests.cpp new file mode 100644 index 00000000..5000a46a --- /dev/null +++ b/osquery/database/tests/plugin_tests.cpp @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2014-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ + +#include "osquery/database/tests/plugin_tests.h" + +namespace osquery { + +class EphemeralDatabasePluginTests : public DatabasePluginTests { + protected: + std::string name() override { return "ephemeral"; } +}; + +// Define the default set of database plugin operation tests. +CREATE_DATABASE_TESTS(EphemeralDatabasePluginTests); + +void DatabasePluginTests::testPluginCheck() { + // Do not worry about multiple set-active calls. + // For testing purposes they should be idempotent. + EXPECT_TRUE(Registry::setActive("database", getName())); + + // Get an instance of the database plugin and call check. + auto plugin = Registry::get("database", getName()); + auto db_plugin = std::dynamic_pointer_cast(plugin); + EXPECT_TRUE(db_plugin->checkDB()); + + // Testing relies on database resetting too. + EXPECT_TRUE(db_plugin->reset()); +} + +void DatabasePluginTests::testPut() { + auto s = getPlugin()->put(kQueries, "test_put", "bar"); + EXPECT_TRUE(s.ok()); + EXPECT_EQ(s.getMessage(), "OK"); +} + +void DatabasePluginTests::testGet() { + getPlugin()->put(kQueries, "test_get", "bar"); + + std::string r; + auto s = getPlugin()->get(kQueries, "test_get", r); + EXPECT_TRUE(s.ok()); + EXPECT_EQ(s.getMessage(), "OK"); + EXPECT_EQ(r, "bar"); +} + +void DatabasePluginTests::testDelete() { + getPlugin()->put(kQueries, "test_delete", "baz"); + auto s = getPlugin()->remove(kQueries, "test_delete"); + EXPECT_TRUE(s.ok()); + EXPECT_EQ(s.getMessage(), "OK"); +} + +void DatabasePluginTests::testScan() { + getPlugin()->put(kQueries, "test_scan_foo1", "baz"); + getPlugin()->put(kQueries, "test_scan_foo2", "baz"); + getPlugin()->put(kQueries, "test_scan_foo3", "baz"); + + std::vector keys; + std::vector expected = {"test_scan_foo1", "test_scan_foo2", + "test_scan_foo3"}; + auto s = getPlugin()->scan(kQueries, keys, ""); + EXPECT_TRUE(s.ok()); + EXPECT_EQ(s.getMessage(), "OK"); + EXPECT_EQ(keys.size(), 3U); + for (const auto& i : expected) { + EXPECT_NE(std::find(keys.begin(), keys.end(), i), keys.end()); + } +} + +void DatabasePluginTests::testScanLimit() { + getPlugin()->put(kQueries, "test_scan_foo1", "baz"); + getPlugin()->put(kQueries, "test_scan_foo2", "baz"); + getPlugin()->put(kQueries, "test_scan_foo3", "baz"); + + std::vector keys; + auto s = getPlugin()->scan(kQueries, keys, "", 2); + EXPECT_TRUE(s.ok()); + EXPECT_EQ(s.getMessage(), "OK"); + EXPECT_EQ(keys.size(), 2U); +} +} diff --git a/osquery/database/tests/plugin_tests.h b/osquery/database/tests/plugin_tests.h new file mode 100644 index 00000000..cf670ab9 --- /dev/null +++ b/osquery/database/tests/plugin_tests.h @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2014-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ + +#pragma once + +#include + +#include + +#include +#include + +#include "osquery/core/test_util.h" + +/// The following test macros allow pretty test output. +#define CREATE_DATABASE_TESTS(n) \ + TEST_F(n, test_plugin_check) { testPluginCheck(); } \ + TEST_F(n, test_put) { testPut(); } \ + TEST_F(n, test_get) { testGet(); } \ + TEST_F(n, test_delete) { testDelete(); } \ + TEST_F(n, test_scan) { testScan(); } \ + TEST_F(n, test_scan_limit) { testScanLimit(); } + +namespace osquery { + +DECLARE_string(database_path); + +class DatabasePluginTests : public testing::Test { + public: + void SetUp() override { + existing_plugin_ = Registry::getActive("database"); + Registry::get("database", existing_plugin_)->tearDown(); + + setName(name()); + path_ = FLAGS_database_path; + boost::filesystem::remove_all(path_); + + auto plugin = Registry::get("database", getName()); + plugin_ = std::dynamic_pointer_cast(plugin); + plugin_->reset(); + } + + void TearDown() override { + Registry::get("database", name_)->tearDown(); + Registry::setActive("database", existing_plugin_); + } + + protected: + /// Path to testing database. + std::string path_; + + protected: + /// Require each plugin tester to implement a set name. + virtual std::string name() = 0; + + private: + void setName(const std::string& name) { name_ = name; } + const std::string& getName() { return name_; } + std::shared_ptr getPlugin() { return plugin_; } + + private: + /// Plugin name + std::string name_; + + /// Plugin casted from setUp, ready to run tests. + std::shared_ptr plugin_{nullptr}; + + /// Previous active database plugin. + std::string existing_plugin_; + + protected: + void testPluginCheck(); + void testPut(); + void testGet(); + void testDelete(); + void testScan(); + void testScanLimit(); +}; +} diff --git a/osquery/database/tests/query_tests.cpp b/osquery/database/tests/query_tests.cpp index 21aec9e5..f4459ff0 100644 --- a/osquery/database/tests/query_tests.cpp +++ b/osquery/database/tests/query_tests.cpp @@ -19,18 +19,9 @@ #include "osquery/core/test_util.h" #include "osquery/database/query.h" -const std::string kTestingQueryDBPath = "/tmp/rocksdb-osquery-querytests"; - namespace osquery { -class QueryTests : public testing::Test { - public: - void SetUp() { db_ = DBHandle::getInstanceAtPath(kTestingQueryDBPath); } - void TearDown() { boost::filesystem::remove_all(kTestingQueryDBPath); } - - public: - std::shared_ptr db_; -}; +class QueryTests : public testing::Test {}; TEST_F(QueryTests, test_private_members) { auto query = getOsqueryScheduledQuery(); @@ -42,21 +33,21 @@ TEST_F(QueryTests, test_add_and_get_current_results) { // Test adding a "current" set of results to a scheduled query instance. auto query = getOsqueryScheduledQuery(); auto cf = Query("foobar", query); - auto status = cf.addNewResults(getTestDBExpectedResults(), db_); + auto status = cf.addNewResults(getTestDBExpectedResults()); EXPECT_TRUE(status.ok()); EXPECT_EQ(status.toString(), "OK"); // Simulate results from several schedule runs, calculate differentials. for (auto result : getTestDBResultStream()) { - // Get the results from the previous query execution (from RocksDB). + // Get the results from the previous query execution (from the DB). QueryData previous_qd; - auto status = cf.getPreviousQueryResults(previous_qd, db_); + auto status = cf.getPreviousQueryResults(previous_qd); EXPECT_TRUE(status.ok()); EXPECT_EQ(status.toString(), "OK"); // Add the "current" results and output the differentials. DiffResults dr; - auto s = cf.addNewResults(result.second, dr, true, db_); + auto s = cf.addNewResults(result.second, dr, true); EXPECT_TRUE(s.ok()); // Call the diffing utility directly. @@ -65,7 +56,7 @@ TEST_F(QueryTests, test_add_and_get_current_results) { // After Query::addNewResults the previous results are now current. QueryData qd; - cf.getPreviousQueryResults(qd, db_); + cf.getPreviousQueryResults(qd); EXPECT_EQ(qd, result.second); } } @@ -74,13 +65,13 @@ TEST_F(QueryTests, test_get_query_results) { // Grab an expected set of query data and add it as the previous result. auto encoded_qd = getSerializedQueryDataJSON(); auto query = getOsqueryScheduledQuery(); - auto status = db_->Put(kQueries, "foobar", encoded_qd.first); + auto status = setDatabaseValue(kQueries, "foobar", encoded_qd.first); EXPECT_TRUE(status.ok()); // Use the Query retrieval API to check the now "previous" result. QueryData previous_qd; auto cf = Query("foobar", query); - status = cf.getPreviousQueryResults(previous_qd, db_); + status = cf.getPreviousQueryResults(previous_qd); EXPECT_TRUE(status.ok()); } @@ -89,31 +80,31 @@ TEST_F(QueryTests, test_query_name_not_found_in_db) { QueryData previous_qd; auto query = getOsqueryScheduledQuery(); auto cf = Query("not_a_real_query", query); - auto status = cf.getPreviousQueryResults(previous_qd, db_); - EXPECT_TRUE(status.toString() == "Query name not found in database"); + auto status = cf.getPreviousQueryResults(previous_qd); EXPECT_TRUE(status.ok()); + EXPECT_EQ(status.toString(), "Query name not found in database"); } TEST_F(QueryTests, test_is_query_name_in_database) { auto query = getOsqueryScheduledQuery(); auto cf = Query("foobar", query); auto encoded_qd = getSerializedQueryDataJSON(); - auto status = db_->Put(kQueries, "foobar", encoded_qd.first); + auto status = setDatabaseValue(kQueries, "foobar", encoded_qd.first); EXPECT_TRUE(status.ok()); // Now test that the query name exists. - EXPECT_TRUE(cf.isQueryNameInDatabase(db_)); + EXPECT_TRUE(cf.isQueryNameInDatabase()); } TEST_F(QueryTests, test_get_stored_query_names) { auto query = getOsqueryScheduledQuery(); auto cf = Query("foobar", query); auto encoded_qd = getSerializedQueryDataJSON(); - auto status = db_->Put(kQueries, "foobar", encoded_qd.first); + auto status = setDatabaseValue(kQueries, "foobar", encoded_qd.first); EXPECT_TRUE(status.ok()); // Stored query names is a factory method included alongside every query. // It will include the set of query names with existing "previous" results. - auto names = cf.getStoredQueryNames(db_); + auto names = cf.getStoredQueryNames(); auto in_vector = std::find(names.begin(), names.end(), "foobar"); EXPECT_NE(in_vector, names.end()); } diff --git a/osquery/database/tests/results_tests.cpp b/osquery/database/tests/results_tests.cpp index f782a5b3..5ef2716b 100644 --- a/osquery/database/tests/results_tests.cpp +++ b/osquery/database/tests/results_tests.cpp @@ -136,111 +136,6 @@ TEST_F(ResultsTests, test_deserialize_query_log_item_json) { EXPECT_EQ(output, results.second); } -TEST_F(ResultsTests, test_serialize_distributed_query_request) { - DistributedQueryRequest r; - r.query = "foo"; - r.id = "bar"; - - pt::ptree tree; - auto s = serializeDistributedQueryRequest(r, tree); - EXPECT_TRUE(s.ok()); - EXPECT_EQ(tree.get("query"), "foo"); - EXPECT_EQ(tree.get("id"), "bar"); -} - -TEST_F(ResultsTests, test_deserialize_distributed_query_request) { - pt::ptree tree; - tree.put("query", "foo"); - tree.put("id", "bar"); - - DistributedQueryRequest r; - auto s = deserializeDistributedQueryRequest(tree, r); - EXPECT_TRUE(s.ok()); - EXPECT_EQ(r.query, "foo"); - EXPECT_EQ(r.id, "bar"); -} - -TEST_F(ResultsTests, test_deserialize_distributed_query_request_json) { - auto json = - "{" - " \"query\": \"foo\"," - " \"id\": \"bar\"" - "}"; - - DistributedQueryRequest r; - auto s = deserializeDistributedQueryRequestJSON(json, r); - EXPECT_TRUE(s.ok()); - EXPECT_EQ(r.query, "foo"); - EXPECT_EQ(r.id, "bar"); -} - -TEST_F(ResultsTests, test_serialize_distributed_query_result) { - DistributedQueryResult r; - r.request.query = "foo"; - r.request.id = "bar"; - - Row r1; - r1["foo"] = "bar"; - r.results = {r1}; - - pt::ptree tree; - auto s = serializeDistributedQueryResult(r, tree); - EXPECT_TRUE(s.ok()); - EXPECT_EQ(tree.get("request.query"), "foo"); - EXPECT_EQ(tree.get("request.id"), "bar"); - auto& results = tree.get_child("results"); - for (const auto& q : results) { - for (const auto& row : q.second) { - EXPECT_EQ(row.first, "foo"); - EXPECT_EQ(q.second.get(row.first), "bar"); - } - } -} - -TEST_F(ResultsTests, test_deserialize_distributed_query_result) { - pt::ptree request; - request.put("id", "foo"); - request.put("query", "bar"); - - pt::ptree row; - row.put("foo", "bar"); - pt::ptree results; - results.push_back(std::make_pair("", row)); - - pt::ptree query_result; - query_result.put_child("request", request); - query_result.put_child("results", results); - - DistributedQueryResult r; - auto s = deserializeDistributedQueryResult(query_result, r); - EXPECT_TRUE(s.ok()); - EXPECT_EQ(r.request.id, "foo"); - EXPECT_EQ(r.request.query, "bar"); - EXPECT_EQ(r.results[0]["foo"], "bar"); -} - -TEST_F(ResultsTests, test_deserialize_distributed_query_result_json) { - auto json = - "{" - " \"request\": {" - " \"id\": \"foo\"," - " \"query\": \"bar\"" - " }," - " \"results\": [" - " {" - " \"foo\": \"bar\"" - " }" - " ]" - "}"; - - DistributedQueryResult r; - auto s = deserializeDistributedQueryResultJSON(json, r); - EXPECT_TRUE(s.ok()); - EXPECT_EQ(r.request.id, "foo"); - EXPECT_EQ(r.request.query, "bar"); - EXPECT_EQ(r.results[0]["foo"], "bar"); -} - TEST_F(ResultsTests, test_adding_duplicate_rows_to_query_data) { Row r1, r2, r3; r1["foo"] = "bar"; diff --git a/osquery/devtools/shell.cpp b/osquery/devtools/shell.cpp index 38ad84d4..eec1d2b7 100644 --- a/osquery/devtools/shell.cpp +++ b/osquery/devtools/shell.cpp @@ -15,11 +15,11 @@ #include #include -#include #include +#include -#include #include +#include #include diff --git a/osquery/dispatcher/scheduler.cpp b/osquery/dispatcher/scheduler.cpp index c030f059..845b84a7 100644 --- a/osquery/dispatcher/scheduler.cpp +++ b/osquery/dispatcher/scheduler.cpp @@ -47,8 +47,8 @@ inline SQL monitor(const std::string& name, const ScheduledQuery& query) { } } // Always called while processes table is working. - Config::getInstance().recordQueryPerformance( - name, t1 - t0, size, r0[0], r1[0]); + Config::getInstance().recordQueryPerformance(name, t1 - t0, size, r0[0], + r1[0]); } return sql; } diff --git a/osquery/distributed/distributed.cpp b/osquery/distributed/distributed.cpp index 8e678789..62cdfdb4 100644 --- a/osquery/distributed/distributed.cpp +++ b/osquery/distributed/distributed.cpp @@ -18,7 +18,6 @@ #include #include - namespace pt = boost::property_tree; namespace osquery { @@ -187,4 +186,124 @@ DistributedQueryRequest Distributed::popRequest() { queries_.erase(queries_.begin()); return q; } + +Status serializeDistributedQueryRequest(const DistributedQueryRequest& r, + pt::ptree& tree) { + tree.put("query", r.query); + tree.put("id", r.id); + return Status(0, "OK"); +} + +Status serializeDistributedQueryRequestJSON(const DistributedQueryRequest& r, + std::string& json) { + pt::ptree tree; + auto s = serializeDistributedQueryRequest(r, tree); + if (!s.ok()) { + return s; + } + std::stringstream ss; + try { + pt::write_json(ss, tree, false); + } catch (const pt::ptree_error& e) { + return Status(1, "Error serializing JSON: " + std::string(e.what())); + } + json = ss.str(); + + return Status(0, "OK"); +} + +Status deserializeDistributedQueryRequest(const pt::ptree& tree, + DistributedQueryRequest& r) { + r.query = tree.get("query", ""); + r.id = tree.get("id", ""); + return Status(0, "OK"); +} + +Status deserializeDistributedQueryRequestJSON(const std::string& json, + DistributedQueryRequest& r) { + std::stringstream ss(json); + pt::ptree tree; + try { + pt::read_json(ss, tree); + } catch (const pt::ptree_error& e) { + return Status(1, "Error serializing JSON: " + std::string(e.what())); + } + return deserializeDistributedQueryRequest(tree, r); +} + +///////////////////////////////////////////////////////////////////////////// +// DistributedQueryResult - small struct containing the results of a +// distributed query +///////////////////////////////////////////////////////////////////////////// + +Status serializeDistributedQueryResult(const DistributedQueryResult& r, + pt::ptree& tree) { + pt::ptree request; + auto s = serializeDistributedQueryRequest(r.request, request); + if (!s.ok()) { + return s; + } + + pt::ptree results; + s = serializeQueryData(r.results, results); + if (!s.ok()) { + return s; + } + + tree.add_child("request", request); + tree.add_child("results", results); + + return Status(0, "OK"); +} + +Status serializeDistributedQueryResultJSON(const DistributedQueryResult& r, + std::string& json) { + pt::ptree tree; + auto s = serializeDistributedQueryResult(r, tree); + if (!s.ok()) { + return s; + } + std::stringstream ss; + try { + pt::write_json(ss, tree, false); + } catch (const pt::ptree_error& e) { + return Status(1, "Error serializing JSON: " + std::string(e.what())); + } + json = ss.str(); + + return Status(0, "OK"); +} + +Status deserializeDistributedQueryResult(const pt::ptree& tree, + DistributedQueryResult& r) { + DistributedQueryRequest request; + auto s = + deserializeDistributedQueryRequest(tree.get_child("request"), request); + if (!s.ok()) { + return s; + } + + QueryData results; + s = deserializeQueryData(tree.get_child("results"), results); + if (!s.ok()) { + return s; + } + + r.request = request; + r.results = results; + + return Status(0, "OK"); +} + +Status deserializeDistributedQueryResultJSON(const std::string& json, + DistributedQueryResult& r) { + std::stringstream ss(json); + pt::ptree tree; + try { + pt::read_json(ss, tree); + } catch (const pt::ptree_error& e) { + return Status(1, "Error serializing JSON: " + std::string(e.what())); + } + return deserializeDistributedQueryResult(tree, r); +} } diff --git a/osquery/distributed/tests/distributed_tests.cpp b/osquery/distributed/tests/distributed_tests.cpp index 3e0ec69b..0cc80cac 100644 --- a/osquery/distributed/tests/distributed_tests.cpp +++ b/osquery/distributed/tests/distributed_tests.cpp @@ -65,6 +65,111 @@ class DistributedTests : public testing::Test { std::string distributed_tls_write_endpoint_; }; +TEST_F(DistributedTests, test_serialize_distributed_query_request) { + DistributedQueryRequest r; + r.query = "foo"; + r.id = "bar"; + + pt::ptree tree; + auto s = serializeDistributedQueryRequest(r, tree); + EXPECT_TRUE(s.ok()); + EXPECT_EQ(tree.get("query"), "foo"); + EXPECT_EQ(tree.get("id"), "bar"); +} + +TEST_F(DistributedTests, test_deserialize_distributed_query_request) { + pt::ptree tree; + tree.put("query", "foo"); + tree.put("id", "bar"); + + DistributedQueryRequest r; + auto s = deserializeDistributedQueryRequest(tree, r); + EXPECT_TRUE(s.ok()); + EXPECT_EQ(r.query, "foo"); + EXPECT_EQ(r.id, "bar"); +} + +TEST_F(DistributedTests, test_deserialize_distributed_query_request_json) { + auto json = + "{" + " \"query\": \"foo\"," + " \"id\": \"bar\"" + "}"; + + DistributedQueryRequest r; + auto s = deserializeDistributedQueryRequestJSON(json, r); + EXPECT_TRUE(s.ok()); + EXPECT_EQ(r.query, "foo"); + EXPECT_EQ(r.id, "bar"); +} + +TEST_F(DistributedTests, test_serialize_distributed_query_result) { + DistributedQueryResult r; + r.request.query = "foo"; + r.request.id = "bar"; + + Row r1; + r1["foo"] = "bar"; + r.results = {r1}; + + pt::ptree tree; + auto s = serializeDistributedQueryResult(r, tree); + EXPECT_TRUE(s.ok()); + EXPECT_EQ(tree.get("request.query"), "foo"); + EXPECT_EQ(tree.get("request.id"), "bar"); + auto& results = tree.get_child("results"); + for (const auto& q : results) { + for (const auto& row : q.second) { + EXPECT_EQ(row.first, "foo"); + EXPECT_EQ(q.second.get(row.first), "bar"); + } + } +} + +TEST_F(DistributedTests, test_deserialize_distributed_query_result) { + pt::ptree request; + request.put("id", "foo"); + request.put("query", "bar"); + + pt::ptree row; + row.put("foo", "bar"); + pt::ptree results; + results.push_back(std::make_pair("", row)); + + pt::ptree query_result; + query_result.put_child("request", request); + query_result.put_child("results", results); + + DistributedQueryResult r; + auto s = deserializeDistributedQueryResult(query_result, r); + EXPECT_TRUE(s.ok()); + EXPECT_EQ(r.request.id, "foo"); + EXPECT_EQ(r.request.query, "bar"); + EXPECT_EQ(r.results[0]["foo"], "bar"); +} + +TEST_F(DistributedTests, test_deserialize_distributed_query_result_json) { + auto json = + "{" + " \"request\": {" + " \"id\": \"foo\"," + " \"query\": \"bar\"" + " }," + " \"results\": [" + " {" + " \"foo\": \"bar\"" + " }" + " ]" + "}"; + + DistributedQueryResult r; + auto s = deserializeDistributedQueryResultJSON(json, r); + EXPECT_TRUE(s.ok()); + EXPECT_EQ(r.request.id, "foo"); + EXPECT_EQ(r.request.query, "bar"); + EXPECT_EQ(r.results[0]["foo"], "bar"); +} + TEST_F(DistributedTests, test_workflow) { auto dist = Distributed(); auto s = dist.pullUpdates(); diff --git a/osquery/events/benchmarks/events_benchmarks.cpp b/osquery/events/benchmarks/events_benchmarks.cpp index 8e57ddf8..536c188c 100644 --- a/osquery/events/benchmarks/events_benchmarks.cpp +++ b/osquery/events/benchmarks/events_benchmarks.cpp @@ -58,9 +58,13 @@ class BenchmarkEventSubscriber } void clearRows() { + auto ee = expire_events_; + auto et = expire_time_; expire_events_ = true; expire_time_ = -1; getIndexes(0, 0); + expire_events_ = ee; + expire_time_ = et; } void benchmarkGet(int low, int high) { auto results = get(low, high); } @@ -113,7 +117,7 @@ BENCHMARK(EVENTS_add_events); static void EVENTS_retrieve_events(benchmark::State& state) { auto sub = std::make_shared(); - for (int i = 0; i < 10000; i++) { + for (int i = 0; i < 1000; i++) { sub->benchmarkAdd(i++); } @@ -125,8 +129,8 @@ static void EVENTS_retrieve_events(benchmark::State& state) { } BENCHMARK(EVENTS_retrieve_events) + ->ArgPair(0, 10) + ->ArgPair(0, 50) ->ArgPair(0, 100) - ->ArgPair(0, 500) - ->ArgPair(0, 1000) - ->ArgPair(0, 10000); + ->ArgPair(0, 1000); } diff --git a/osquery/events/events.cpp b/osquery/events/events.cpp index 3034ca2d..c7b4162b 100644 --- a/osquery/events/events.cpp +++ b/osquery/events/events.cpp @@ -21,7 +21,6 @@ #include #include "osquery/core/conversions.h" -#include "osquery/database/db_handle.h" namespace osquery { @@ -92,9 +91,8 @@ QueryData EventSubscriberPlugin::genTable(QueryContext& context) { // Store the optimize time such that it can be restored if the daemon is // restarted. - auto db = DBHandle::getInstance(); auto index_key = "optimize." + dbNamespace(); - db->Put(kEvents, index_key, std::to_string(optimize_time_)); + setDatabaseValue(kEvents, index_key, std::to_string(optimize_time_)); } return get(start, stop); @@ -135,7 +133,6 @@ void EventPublisherPlugin::fire(const EventContextRef& ec, EventTime time) { std::set EventSubscriberPlugin::getIndexes(EventTime start, EventTime stop, size_t list_key) { - auto db = DBHandle::getInstance(); auto index_key = "indexes." + dbNamespace(); std::set indexes; @@ -152,7 +149,7 @@ std::set EventSubscriberPlugin::getIndexes(EventTime start, std::string time_list; auto list_type = boost::lexical_cast(size); - db->Get(kEvents, index_key + "." + list_type, time_list); + getDatabaseValue(kEvents, index_key + "." + list_type, time_list); if (time_list.empty()) { // No events in this binning size. return indexes; @@ -240,7 +237,6 @@ std::set EventSubscriberPlugin::getIndexes(EventTime start, void EventSubscriberPlugin::expireRecords(const std::string& list_type, const std::string& index, bool all) { - auto db = DBHandle::getInstance(); auto record_key = "records." + dbNamespace(); auto data_key = "data." + dbNamespace(); @@ -250,7 +246,7 @@ void EventSubscriberPlugin::expireRecords(const std::string& list_type, auto expired_records = getRecords({list_type + "." + index}); for (const auto& record : expired_records) { if (all) { - db->Delete(kEvents, data_key + "." + record.first); + deleteDatabaseValue(kEvents, data_key + "." + record.first); } else if (record.second > expire_time_) { persisting_records.push_back(record.first + ":" + std::to_string(record.second)); @@ -259,10 +255,11 @@ void EventSubscriberPlugin::expireRecords(const std::string& list_type, // Either drop or overwrite the record list. if (all) { - db->Delete(kEvents, record_key + "." + list_type + "." + index); + deleteDatabaseValue(kEvents, record_key + "." + list_type + "." + index); } else { auto new_records = boost::algorithm::join(persisting_records, ","); - db->Put(kEvents, record_key + "." + list_type + "." + index, new_records); + setDatabaseValue( + kEvents, record_key + "." + list_type + "." + index, new_records); } } @@ -270,7 +267,6 @@ void EventSubscriberPlugin::expireIndexes( const std::string& list_type, const std::vector& indexes, const std::vector& expirations) { - auto db = DBHandle::getInstance(); auto index_key = "indexes." + dbNamespace(); // Construct a mutable list of persisting indexes to rewrite as records. @@ -285,16 +281,15 @@ void EventSubscriberPlugin::expireIndexes( // Update the list of indexes with the non-expired indexes. auto new_indexes = boost::algorithm::join(persisting_indexes, ","); - db->Put(kEvents, index_key + "." + list_type, new_indexes); + setDatabaseValue(kEvents, index_key + "." + list_type, new_indexes); } void EventSubscriberPlugin::expireCheck() { - auto db = DBHandle::getInstance(); auto data_key = "data." + dbNamespace(); auto eid_key = "eid." + dbNamespace(); std::vector keys; - db->ScanPrefix(kEvents, keys, data_key); + scanDatabaseKeys(kEvents, keys, data_key); if (keys.size() <= FLAGS_events_max) { return; } @@ -305,13 +300,13 @@ void EventSubscriberPlugin::expireCheck() { // Inspect the N-FLAGS_events_max -th event's value and expire before the // time within the content. std::string last_key; - db->Get(kEvents, eid_key, last_key); + getDatabaseValue(kEvents, eid_key, last_key); // The EID is the next-index. size_t max_key = boost::lexical_cast(last_key) - FLAGS_events_max - 1; // Convert the key index into a time using the content. std::string content; - db->Get(kEvents, data_key + "." + std::to_string(max_key), content); + getDatabaseValue(kEvents, data_key + "." + std::to_string(max_key), content); // Decode the value into a row structure to extract the time. Row r; @@ -332,7 +327,6 @@ void EventSubscriberPlugin::expireCheck() { std::vector EventSubscriberPlugin::getRecords( const std::set& indexes) { - auto db = DBHandle::getInstance(); auto record_key = "records." + dbNamespace(); std::vector records; @@ -340,7 +334,7 @@ std::vector EventSubscriberPlugin::getRecords( std::vector bin_records; { std::string record_value; - db->Get(kEvents, record_key + "." + index, record_value); + getDatabaseValue(kEvents, record_key + "." + index, record_value); if (record_value.empty()) { // There are actually no events in this bin, interesting error case. continue; @@ -364,7 +358,6 @@ std::vector EventSubscriberPlugin::getRecords( Status EventSubscriberPlugin::recordEvent(EventID& eid, EventTime time) { Status status; - auto db = DBHandle::getInstance(); std::string time_value = boost::lexical_cast(time); // The record is identified by the event type then module name. @@ -386,27 +379,29 @@ Status EventSubscriberPlugin::recordEvent(EventID& eid, EventTime time) { boost::lock_guard lock(event_record_lock_); // Append the record (eid, unix_time) to the list bin. std::string record_value; - status = db->Get( + status = getDatabaseValue( kEvents, record_key + "." + list_key + "." + list_id, record_value); if (record_value.length() == 0) { // This is a new list_id for list_key, append the ID to the indirect // lookup for this list_key. std::string index_value; - status = db->Get(kEvents, index_key + "." + list_key, index_value); + status = + getDatabaseValue(kEvents, index_key + "." + list_key, index_value); if (index_value.length() == 0) { // A new index. index_value = list_id; } else { index_value += "," + list_id; } - status = db->Put(kEvents, index_key + "." + list_key, index_value); + status = + setDatabaseValue(kEvents, index_key + "." + list_key, index_value); record_value = eid + ":" + time_value; } else { // Tokenize a record using ',' and the EID/time using ':'. record_value += "," + eid + ":" + time_value; } - status = db->Put( + status = setDatabaseValue( kEvents, record_key + "." + list_key + "." + list_id, record_value); if (!status.ok()) { LOG(ERROR) << "Could not put Event Record key: " << record_key; @@ -419,7 +414,6 @@ Status EventSubscriberPlugin::recordEvent(EventID& eid, EventTime time) { EventID EventSubscriberPlugin::getEventID() { Status status; - auto db = DBHandle::getInstance(); // First get an event ID from the meta key. std::string eid_key = "eid." + dbNamespace(); std::string last_eid_value; @@ -427,14 +421,14 @@ EventID EventSubscriberPlugin::getEventID() { { boost::lock_guard lock(event_id_lock_); - status = db->Get(kEvents, eid_key, last_eid_value); + status = getDatabaseValue(kEvents, eid_key, last_eid_value); if (!status.ok() || last_eid_value.empty()) { last_eid_value = "0"; } last_eid_ = boost::lexical_cast(last_eid_value) + 1; eid_value = boost::lexical_cast(last_eid_); - status = db->Put(kEvents, eid_key, eid_value); + status = setDatabaseValue(kEvents, eid_key, eid_value); } if (!status.ok()) { @@ -448,7 +442,6 @@ QueryData EventSubscriberPlugin::get(EventTime start, EventTime stop) { QueryData results; // Get the records for this time range. - auto db = DBHandle::getInstance(); auto indexes = getIndexes(start, stop); auto records = getRecords(indexes); std::string events_key = "data." + dbNamespace(); @@ -464,7 +457,7 @@ QueryData EventSubscriberPlugin::get(EventTime start, EventTime stop) { std::string data_value; for (const auto& record : mapped_records) { Row r; - auto status = db->Get(kEvents, record, data_value); + auto status = getDatabaseValue(kEvents, record, data_value); if (data_value.length() == 0) { // There is no record here, interesting error case. continue; @@ -485,7 +478,6 @@ QueryData EventSubscriberPlugin::get(EventTime start, EventTime stop) { } Status EventSubscriberPlugin::add(Row& r, EventTime event_time) { - auto db = DBHandle::getInstance(); // Get and increment the EID for this module. EventID eid = getEventID(); // Without encouraging a missing event time, do not support a 0-time. @@ -506,7 +498,7 @@ Status EventSubscriberPlugin::add(Row& r, EventTime event_time) { // Store the event data. std::string event_key = "data." + dbNamespace() + "." + eid; - status = db->Put(kEvents, event_key, data); + status = setDatabaseValue(kEvents, event_key, data); // Record the event in the indexing bins, using the index time. recordEvent(eid, event_time); return status; @@ -692,10 +684,9 @@ Status EventFactory::registerEventSubscriber(const PluginRef& sub) { // Restore optimize times for a daemon. if (kToolType == OSQUERY_TOOL_DAEMON && FLAGS_events_optimize) { - auto db = DBHandle::getInstance(); auto index_key = "optimize." + specialized_sub->dbNamespace(); std::string content; - if (db->Get(kEvents, index_key, content)) { + if (getDatabaseValue(kEvents, index_key, content)) { long long optimize_time = 0; safeStrtoll(content, 10, optimize_time); specialized_sub->optimize_time_ = static_cast(optimize_time); diff --git a/osquery/events/tests/events_database_tests.cpp b/osquery/events/tests/events_database_tests.cpp index f696cf21..986281db 100644 --- a/osquery/events/tests/events_database_tests.cpp +++ b/osquery/events/tests/events_database_tests.cpp @@ -19,8 +19,6 @@ #include #include -#include "osquery/database/db_handle.h" - namespace osquery { DECLARE_uint64(events_expiry); @@ -209,8 +207,8 @@ TEST_F(EventsDatabaseTests, test_gentable) { // The optimize time should have been written to the database. // It should be the same as the current (relative) optimize time. std::string content; - getDatabaseValue( - "events", "optimize.DBFakePublisher.DBFakeSubscriber", content); + getDatabaseValue("events", "optimize.DBFakePublisher.DBFakeSubscriber", + content); EXPECT_EQ(std::to_string(sub->optimize_time_), content); keys.clear(); diff --git a/osquery/extensions/interface.cpp b/osquery/extensions/interface.cpp index b1723780..1861c1ea 100644 --- a/osquery/extensions/interface.cpp +++ b/osquery/extensions/interface.cpp @@ -44,7 +44,6 @@ void ExtensionHandler::call(ExtensionResponse& _return, _return.status.code = status.getCode(); _return.status.message = status.getMessage(); _return.status.uuid = uuid_; - if (status.ok()) { for (const auto& response_item : response) { // Translate a PluginResponse to an ExtensionPluginResponse. @@ -176,18 +175,18 @@ bool ExtensionManagerHandler::exists(const std::string& name) { ExtensionRunnerCore::~ExtensionRunnerCore() { remove(path_); } void ExtensionRunnerCore::stop() { - boost::lock_guard lock(service_start_); - service_stopping_ = true; - if (transport_ != nullptr) { - transport_->interrupt(); + { + std::unique_lock lock(service_start_); + service_stopping_ = true; + if (transport_ != nullptr) { + transport_->interrupt(); + transport_->interruptChildren(); + } } - { - // In most cases the service thread has started before the stop request. - boost::lock_guard lock(service_run_); - if (server_ != nullptr) { - server_->stop(); - } + // In most cases the service thread has started before the stop request. + if (server_ != nullptr) { + server_->stop(); } } @@ -202,7 +201,7 @@ inline void removeStalePaths(const std::string& manager) { void ExtensionRunnerCore::startServer(TProcessorRef processor) { { - boost::lock_guard lock(service_start_); + std::unique_lock lock(service_start_); // A request to stop the service may occur before the thread starts. if (service_stopping_) { return; @@ -221,10 +220,7 @@ void ExtensionRunnerCore::startServer(TProcessorRef processor) { processor, transport_, transport_fac, protocol_fac)); } - { - boost::lock_guard lock(service_run_); - server_->serve(); - } + server_->serve(); } void ExtensionRunner::start() { @@ -243,7 +239,7 @@ void ExtensionRunner::start() { ExtensionManagerRunner::~ExtensionManagerRunner() { // Only attempt to remove stale paths if the server was started. - boost::lock_guard lock(service_start_); + std::unique_lock lock(service_start_); if (server_ != nullptr) { removeStalePaths(path_); } diff --git a/osquery/extensions/interface.h b/osquery/extensions/interface.h index 8fca136a..786062ad 100644 --- a/osquery/extensions/interface.h +++ b/osquery/extensions/interface.h @@ -261,11 +261,7 @@ class ExtensionRunnerCore : public InternalRunnable { TThreadedServerRef server_{nullptr}; /// Protect the service start and stop, this mutex protects server creation. - boost::mutex service_start_; - - private: - /// Protect the service start and stop, this mutex protects the transport. - boost::mutex service_run_; + std::mutex service_start_; /// Record a dispatcher's request to stop the service. bool service_stopping_{false}; diff --git a/osquery/logger/CMakeLists.txt b/osquery/logger/CMakeLists.txt index 6df50464..0ad3a229 100644 --- a/osquery/logger/CMakeLists.txt +++ b/osquery/logger/CMakeLists.txt @@ -1,19 +1,16 @@ -ADD_OSQUERY_LIBRARY(TRUE osquery_logger - logger.cpp -) +file(GLOB OSQUERY_LOGGER "*.cpp") +ADD_OSQUERY_LIBRARY(TRUE osquery_logger ${OSQUERY_LOGGER}) + add_dependencies(osquery_logger libglog) -ADD_OSQUERY_LIBRARY(FALSE osquery_logger_plugins - plugins/filesystem.cpp - plugins/tls.cpp - plugins/syslog.cpp -) +file(GLOB OSQUERY_LOGGER_PLUGINS "plugins/*.cpp") +ADD_OSQUERY_LIBRARY(FALSE osquery_logger_plugins ${OSQUERY_LOGGER_PLUGINS}) + +file(GLOB OSQUERY_LOGGER_TESTS "tests/*.cpp") +ADD_OSQUERY_TEST(TRUE ${OSQUERY_LOGGER_TESTS}) # Keep the logger testing in the additional to test filesystem logging. # There is a significant difference between the Glog-backed filesystem plugin # and other, which use a Glog sink. They must be tested in tandem. -file(GLOB OSQUERY_LOGGER_TESTS "tests/*.cpp") -ADD_OSQUERY_TEST(TRUE ${OSQUERY_LOGGER_TESTS}) - file(GLOB OSQUERY_LOGGER_PLUGIN_TESTS "plugins/tests/*.cpp") ADD_OSQUERY_TEST(FALSE ${OSQUERY_LOGGER_PLUGIN_TESTS}) diff --git a/osquery/logger/plugins/tls.cpp b/osquery/logger/plugins/tls.cpp index 2b2d5ccf..9b191215 100644 --- a/osquery/logger/plugins/tls.cpp +++ b/osquery/logger/plugins/tls.cpp @@ -8,21 +8,20 @@ * */ -#include #include +#include -#include #include +#include #include #include #include #include "osquery/remote/requests.h" -#include "osquery/remote/transports/tls.h" #include "osquery/remote/serializers/json.h" +#include "osquery/remote/transports/tls.h" #include "osquery/remote/utility.h" -#include "osquery/database/db_handle.h" #include "osquery/logger/plugins/tls.h" @@ -144,8 +143,7 @@ Status TLSLogForwarderRunner::send(std::vector& log_data, // Read each logged line into JSON and populate a list of lines. // The result list will use the 'data' key. pt::ptree children; - iterate(log_data, - ([&children](std::string& item) { + iterate(log_data, ([&children](std::string& item) { pt::ptree child; try { std::stringstream input; @@ -169,21 +167,16 @@ Status TLSLogForwarderRunner::send(std::vector& log_data, } void TLSLogForwarderRunner::check() { - // Instead of using the 'help' database API, prefer to interact with the - // DBHandle directly for additional performance. - auto handle = DBHandle::getInstance(); - // Get a list of all the buffered log items, with a max of 1024 lines. std::vector indexes; - auto status = handle->Scan(kLogs, indexes, kTLSMaxLogLines); + auto status = scanDatabaseKeys(kLogs, indexes, kTLSMaxLogLines); // For each index, accumulate the log line into the result or status set. std::vector results, statuses; - iterate(indexes, - ([&handle, &results, &statuses](std::string& index) { + iterate(indexes, ([&results, &statuses](std::string& index) { std::string value; auto& target = ((index.at(0) == 'r') ? results : statuses); - if (handle->Get(kLogs, index, value)) { + if (getDatabaseValue(kLogs, index, value)) { // Enforce a max log line size for TLS logging. if (value.size() > FLAGS_logger_tls_max) { LOG(WARNING) << "Line exceeds TLS logger max: " << value.size(); @@ -201,8 +194,7 @@ void TLSLogForwarderRunner::check() { << status.getMessage() << ")"; } else { // Clear the results logs once they were sent. - iterate(indexes, - ([&results](std::string& index) { + iterate(indexes, ([&results](std::string& index) { if (index.at(0) != 'r') { return; } @@ -218,8 +210,7 @@ void TLSLogForwarderRunner::check() { << status.getMessage() << ")"; } else { // Clear the status logs once they were sent. - iterate(indexes, - ([&results](std::string& index) { + iterate(indexes, ([&results](std::string& index) { if (index.at(0) != 's') { return; } diff --git a/osquery/main/tests.cpp b/osquery/main/tests.cpp index a6de8f25..3b652ada 100644 --- a/osquery/main/tests.cpp +++ b/osquery/main/tests.cpp @@ -19,5 +19,8 @@ int main(int argc, char* argv[]) { testing::InitGoogleTest(&argc, argv); // Optionally enable Goggle Logging // google::InitGoogleLogging(argv[0]); - return RUN_ALL_TESTS(); + auto result = RUN_ALL_TESTS(); + + osquery::shutdownTesting(); + return result; } diff --git a/osquery/sql/benchmarks/sql_benchmarks.cpp b/osquery/sql/benchmarks/sql_benchmarks.cpp index 84783068..a1e2a086 100644 --- a/osquery/sql/benchmarks/sql_benchmarks.cpp +++ b/osquery/sql/benchmarks/sql_benchmarks.cpp @@ -62,6 +62,40 @@ static void SQL_virtual_table_internal(benchmark::State& state) { BENCHMARK(SQL_virtual_table_internal); +static void SQL_virtual_table_internal_global(benchmark::State& state) { + Registry::add("table", "benchmark"); + PluginResponse res; + Registry::call("table", "benchmark", {{"action", "columns"}}, res); + + while (state.KeepRunning()) { + // Get a connection to the persistent database. + auto dbc = SQLiteDBManager::get(); + attachTableInternal("benchmark", columnDefinition(res), dbc); + + QueryData results; + queryInternal("select * from benchmark", results, dbc->db()); + } +} + +BENCHMARK(SQL_virtual_table_internal_global); + +static void SQL_virtual_table_internal_unique(benchmark::State& state) { + Registry::add("table", "benchmark"); + PluginResponse res; + Registry::call("table", "benchmark", {{"action", "columns"}}, res); + + while (state.KeepRunning()) { + // Get a new database connection (to a unique database). + auto dbc = SQLiteDBManager::getUnique(); + attachTableInternal("benchmark", columnDefinition(res), dbc); + + QueryData results; + queryInternal("select * from benchmark", results, dbc->db()); + } +} + +BENCHMARK(SQL_virtual_table_internal_unique); + class BenchmarkLongTablePlugin : public TablePlugin { private: TableColumns columns() const { @@ -83,7 +117,7 @@ static void SQL_virtual_table_internal_long(benchmark::State& state) { Registry::call("table", "long_benchmark", {{"action", "columns"}}, res); // Attach a sample virtual table. - auto dbc = SQLiteDBManager::get(); + auto dbc = SQLiteDBManager::getUnique(); attachTableInternal("long_benchmark", columnDefinition(res), dbc); while (state.KeepRunning()) { @@ -123,7 +157,7 @@ static void SQL_virtual_table_internal_wide(benchmark::State& state) { Registry::call("table", "wide_benchmark", {{"action", "columns"}}, res); // Attach a sample virtual table. - auto dbc = SQLiteDBManager::get(); + auto dbc = SQLiteDBManager::getUnique(); attachTableInternal("wide_benchmark", columnDefinition(res), dbc); while (state.KeepRunning()) { diff --git a/osquery/sql/sqlite_util.cpp b/osquery/sql/sqlite_util.cpp index be1b3ac0..85ac0374 100644 --- a/osquery/sql/sqlite_util.cpp +++ b/osquery/sql/sqlite_util.cpp @@ -9,8 +9,8 @@ */ #include -#include #include +#include #include #include "osquery/sql/sqlite_util.h" @@ -49,6 +49,13 @@ const std::map kSQLiteReturnCodes = { {101, "SQLITE_DONE"}, }; +const std::map kMemoryDBSettings = { + {"synchronous", "OFF"}, {"count_changes", "OFF"}, + {"default_temp_store", "0"}, {"auto_vacuum", "FULL"}, + {"journal_mode", "OFF"}, {"cache_size", "0"}, + {"page_count", "0"}, +}; + #define OpComparator(x) \ { x, QueryPlanner::Opcode(OpReg::P2, INTEGER_TYPE) } #define Arithmetic(x) \ @@ -140,9 +147,19 @@ SQLiteDBInstance::SQLiteDBInstance(sqlite3*& db, std::mutex& mtx) } } +static inline void openOptimized(sqlite3*& db) { + sqlite3_open(":memory:", &db); + + std::string settings; + for (const auto& setting : kMemoryDBSettings) { + settings += "PRAGMA " + setting.first + "=" + setting.second + "; "; + } + sqlite3_exec(db, settings.c_str(), nullptr, nullptr, nullptr); +} + void SQLiteDBInstance::init() { primary_ = false; - sqlite3_open(":memory:", &db_); + openOptimized(db_); } void SQLiteDBInstance::addAffectedTable(VirtualTableContent* table) { @@ -175,7 +192,7 @@ SQLiteDBInstance::~SQLiteDBInstance() { } SQLiteDBManager::SQLiteDBManager() : db_(nullptr) { - sqlite3_soft_heap_limit64(SQLITE_SOFT_HEAP_LIMIT); + sqlite3_soft_heap_limit64(1); setDisabledTables(Flag::getValue("disable_tables")); } @@ -202,7 +219,7 @@ SQLiteDBInstanceRef SQLiteDBManager::getConnection(bool primary) { if (self.db_ == nullptr) { // Create primary SQLite DB instance. - sqlite3_open(":memory:", &self.db_); + openOptimized(self.db_); self.connection_ = SQLiteDBInstanceRef(new SQLiteDBInstance(self.db_)); attachVirtualTables(self.connection_); } @@ -301,7 +318,6 @@ Status queryInternal(const std::string& q, QueryData& results, sqlite3* db) { sqlite3_free(err); return Status(1, "Error running query: " + error_string); } - return Status(0, "OK"); } diff --git a/osquery/sql/virtual_table.h b/osquery/sql/virtual_table.h index 48824b3d..3846a8da 100644 --- a/osquery/sql/virtual_table.h +++ b/osquery/sql/virtual_table.h @@ -10,6 +10,8 @@ #pragma once +#include + #include #include "osquery/core/conversions.h" @@ -22,7 +24,7 @@ namespace osquery { * * Only used in the SQLite virtual table module methods. */ -struct BaseCursor { +struct BaseCursor : private boost::noncopyable { /// SQLite virtual table cursor. sqlite3_vtab_cursor base; /// Track cursors for optional planner output. @@ -41,7 +43,7 @@ struct BaseCursor { * Only used in the SQLite virtual table module methods. * This adds each table plugin class to the state tracking in SQLite. */ -struct VirtualTable { +struct VirtualTable : private boost::noncopyable { /// The SQLite-provided virtual table structure. sqlite3_vtab base; diff --git a/osquery/tables/system/linux/processes.cpp b/osquery/tables/system/linux/processes.cpp index 54a031d6..52abe4a1 100644 --- a/osquery/tables/system/linux/processes.cpp +++ b/osquery/tables/system/linux/processes.cpp @@ -8,8 +8,8 @@ * */ -#include #include +#include #include #include @@ -17,9 +17,9 @@ #include #include -#include #include #include +#include #include "osquery/core/conversions.h" @@ -143,7 +143,7 @@ void genProcessMap(const std::string& pid, QueryData& results) { // BSS with name in pathname. r["pseudo"] = (fields[4] == "0" && !r["path"].empty()) ? "1" : "0"; - results.push_back(r); + results.push_back(std::move(r)); } } diff --git a/third-party b/third-party index bd9d658f..46791746 160000 --- a/third-party +++ b/third-party @@ -1 +1 @@ -Subproject commit bd9d658ff4fbc642384376663c1f9ea4f66be7be +Subproject commit 467917469b31f567f5f6a755641cfe8179a68394 diff --git a/tools/tests/stress.py b/tools/tests/stress.py index eb4eac29..055bb44d 100755 --- a/tools/tests/stress.py +++ b/tools/tests/stress.py @@ -87,15 +87,21 @@ def audit(args): def single(args): start_time = time.time() - proc = subprocess.Popen(args, - shell=True, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE) + if ARGS.verbose: + proc = subprocess.Popen(args, shell=True) + else: + proc = subprocess.Popen(args, + shell=True, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE) + if ARGS.verbose: + print("PID: %d" % (proc.pid)) stdout, stderr = proc.communicate() end_time = time.time() - start_time if proc.returncode is not 0: - print(stdout) - print(stderr) + if not ARGS.verbose: + print(stdout) + print(stderr) print("%s Test failed. (total %6.4fs)" % ( red("FAILED"), end_time)) sys.exit(proc.returncode) @@ -129,14 +135,16 @@ if __name__ == "__main__": help="Arguments to pass to test binary") parser.add_argument("--stat", action="store_true", default=False, help="Only print numerical values") + parser.add_argument("--verbose", action="store_true", default=False, + help="Do not consume stderr/stdout") parser.add_argument("run", nargs="?", help="Run specific test binary") - args = parser.parse_args() + ARGS = parser.parse_args() # A baseline was requested, first run baselines then normal. - if args.baseline: + if ARGS.baseline: print("Running baseline tests...") - stress(vars(args)) - args.baseline = False + stress(vars(ARGS)) + ARGS.baseline = False print("Finished. Running tests...") - stress(vars(args)) + stress(vars(ARGS)) diff --git a/tools/tests/test_base.py b/tools/tests/test_base.py index 6b17b059..d3512a16 100644 --- a/tools/tests/test_base.py +++ b/tools/tests/test_base.py @@ -165,8 +165,10 @@ class ProcRunner(object): pid = 0 try: if self.silent: - self.proc = subprocess.Popen([self.path] + self.args, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.proc = subprocess.Popen( + [self.path] + self.args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) else: self.proc = subprocess.Popen([self.path] + self.args) pid = self.proc.pid diff --git a/tools/tests/test_extensions.py b/tools/tests/test_extensions.py index 345fac9e..21eb6d98 100755 --- a/tools/tests/test_extensions.py +++ b/tools/tests/test_extensions.py @@ -303,6 +303,7 @@ class ExtensionTests(test_base.ProcessGenerator, unittest.TestCase): # Now start a daemon daemon = self._run_daemon({ "disable_watchdog": True, + "verbose": True, "extensions_timeout": EXTENSION_TIMEOUT, "extensions_socket": extension.options["extensions_socket"], }) diff --git a/tools/tests/test_osqueryi.py b/tools/tests/test_osqueryi.py index adb57628..b6283d3b 100755 --- a/tools/tests/test_osqueryi.py +++ b/tools/tests/test_osqueryi.py @@ -69,7 +69,7 @@ class OsqueryiTest(unittest.TestCase): self.assertEqual(proc.proc.poll(), 0) @test_base.flaky - def test_config_check_failure(self): + def test_config_check_failure_invalid_path(self): '''Test that a missing config fails''' proc = test_base.TimeoutRunner([ self.binary, @@ -83,6 +83,8 @@ class OsqueryiTest(unittest.TestCase): print(proc.stderr) self.assertEqual(proc.proc.poll(), 1) + @test_base.flaky + def test_config_check_failure_valid_path(self): # Now with a valid path, but invalid content. proc = test_base.TimeoutRunner([ self.binary, @@ -94,6 +96,8 @@ class OsqueryiTest(unittest.TestCase): self.assertEqual(proc.proc.poll(), 1) self.assertNotEqual(proc.stderr, "") + @test_base.flaky + def test_config_check_failure_missing_plugin(self): # Finally with a missing config plugin proc = test_base.TimeoutRunner([ self.binary,